From 509301518038f6644794a41efd196b228ebbf46a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 11:22:54 -0500 Subject: [PATCH] feat(amd): port DSR1 FP4 to srt-slurm --- .../dsr1/mi355x/fp4-disagg-fixed-seq.yaml | 485 ++++++++++++++++++ configs/amd-master.yaml | 214 ++++++++ perf-changelog.yaml | 15 + 3 files changed, 714 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml new file mode 100644 index 0000000000..ba0e13364f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml @@ -0,0 +1,485 @@ +base: + name: "mi355x-dsr1-fp4-disagg-fixed-seq" + + model: + path: "/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + container: "sglang-rocm-v0.5.17-mi35x" + precision: "fp4" + + identity: + model: {repo: "amd/DeepSeek-R1-0528-MXFP4-v2"} + container: {image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809"} + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260809+g7120f3ee13" + + slurm: {time_limit: "08:00:00"} + + resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + + frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + + backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + GLOO_SOCKET_IFNAME: "eno0" + NCCL_SOCKET_IFNAME: "eno0" + MORI_RDMA_TC: "104" + MORI_IO_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_SHMEM_MODE: "ISOLATION" + MORI_ENABLE_SDMA: "false" + MORI_EP_LAUNCH_CONFIG_MODE: "AUTO" + SGLANG_USE_AITER: "1" + AITER_LOG_LEVEL: "ERROR" + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_ENABLE_OVERLAP_PLAN_STREAM: "0" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "8192" + SGLANG_MORI_MOE_MAX_INPUT_TOKENS: "32768" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "1024" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + decode_environment: + <<: *common_environment + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MORI_MOE_MAX_INPUT_TOKENS: "2703" + sglang_config: + prefill: &tp8_stp + served-model-name: "amd/DeepSeek-R1-0528-MXFP4-v2" + trust-remote-code: true + tp-size: 8 + disaggregation-transfer-backend: mori + kv-cache-dtype: fp8_e4m3 + attention-backend: aiter + moe-dense-tp-size: 1 + load-balance-method: round_robin + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.80 + max-running-requests: 128 + chunked-prefill-size: 16384 + disable-radix-cache: true + cuda-graph-bs: &cg128 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + enable-metrics: true + enable-cache-report: true + decode: + <<: *tp8_stp + mem-fraction-static: 0.85 + chunked-prefill-size: 262144 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: *cg128 + + sbatch_directives: + cpus-per-task: "128" + mem: "0" + + srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + + health_check: {max_attempts: 720, interval_seconds: 5} + + benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}/fixed-seq" + mkdir -p "${result_root}" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + chat_args=() + if [[ "${USE_CHAT_TEMPLATE}" == "1" ]]; then + chat_args+=(--use-chat-template) + fi + for concurrency in ${CONCURRENCY_LIST}; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then num_prompts=16; fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model amd/DeepSeek-R1-0528-MXFP4-v2 \ + --tokenizer /model \ + --trust-remote-code \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + "${chat_args[@]}" \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}" \ + --result-filename "dsr1-fp4-${BENCHMARK_VARIANT}-isl8192-osl1024-c${concurrency}.json" + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + CONCURRENCY_LIST: "1 2 4 8" + BENCHMARK_VARIANT: "stp-1p1d-tp8" + USE_CHAT_TEMPLATE: "0" + +override_stp_1p1d_tp8: + name: "mi355x-dsr1-fp4-stp-1p1d-tp8-fixed-seq" + benchmark: + env: + CONCURRENCY_LIST: "1 2 4 8" + BENCHMARK_VARIANT: "stp-1p1d-tp8" + +override_stp_1p2d_tp8: + name: "mi355x-dsr1-fp4-stp-1p2d-tp8-fixed-seq" + resources: + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + benchmark: + env: + CONCURRENCY_LIST: "2 4 8 16 32 64 128 256" + BENCHMARK_VARIANT: "stp-1p2d-tp8" + +override_stp_1p2d_tp4_tp8: + name: "mi355x-dsr1-fp4-stp-1p2d-tp4-tp8-fixed-seq" + resources: + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 4 + gpus_per_decode: 8 + backend: + sglang_config: + prefill: + tp-size: 4 + benchmark: + env: + CONCURRENCY_LIST: "64 128 256" + BENCHMARK_VARIANT: "stp-1p2d-tp4-tp8" + +override_stp_1p1d_dep8: + name: "mi355x-dsr1-fp4-stp-1p1d-dep8-fixed-seq" + backend: + prefill_environment: + MORI_ENABLE_SDMA: "true" + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "64" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "128" + sglang_config: + prefill: &dep8_stp_c512 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + max-running-requests: 512 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + <<: *dep8_stp_c512 + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: &cg512 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512] + benchmark: + env: + CONCURRENCY_LIST: "128 256 512" + BENCHMARK_VARIANT: "stp-1p1d-dep8" + +override_stp_2p1d_dep8: + name: "mi355x-dsr1-fp4-stp-2p1d-dep8-fixed-seq" + resources: + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + backend: + prefill_environment: + MORI_ENABLE_SDMA: "true" + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "1024" + sglang_config: + prefill: &dep8_stp_c4096 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + max-running-requests: 4096 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + <<: *dep8_stp_c4096 + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: *cg512 + benchmark: + env: + CONCURRENCY_LIST: "1024 2048 4096" + BENCHMARK_VARIANT: "stp-2p1d-dep8" + +override_mtp3_1p1d_tp8: + name: "mi355x-dsr1-fp4-mtp3-1p1d-tp8-fixed-seq" + backend: + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "2048" + sglang_config: + prefill: &tp8_mtp3 + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + decode: + <<: *tp8_mtp3 + benchmark: + env: + CONCURRENCY_LIST: "1 2 4 8" + BENCHMARK_VARIANT: "mtp3-1p1d-tp8" + USE_CHAT_TEMPLATE: "1" + +override_mtp3_1p2d_tp8_wide: + name: "mi355x-dsr1-fp4-mtp3-1p2d-tp8-wide-fixed-seq" + resources: + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + backend: + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "2048" + sglang_config: + prefill: &mtp3_tp8 + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + decode: + <<: *mtp3_tp8 + benchmark: + env: + CONCURRENCY_LIST: "2 4 8 16 32 64" + BENCHMARK_VARIANT: "mtp3-1p2d-tp8-wide" + USE_CHAT_TEMPLATE: "1" + +override_mtp3_1p2d_tp8_narrow: + name: "mi355x-dsr1-fp4-mtp3-1p2d-tp8-narrow-fixed-seq" + resources: + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + backend: + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "2048" + sglang_config: + prefill: *mtp3_tp8 + decode: *mtp3_tp8 + benchmark: + env: + CONCURRENCY_LIST: "2 4 8 16 32" + BENCHMARK_VARIANT: "mtp3-1p2d-tp8-narrow" + USE_CHAT_TEMPLATE: "1" + +override_mtp2_1p2d_tp8: + name: "mi355x-dsr1-fp4-mtp2-1p2d-tp8-fixed-seq" + resources: + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + backend: + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1536" + sglang_config: + prefill: &mtp2_tp8 + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 2 + speculative-num-draft-tokens: 3 + decode: + <<: *mtp2_tp8 + benchmark: + env: + CONCURRENCY_LIST: "64 128 256" + BENCHMARK_VARIANT: "mtp2-1p2d-tp8" + USE_CHAT_TEMPLATE: "1" + +override_mtp3_1p1d_dep8: + name: "mi355x-dsr1-fp4-mtp3-1p1d-dep8-fixed-seq" + backend: + prefill_environment: + MORI_ENABLE_SDMA: "true" + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "320" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "160" + sglang_config: + prefill: &dep8_mtp3_c640 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + max-running-requests: 640 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + <<: *dep8_mtp3_c640 + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: *cg512 + benchmark: + env: + CONCURRENCY_LIST: "64 128 256 512 640" + BENCHMARK_VARIANT: "mtp3-1p1d-dep8" + USE_CHAT_TEMPLATE: "1" + +override_mtp1_1p1d_dep8: + name: "mi355x-dsr1-fp4-mtp1-1p1d-dep8-fixed-seq" + backend: + prefill_environment: + MORI_ENABLE_SDMA: "true" + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "128" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "128" + sglang_config: + prefill: &dep8_mtp1_c512 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 1 + speculative-num-draft-tokens: 2 + max-running-requests: 512 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + <<: *dep8_mtp1_c512 + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: *cg512 + benchmark: + env: + CONCURRENCY_LIST: "64 128 256 512" + BENCHMARK_VARIANT: "mtp1-1p1d-dep8" + USE_CHAT_TEMPLATE: "1" + +override_mtp1_2p1d_dep8: + name: "mi355x-dsr1-fp4-mtp1-2p1d-dep8-fixed-seq" + resources: + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + backend: + prefill_environment: + MORI_ENABLE_SDMA: "true" + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "1024" + sglang_config: + prefill: &dep8_mtp1_c4096 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 1 + speculative-num-draft-tokens: 2 + max-running-requests: 4096 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + <<: *dep8_mtp1_c4096 + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: *cg512 + benchmark: + env: + CONCURRENCY_LIST: "1024 2048 4096" + BENCHMARK_VARIANT: "mtp1-2p1d-dep8" + USE_CHAT_TEMPLATE: "1" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b47a07488e..eab63d0975 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -856,3 +856,217 @@ qwen3-0.6b-fp16-mi355x-sglang-srt-disagg: # Production-scale migration of the existing Qwen3.5 FP8 MI355X P/D lane to # srt-slurm. One full TP8 node serves prefill, one TP8 node serves decode, and # the request/KV planes remain native SGLang Router + AMD MoRI. + +dsr1-fp4-mi355x-sglang-disagg: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: amd/DeepSeek-R1-0528-MXFP4-v2 + model-prefix: dsr1 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "none" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p1d_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "none" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p2d_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "none" + conc-list: [1] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p2d_tp4_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "none" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} + - spec-decoding: "none" + conc-list: [1] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_2p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} + +dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: amd/DeepSeek-R1-0528-MXFP4-v2 + model-prefix: dsr1 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p1d_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p2d_tp8_wide" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp1_2p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} + +dsr1-fp4-mi355x-sglang-disagg-mtp: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: amd/DeepSeek-R1-0528-MXFP4-v2 + model-prefix: dsr1 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p1d_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p2d_tp8_narrow" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp2_1p2d_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp1_1p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp1_2p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f94f370b83..d83fd7297d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6558,3 +6558,18 @@ - "Use the native SGLang router for request routing and AMD MoRI for disaggregated KV transfer, without Dynamo, NIXL, or the deleted amd_utils control plane." - "Remove the active and deprecated MI355X matrix entries that depended on amd_utils; configuration-specific child PRs reintroduce the active production lanes as native srt-slurm recipes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 + +- config-keys: + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp + - dsr1-fp4-mi355x-sglang-disagg-mtp + scenario-type: + - fixed-seq-len + description: + - "Replace the DeepSeek-R1 MXFP4 model-specific amd_utils launcher with native srt-slurm orchestration using SGLang Router and AMD MoRI on MI355X." + - "Preserve the complete production STP and MTP 8k1k search space as 12 explicit runtime selectors: TP8 and DEP8/DPA 1P1D, 1P2D, and 2P1D topologies spanning c1 through c4096 with one-, two-, and three-step NEXTN variants." + - "Use lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809, the existing shared 78-shard MXFP4 checkpoint, FP8 KV cache, AITer attention, role-specific MoRI tuning, and the SGLang/DeepSeek-R1-NextN draft model." + - "Run the repository's unchanged benchmark_serving.py through srt-slurm's custom benchmark contract, with one allocation per topology and tuning family to amortize model initialization across each concurrency sweep." + - "Validate local production checkpoints before submission and prefetch optional external draft repositories once into the canonical shared Hugging Face cache instead of downloading them independently on every prefill and decode worker." + - "Pin SemiAnalysisAI/srt-slurm PR #7 at 10a58d2ebb4d756c423424049c736b909b72e14f so DP-attention SGLang workers inherit the validated P/D rank-routing contract, multi-node rank-offset preservation, and ATOM recipe flag normalization." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2633