From 83e2635a1822d8ce9b7c4e2d06beb7d1e6820182 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:26:23 +0800 Subject: [PATCH 01/20] perf(agentx): refresh GB300 DSV4 with DSpark6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: - The GB300 DeepSeek-V4-Pro AgentX matrix still used the older EAGLE/MTP serving path and an outdated SGLang nightly. - The previous draft targeted vLLM recipes instead of the existing SGLang topology ladder. Fix: - Replace the SGLang aggregate and disaggregated GB300 recipes with DSpark block size 6 on DeepSeek-V4-Pro-DSpark. - Use nightly-dev-cu13-20260816-4a6dc267, which already includes the required DSpark performance and MegaMoE support, without a runtime patch overlay. - Force thinking-on AL 3.82 with static ragged verification, retain MegaMoE on DEP workers, and set the 8P4D throughput endpoint to concurrency 1536. Validation: - Passed srtctl dry-run for all seven recipes. - Passed 224 matrix and schema tests. - Generated the eight targeted matrix jobs at concurrency 1, 4, 8, 32, 192, 400, 640, and 1536. 中文：将 GB300 DeepSeek-V4-Pro AgentX 的 SGLang 配置从 EAGLE/MTP 更新为 DSpark6，使用已包含所需上游修复的 20260816 nightly，不再注入补丁；固定 AL 3.82 与 static ragged verify，并将 8P4D 吞吐端点设为并发 1536。七份 recipe dry-run、224 项矩阵与 schema 测试以及八个目标矩阵任务生成均已通过。 --- .../agentic/agg-gb300-tp4-mtp-kvoffload.yaml | 156 ------------------ .../agentic/agg-gb300-tp4-mtp-lowlatency.yaml | 121 ++++++++++++++ .../agentic/agg-gb300-tp8-mtp-lowlatency.yaml | 121 ++++++++++++++ ...b300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml} | 50 ++++-- ...0-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml} | 46 ++++-- ...0-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml} | 46 ++++-- ...0-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml} | 46 ++++-- ...-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml} | 54 +++--- configs/nvidia-master.yaml | 71 ++++---- perf-changelog.yaml | 12 ++ 10 files changed, 450 insertions(+), 273 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml => disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml => disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml => disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml => disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml => disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml} (82%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml deleted file mode 100644 index 7bb82d2b5..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "agg-gb300-tp4-mtp-kvoffload" - -# Agentic-coding SGLang aggregated recipe for DeepSeek-V4-Pro on GB300 -# (single aggregated worker, TP4, MTP + hierarchical-cache KV offload). -# -# Uses the flat single-variant schema the agentic CI flow expects. Concurrency is NOT -# a recipe field here: the GHA matrix fans out one job per concurrency from -# the master-config conc-list, exporting CONC into agentic_srt.sh. Modeled on -# the flat vllm/deepseek-v4/agentic recipes (agentic infra + benchmark wiring) -# and the flat sglang/deepseek-v4/8k1k recipes (sglang backend schema). - -model: - path: "deepseek-v4-pro" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - # Dedicated etcd/nats node — matches the vllm agentic recipes; under the - # dynamo router + multiple frontends the infra services need their own node. - etcd_nats_dedicated_node: true - # cc-traces prompts (~125K tokens) serialize to ~2.8MB; the 1 MiB NATS - # default drops them. 32 MiB matches the agentic vllm recipes (~10x headroom - # over the largest observed payload); schema recommends 24+ for long ISL. - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - # The sglang image is PEP 668 externally-managed; the runtime dynamo - # install (dynamo_wheels.py / source build) runs pip and fails with - # "externally-managed-environment" without this. Lets pip install into - # the system env. Applies to both dynamo.hash (source) and dynamo.wheel. - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - aggregated_environment: - SGLANG_DEFAULT_THINKING: '1' - # TP4/bs128 can spend over an hour capturing the DSV4 EAGLE draft-extend - # graph. Keep the target/decode graphs and use the upstream eager fallback. - SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. - PIP_BREAK_SYSTEM_PACKAGES: "1" - # from submission for B300 - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - # for kvcache offload - SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" - - sglang_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - enable-metrics: true - trust-remote-code: true - stream-interval: 50 - watchdog-timeout: 1000000 - mem-fraction-static: 0.90 - chunked-prefill-size: 8192 # can only support 11000 for TP - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 128 - cuda-graph-max-bs: 128 - - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 4 - ep-size: 1 - # kv-cache offload - enable-hierarchical-cache: true - hicache-ratio: 6 - hicache-write-policy: write_through - hicache-io-backend: kernel - # mtp section - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user into the container as a non-root - # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to - # uid 0 inside the container. srt-slurm renders empty-string values as - # flag-only srun args. - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - # Aggregated: one worker serves both prefill and decode, so GPU accounting - # is the single-worker form num_gpus = TP (not the disagg prefill+decode - # sum). The multinode post-processing path assumes disagg and would divide - # throughput by prefill_gpus + decode_gpus; force the single-node code path - # in process_agentic_result.py instead. TP must match sglang_config tp-size. - IS_MULTINODE: "false" - TP: "4" - # Stamp X-Dynamo-Session-ID on every turn so both nginx and Dynamo's - # router keep the session on the worker that owns its KV prefix. - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - # Container-side path of the aiperf mmap dataset cache; host-side mount - # wired via launch_gb300-*.sh srtslurm.yaml default_mounts. Without it, - # aiperf re-tokenizes + re-writes the dataset mmap on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also via default_mounts) so the trace dataset - # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml new file mode 100644 index 000000000..7addf32ae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml @@ -0,0 +1,121 @@ +name: "agg-gb300-tp4-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP4 worker occupies one +# four-GPU GB300 node and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-dspark" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + aggregated_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + enable-metrics: true + trust-remote-code: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 4 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_WARMUP_REQUESTS_PER_LANE: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml new file mode 100644 index 000000000..287a5d1fe --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml @@ -0,0 +1,121 @@ +name: "agg-gb300-tp8-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP8 worker spans two +# four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-dspark" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + aggregated_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + enable-metrics: true + trust-remote-code: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 8 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_WARMUP_REQUESTS_PER_LANE: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml index e41438765..f86d91f39 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload" +name: "disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 1D x TP4, MTP + hierarchical-cache KV offload), tuned for concurrency 80. +# (2P x DEP8 / 1D x TP4, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 32. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -57,6 +63,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -90,13 +100,14 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - # Match the TP4 aggregate worker: avoid the pathological DSV4 EAGLE + # Match the TP4 aggregate worker: avoid the pathological DSV4 speculative # draft-extend graph capture while retaining the target/decode graphs. SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -130,7 +141,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -146,18 +157,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 128 cuda-graph-max-bs: 128 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -165,7 +177,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -187,10 +199,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -214,3 +227,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml index 74b371e94..2e783f7e2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" +name: "disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 256. +# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 192. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -57,6 +63,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -90,10 +100,11 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,7 +138,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -143,18 +154,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 256 cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,7 +174,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -185,10 +197,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -212,3 +225,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml index 6b585fba8..fc2ca94b4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload" +name: "disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload" # Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 -# (4P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 512. +# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 400. # # DEP8-prefill variant aligned with the measured Pareto point: prefill uses # tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. @@ -9,13 +9,19 @@ name: "disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload" # from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -56,6 +62,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -89,10 +99,11 @@ backend: SGLANG_DSV4_MHC_PREWARM: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,7 +138,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -143,18 +154,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 256 cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,7 +174,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -185,10 +197,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -212,3 +225,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml index b2cac21c7..04a9de4da 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload" +name: "disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (6P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 768. +# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 640. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -57,6 +63,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -90,10 +100,11 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,7 +138,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -143,18 +154,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 512 cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,7 +174,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -185,10 +197,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -212,3 +225,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml similarity index 82% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml index 709962936..7a2b5d20b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload" +name: "disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (12P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 1536. +# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1536. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -27,9 +33,9 @@ health_check: resources: gpu_type: gb300 gpus_per_node: 4 - prefill_nodes: 12 + prefill_nodes: 8 decode_nodes: 4 - prefill_workers: 6 + prefill_workers: 4 decode_workers: 1 gpus_per_prefill: 8 gpus_per_decode: 16 @@ -60,7 +66,11 @@ backend: type: sglang prefill_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -93,10 +103,11 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -130,7 +141,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -146,18 +157,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 1024 cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -165,7 +177,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -184,14 +196,15 @@ backend: mem-fraction-static: 0.9 swa-full-tokens-ratio: 0.02 max-running-requests: 3072 - cuda-graph-max-bs: 1024 + cuda-graph-max-bs: 192 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -215,3 +228,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 62ba9cff3..493480536 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7692,33 +7692,42 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: dp-attn: true dsv4-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 - model: deepseek-ai/DeepSeek-V4-Pro + image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 + model: deepseek-ai/DeepSeek-V4-Pro-DSpark model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev20260718" } + router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } multinode: true disagg: false scenarios: agentic-coding: - - dram-utilization: 0.80 - search-space: - - spec-decoding: mtp - conc-list: [2, 4, 8, 16] - kv-offloading: dram - kv-offload-backend: { name: hicache } + - search-space: + - spec-decoding: draft_model + conc-list: [1, 4] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + - search-space: + - spec-decoding: draft_model + conc-list: [8] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml" - # Aggregated worker: prefill's GPUs also serve decode, so decode has no - # separate allocation (num-worker: 0). Deployment is recipe-driven - # (agg_workers: 1); this only makes GPU accounting count 4, not 4+4. + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml" decode: num-worker: 0 tp: 4 @@ -7726,13 +7735,13 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: dp-attn: false dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 - model: deepseek-ai/DeepSeek-V4-Pro + image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 + model: deepseek-ai/DeepSeek-V4-Pro-DSpark model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev20260718" } + router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -7740,8 +7749,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: agentic-coding: - dram-utilization: 0.80 search-space: - - spec-decoding: mtp - conc-list: [80] + - spec-decoding: draft_model + conc-list: [32] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7750,14 +7759,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - - spec-decoding: mtp - conc-list: [256] + - spec-decoding: draft_model + conc-list: [192] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7766,14 +7775,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp - conc-list: [512] + - spec-decoding: draft_model + conc-list: [400] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7782,14 +7791,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp - conc-list: [768] + - spec-decoding: draft_model + conc-list: [640] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7798,23 +7807,23 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp + - spec-decoding: draft_model conc-list: [1536] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: - num-worker: 6 + num-worker: 4 tp: 8 ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index da3f8d6dd..b431fe0ff 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6018,3 +6018,15 @@ description: - "Refresh the MiniMax-M3 B200 AgentX submission: no-offload TP8 C1 and TP4 C1/C5/C10/C15/C20, plus a TP4 SimpleCPU KV-offload sweep at C15–C40 using the full-capacity DRAM budget." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2611 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Replace the GB300 DeepSeek-V4-Pro AgentX EAGLE/MTP serving path with DSpark block size 6 across two aggregate and five disaggregated recipes" + - "Use DeepSeek-V4-Pro-DSpark with SGLANG_RAGGED_VERIFY_MODE=static, MegaMoE for DEP workers, and forced thinking-on acceptance length 3.82" + - "Refresh SGLang to nightly-dev-cu13-20260816-4a6dc267, which already contains the required DSpark performance and MegaMoE support" + - "Retune the Pareto ladder to concurrency 1/4/8, 32, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 From 8a4a0a0d29931011f2aa21edc0b14939b80c356f Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:33:18 +0800 Subject: [PATCH 02/20] fix(agentx): keep DSpark synthetic AL out of eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: - The InferenceX port hard-coded SGLANG_SIMULATE_ACC_* in every recipe instead of carrying NVIDIA InferenceMAX PR #214 throughput-only master-config injection. - Eval-only launch therefore skipped the injector but still inherited synthetic acceptance from the checked-in recipe. - The DSpark AL 3.82 source curve and the focused injection regression tests were also omitted. Fix: - Keep SGLANG_RAGGED_VERIFY_MODE=static in all six recipes while moving synthetic AL 3.82 into each throughput search-space entry. - Commit the probabilistic DeepSeek-V4-Pro-DSpark golden curve from Actions run 31279568355 and reference it from the bilingual index and performance changelog. - Add tests covering all six recipes, throughput injection, eval preservation, launcher ordering, DSpark6 parameters, tokenizer paths, and the committed golden value. Validation: - Passed matrix, schema, changelog, runner, and synthetic-acceptance tests. - Generated seven throughput points and four eval-only points at the intended concurrencies. - Passed YAML parsing, changelog validation, formatting checks, and git diff checks. 中文：将模拟接受长度改为仅在吞吐测试启动时注入，精度评测继续使用真实 DSpark 验证；同时补充黄金 AL 曲线、模型路径映射和回归测试，并删除表现不佳的 c32 点。 --- .../agentic/agg-gb300-tp4-mtp-lowlatency.yaml | 3 - .../agentic/agg-gb300-tp8-mtp-lowlatency.yaml | 3 - ...gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml | 230 ------------------ ...00-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml | 6 - ...00-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml | 6 - ...00-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml | 6 - ...0-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml | 6 - configs/nvidia-master.yaml | 28 +-- golden_al_distribution/README.md | 1 + golden_al_distribution/README_zh.md | 1 + .../dsv4_dspark_probabilistic.yaml | 28 +++ perf-changelog.yaml | 6 +- runners/launch_gb300-nv.sh | 6 + runners/test_synthetic_acceptance.py | 134 ++++++++++ 14 files changed, 185 insertions(+), 279 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml create mode 100644 golden_al_distribution/dsv4_dspark_probabilistic.yaml create mode 100644 runners/test_synthetic_acceptance.py diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml index 7addf32ae..7cfd733f9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml @@ -55,9 +55,6 @@ backend: type: sglang aggregated_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DEFAULT_THINKING: "1" SGLANG_DSV4_REASONING_EFFORT: high PIP_BREAK_SYSTEM_PACKAGES: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml index 287a5d1fe..e6bc12780 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml @@ -55,9 +55,6 @@ backend: type: sglang aggregated_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DEFAULT_THINKING: "1" SGLANG_DSV4_REASONING_EFFORT: high PIP_BREAK_SYSTEM_PACKAGES: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml deleted file mode 100644 index f86d91f39..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml +++ /dev/null @@ -1,230 +0,0 @@ -name: "disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 1D x TP4, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 32. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-dspark" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" - -dynamo: - install: true - hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - - decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - # Match the TP4 aggregate worker: avoid the pathological DSV4 speculative - # draft-extend graph capture while retaining the target/decode graphs. - SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '2048' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - - sglang_config: - prefill: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark - enable-metrics: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - swa-full-tokens-ratio: 0.02 - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark - enable-metrics: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 4 - dp-size: 1 - ep-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - moe-runner-backend: flashinfer_mxfp4 - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - swa-full-tokens-ratio: 0.02 - max-running-requests: 128 - cuda-graph-max-bs: 128 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml index 2e783f7e2..976a6c998 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml @@ -64,9 +64,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -101,9 +98,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml index fc2ca94b4..364cd582d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml @@ -63,9 +63,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -100,9 +97,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml index 04a9de4da..79ecbb704 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml @@ -64,9 +64,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -101,9 +98,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml index 7a2b5d20b..d587144c8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml @@ -67,9 +67,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -104,9 +101,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 493480536..1fb76115b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7712,6 +7712,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml" decode: num-worker: 0 @@ -7727,6 +7729,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml" decode: num-worker: 0 @@ -7749,22 +7753,6 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: agentic-coding: - dram-utilization: 0.80 search-space: - - spec-decoding: draft_model - conc-list: [32] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - spec-decoding: draft_model conc-list: [192] kv-offloading: dram @@ -7775,6 +7763,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -7791,6 +7781,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -7807,6 +7799,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -7823,6 +7817,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" decode: num-worker: 1 diff --git a/golden_al_distribution/README.md b/golden_al_distribution/README.md index 95e4a5622..2713f39a6 100644 --- a/golden_al_distribution/README.md +++ b/golden_al_distribution/README.md @@ -109,6 +109,7 @@ Before accepting an updated curve, reviewers should verify: | Model | Method | Golden YAML | Source run | | --- | --- | --- | --- | | DeepSeek V4 Pro | MTP | [`dsv4_mtp.yaml`](dsv4_mtp.yaml) | [27180633016](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27180633016) | +| DeepSeek V4 Pro DSpark | DSpark (probabilistic drafting) | [`dsv4_dspark_probabilistic.yaml`](dsv4_dspark_probabilistic.yaml) | [31279568355](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355) | | Qwen3.5 397B-A17B | MTP | [`qwen3.5_mtp.yaml`](qwen3.5_mtp.yaml) | [27317114007](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27317114007) | | Kimi K2.5 | EAGLE3 | [`kimik2.5_eagle3.yaml`](kimik2.5_eagle3.yaml) | [28122195822](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28122195822) | | MiniMax-M3 | EAGLE3 | [`minimaxm3_eagle3.yaml`](minimaxm3_eagle3.yaml) | [28061204145](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28061204145) | diff --git a/golden_al_distribution/README_zh.md b/golden_al_distribution/README_zh.md index a56a005f2..baf680612 100644 --- a/golden_al_distribution/README_zh.md +++ b/golden_al_distribution/README_zh.md @@ -109,6 +109,7 @@ gh workflow run speedbench-al.yml \ | 模型 | 方法 | 黄金 YAML | 源 run | | --- | --- | --- | --- | | DeepSeek V4 Pro | MTP | [`dsv4_mtp.yaml`](dsv4_mtp.yaml) | [27180633016](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27180633016) | +| DeepSeek V4 Pro DSpark | DSpark（概率式草稿采样） | [`dsv4_dspark_probabilistic.yaml`](dsv4_dspark_probabilistic.yaml) | [31279568355](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355) | | Qwen3.5 397B-A17B | MTP | [`qwen3.5_mtp.yaml`](qwen3.5_mtp.yaml) | [27317114007](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27317114007) | | Kimi K2.5 | EAGLE3 | [`kimik2.5_eagle3.yaml`](kimik2.5_eagle3.yaml) | [28122195822](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28122195822) | | MiniMax-M3 | EAGLE3 | [`minimaxm3_eagle3.yaml`](minimaxm3_eagle3.yaml) | [28061204145](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28061204145) | diff --git a/golden_al_distribution/dsv4_dspark_probabilistic.yaml b/golden_al_distribution/dsv4_dspark_probabilistic.yaml new file mode 100644 index 000000000..516aeb2ab --- /dev/null +++ b/golden_al_distribution/dsv4_dspark_probabilistic.yaml @@ -0,0 +1,28 @@ +# Source GitHub Actions run: https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355 +# Acceptance Length (AL) reference values measured with SPEED-Bench. +# dataset: coding | temperature: 1.0 | output_len: 4096 +# thinking_on chat_template_kwargs: {"thinking": true, "reasoning_effort": "high"} +# speculative-config: method=dspark | draft_sample_method=probabilistic +# Measured on deepseek-v4-pro-dspark (B300, vLLM DSpark), per num_speculative_tokens. +# Auto-generated by benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh (speedbench-al.yml). +# +# key = num_speculative_tokens (DSpark level); value = golden AL +deepseek-v4-pro-dspark: + thinking_on: + 1: 1.86 + 2: 2.55 + 3: 3.08 + 4: 3.43 + 5: 3.74 + 6: 3.82 + 7: 3.86 + 8: 3.75 + thinking_off: + 1: 1.93 + 2: 2.76 + 3: 3.47 + 4: 4.10 + 5: 4.52 + 6: 4.87 + 7: 4.94 + 8: 4.94 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b431fe0ff..c8f3121ec 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6025,8 +6025,8 @@ scenario-type: - agentic-coding description: - - "Replace the GB300 DeepSeek-V4-Pro AgentX EAGLE/MTP serving path with DSpark block size 6 across two aggregate and five disaggregated recipes" - - "Use DeepSeek-V4-Pro-DSpark with SGLANG_RAGGED_VERIFY_MODE=static, MegaMoE for DEP workers, and forced thinking-on acceptance length 3.82" + - "Replace the GB300 DeepSeek-V4-Pro AgentX EAGLE/MTP serving path with DSpark block size 6 across two aggregate and four disaggregated recipes" + - "Use DeepSeek-V4-Pro-DSpark with SGLANG_RAGGED_VERIFY_MODE=static, MegaMoE for DEP workers, and throughput-only thinking-on acceptance length 3.82 from the committed probabilistic DSpark curve (golden_al_distribution/dsv4_dspark_probabilistic.yaml, run 31279568355); eval keeps real DSpark verification" - "Refresh SGLang to nightly-dev-cu13-20260816-4a6dc267, which already contains the required DSpark performance and MegaMoE support" - - "Retune the Pareto ladder to concurrency 1/4/8, 32, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" + - "Retune the Pareto ladder to concurrency 1/4/8, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 83252461d..cc583fb7f 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -44,6 +44,12 @@ elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export SERVED_MODEL_NAME="deepseek-r1-fp8" export MODEL_PATH=/scratch/models/DeepSeek-R1-0528 export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" +elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $MODEL == "deepseek-ai/DeepSeek-V4-Pro-DSpark" ]]; then + # The DSpark checkpoint is staged in the shared runner cache visible from + # the GB300 compute nodes. Keep a distinct alias from the legacy MTP + # checkpoint so other DSv4 lanes retain their node-local model mapping. + export MODEL_PATH="$HF_HUB_CACHE_HOST_PATH/inferencex-models/DeepSeek-V4-Pro-DSpark" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-dspark" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then # Use the node-local /scratch SSD for the 806 GB DSv4-Pro # checkpoint. Faster than the Vast NFS path, but this dir only diff --git a/runners/test_synthetic_acceptance.py b/runners/test_synthetic_acceptance.py new file mode 100644 index 000000000..874c87ec6 --- /dev/null +++ b/runners/test_synthetic_acceptance.py @@ -0,0 +1,134 @@ +import os +import subprocess +import sys +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +INJECTOR = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py" +LAUNCHER = REPO_ROOT / "runners" / "launch_gb300-nv.sh" +MASTER = REPO_ROOT / "configs" / "nvidia-master.yaml" +RECIPES = REPO_ROOT / "benchmarks" / "multi_node" / "srt-slurm-recipes" +GOLDEN_AL = REPO_ROOT / "golden_al_distribution" / "dsv4_dspark_probabilistic.yaml" +CONFIG_KEYS = ( + "dsv4-fp4-gb300-dynamo-sglang-agentic-agg", + "dsv4-fp4-gb300-dynamo-sglang-agentic-disagg", +) + + +def run_injector( + recipe: Path, *, eval_only: bool, acceptance_length: str = "3.82" +) -> subprocess.CompletedProcess[str]: + env = { + **os.environ, + "EVAL_ONLY": str(eval_only).lower(), + "SYNTHETIC_ACCEPTANCE": "true", + "SYNTHETIC_ACCEPTANCE_LENGTH": acceptance_length, + } + return subprocess.run( + [sys.executable, str(INJECTOR), str(recipe), "dynamo-sglang"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + +def _configured_recipe_paths(master: dict) -> set[Path]: + recipe_paths = set() + for config_key in CONFIG_KEYS: + assert master[config_key]["model"] == "deepseek-ai/DeepSeek-V4-Pro-DSpark" + for group in master[config_key]["scenarios"]["agentic-coding"]: + for point in group["search-space"]: + assert point["spec-decoding"] == "draft_model" + settings = point["prefill"]["additional-settings"] + assert "SYNTHETIC_ACCEPTANCE=true" in settings + assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" in settings + config = next( + item for item in settings if item.startswith("CONFIG_FILE=") + ) + recipe_paths.add(RECIPES / config.removeprefix("CONFIG_FILE=recipes/")) + return recipe_paths + + +def test_sglang_injection_is_throughput_only(tmp_path: Path) -> None: + original = " prefill_environment:\n A: B\n decode_environment:\n C: D\n" + + throughput_recipe = tmp_path / "throughput.yaml" + throughput_recipe.write_text(original) + result = run_injector(throughput_recipe, eval_only=False) + assert result.returncode == 0, result.stderr + assert throughput_recipe.read_text().count("SGLANG_SIMULATE_ACC_LEN") == 2 + + eval_recipe = tmp_path / "eval.yaml" + eval_recipe.write_text(original) + result = run_injector(eval_recipe, eval_only=True) + assert result.returncode == 0, result.stderr + assert eval_recipe.read_text() == original + + +def test_gb300_dsv4_configs_inject_only_at_launch() -> None: + master = yaml.safe_load(MASTER.read_text()) + recipe_paths = _configured_recipe_paths(master) + + assert len(recipe_paths) == 6 + for recipe in recipe_paths: + text = recipe.read_text() + parsed = yaml.safe_load(text) + worker_count = 1 if recipe.name.startswith("agg-") else 2 + assert "SGLANG_SIMULATE_ACC_" not in text + assert text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count + assert text.count("speculative-algorithm: DSPARK") == worker_count + assert text.count("speculative-dspark-block-size: 6") == worker_count + assert text.count("speculative-num-draft-tokens: 7") == worker_count + assert "speculative-algorithm: EAGLE" not in text + assert 'path: "deepseek-v4-pro-dspark"' in text + assert parsed["benchmark"]["env"]["INFMAX_AIPERF_TOKENIZER"] == "/model" + assert "AIPERF_TOKENIZER" not in parsed["benchmark"]["env"] + + launcher = LAUNCHER.read_text() + source = 'source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh"' + inject = 'inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1' + apply = "SRTCTL_OUTPUT=$(srtctl apply" + assert launcher.index(source) < launcher.index(inject) < launcher.index(apply) + + +def test_gb300_dsv4_dspark6_matches_committed_golden_curve() -> None: + golden = yaml.safe_load(GOLDEN_AL.read_text()) + assert golden["deepseek-v4-pro-dspark"]["thinking_on"][6] == 3.82 + + master = yaml.safe_load(MASTER.read_text()) + for config_key in CONFIG_KEYS: + for group in master[config_key]["scenarios"]["agentic-coding"]: + for point in group["search-space"]: + settings = point["prefill"]["additional-settings"] + assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" in settings + + +def test_gb300_dsv4_dspark6_uses_golden_acceptance_only_for_throughput( + tmp_path: Path, +) -> None: + master = yaml.safe_load(MASTER.read_text()) + for recipe in _configured_recipe_paths(master): + original = recipe.read_text() + worker_count = 1 if recipe.name.startswith("agg-") else 2 + + throughput_recipe = tmp_path / recipe.name + throughput_recipe.write_text(original) + result = run_injector(throughput_recipe, eval_only=False) + assert result.returncode == 0, result.stderr + throughput_text = throughput_recipe.read_text() + assert throughput_text.count('SGLANG_SIMULATE_ACC_LEN: "3.82"') == worker_count + assert ( + throughput_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count + ) + + eval_recipe = tmp_path / f"eval-{recipe.name}" + eval_recipe.write_text(original) + result = run_injector(eval_recipe, eval_only=True) + assert result.returncode == 0, result.stderr + eval_text = eval_recipe.read_text() + assert eval_text == original + assert "SGLANG_SIMULATE_ACC_" not in eval_text + assert eval_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count From 927f3d38fd83f7908a3d55ffa33612ea9a017af4 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:51:09 +0800 Subject: [PATCH 03/20] perf(b300): add DSpark6 Dynamo AgentX recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: SemiAnalysis InferenceX lacked the validated B300 Dynamo-SGLang DSpark6 AgentX configurations, including session-affinity routing and sufficient host HiCache capacity for DEP4 and DEP8.\n\nFix: add DEP4/DEP8 aggregate recipes, the B300 master sweep, DSpark checkpoint selection, srt-slurm v1.0.38 overlay, serialized image imports, workspace/tokenizer mounts, and the shared synthetic-acceptance injector used by GB300.\n\nValidation: bash syntax, YAML parsing, and git diff whitespace checks pass.\n\n根因：SemiAnalysis InferenceX 缺少已验证的 B300 Dynamo-SGLang DSpark6 AgentX 配置，包括会话亲和路由，以及 DEP4/DEP8 所需的主机端 HiCache 容量。\n\n修复：新增 DEP4/DEP8 聚合 recipe 与 B300 master sweep，并加入 DSpark checkpoint 选择、srt-slurm v1.0.38 overlay、镜像导入串行化、workspace/tokenizer 挂载，以及与 GB300 共用的合成验收率注入逻辑。\n\n验证：Bash 语法、YAML 解析及 git diff 空白检查均通过。 --- benchmarks/benchmark_lib.sh | 5 + .../agentic/agg-b300-dep4-mtp-kvoffload.yaml | 131 ++++++++++++++++++ .../agentic/agg-b300-dep8-mtp-kvoffload.yaml | 129 +++++++++++++++++ configs/nvidia-master.yaml | 51 +++++++ runners/launch_b300-nv.sh | 75 +++++++++- 5 files changed, 386 insertions(+), 5 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index e7bdf2754..1a4b6db64 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -2071,6 +2071,11 @@ build_replay_cmd() { # from vLLM) is unaffected by this flag and still gives us KV usage, # prefix cache hit rate, etc. REPLAY_CMD+=" --no-gpu-telemetry" + # A staged checkpoint can provide the tokenizer without touching the + # shared Hugging Face cache. This is required for local-dir model mounts. + if [[ -n "${INFMAX_AIPERF_TOKENIZER:-}" ]]; then + REPLAY_CMD+=" --tokenizer $INFMAX_AIPERF_TOKENIZER" + fi # aiperf's dataset manager (separate from the inference parser) loads # the model's tokenizer for trace-prompt tokenization regardless of # --use-server-token-count. Models like kimi (amd/Kimi-K2.5-MXFP4, diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml new file mode 100644 index 000000000..2909f6eb7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml @@ -0,0 +1,131 @@ +name: "agg-b300-dep4-mtp-kvoffload" + +model: + path: "deepseek-v4-pro-dspark" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "b300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + # Let srt-slurm allocate and register the worker-local publisher endpoint; + # Dynamo needs discovery metadata in addition to the SGLang CLI flag. + kv_events_config: + aggregated: true + aggregated_environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + trust-remote-code: true + enable-metrics: true + # DSpark draft weights raise the measured minimum viable fraction to + # 0.9258 on B300; use the same 0.94 fraction as the GB300 TP4 recipe. + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + max-running-requests: 128 + cuda-graph-max-bs: 32 + stream-interval: 20 + incremental-streaming-output: true + watchdog-timeout: 1800 + dp-size: 4 + tp-size: 4 + ep-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + load-balance-method: total_tokens + moe-a2a-backend: megamoe + enable-hierarchical-cache: true + # DEP4's smaller device KV pool otherwise leaves only ~28 GB of host + # prefix cache per node; ratio 75 raises it to ~700 GB, matching the + # effective per-session capacity of the validated DEP8 configuration. + hicache-ratio: 75 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + weight-loader-drop-cache-after-load: true + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml new file mode 100644 index 000000000..e5d9beaea --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -0,0 +1,129 @@ +name: "agg-b300-dep8-mtp-kvoffload" + +model: + path: "deepseek-v4-pro-dspark" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "b300" + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + # Let srt-slurm allocate and register the worker-local publisher endpoint; + # Dynamo needs discovery metadata in addition to the SGLang CLI flag. + kv_events_config: + aggregated: true + aggregated_environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + trust-remote-code: true + enable-metrics: true + mem-fraction-static: 0.90 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + max-running-requests: 1152 + cuda-graph-max-bs: 144 + stream-interval: 20 + incremental-streaming-output: true + watchdog-timeout: 1800 + dp-size: 8 + tp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + load-balance-method: total_tokens + moe-a2a-backend: megamoe + enable-hierarchical-cache: true + # Ratio 1.75 completes c576, but its busiest DP rank reaches 99.4% host + # cache usage. Retry 2.0 with the smaller DSpark SWA pool (0.02) to add + # enough prefix-cache headroom without approaching the 2.5 startup OOM. + hicache-ratio: 2.0 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + weight-loader-drop-cache-after-load: true + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1fb76115b..ea2f18935 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8555,6 +8555,57 @@ glm5.1-fp8-b200-tilert: dp-attn: false additional-settings: - "DECODE_NODES=1" + +dsv4-fp4-b300-dynamo-sglang-agentic-mtp-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 + model: deepseek-ai/DeepSeek-V4-Pro-DSpark + model-prefix: dsv4 + runner: cluster:b300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.95 + search-space: + - spec-decoding: draft_model + conc-list: [48, 64] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml" + decode: + num-worker: 0 + tp: 4 + ep: 4 + dp-attn: true + - spec-decoding: draft_model + conc-list: [128, 256, 384, 512, 576] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 8 + dp-attn: true - isl: 8192 osl: 1024 search-space: diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index f7878a8ad..827c684ed 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -1,5 +1,7 @@ #!/usr/bin/bash +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" + # System-specific configuration for B300 NV Slurm cluster (sa-shared) SLURM_PARTITION="batch_1" SLURM_ACCOUNT="benchmark" @@ -41,6 +43,9 @@ elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo- fi export MODEL_PATH="${SELECTED_MODEL_PATH:-/data/models/dsv4-pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" +elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" && $MODEL == "deepseek-ai/DeepSeek-V4-Pro-DSpark" ]]; then + export MODEL_PATH="${MODEL_PATH:-/data/models/DeepSeek-V4-Pro-DSpark}" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-dspark" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" ]]; then export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" @@ -70,7 +75,17 @@ if [ -d "$SRT_REPO_DIR" ]; then fi # TODO(CJQ): make first class upon srt-slurm upstream refactor -if [[ "$IS_AGENTIC" == "1" ]]; then +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then + # Match the GB300 DSV4 AgentX path: v1.0.38 provides --no-preflight, + # session-affinity frontend support, backend metric injection for custom + # benchmarks, and the current aggregated Dynamo-SGLang recipe schema. + git clone --branch v1.0.38 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + mkdir -p recipes/sglang/deepseek-v4/agentic + cp -R \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/." \ + recipes/sglang/deepseek-v4/agentic +elif [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then @@ -135,9 +150,45 @@ NGINX_IMAGE="nginx:1.27.4" SQUASH_FILE="/data/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" NGINX_SQUASH_FILE="/data/squash/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" -# Import containers via enroot -srun -N 1 -A $SLURM_ACCOUNT -p $SLURM_PARTITION bash -c "enroot import -o $SQUASH_FILE docker://$IMAGE" -srun -N 1 -A $SLURM_ACCOUNT -p $SLURM_PARTITION bash -c "enroot import -o $NGINX_SQUASH_FILE docker://$NGINX_IMAGE" +# Import containers via enroot. Perform the validity check on the allocated +# compute node, not before submitting srun: a job can wait in the queue while +# another runner creates the same shared squash file. Serialize imports so a +# concurrent cache miss cannot race on the output either. +import_squash() { + local squash_file="$1" + local image_ref="$2" + + srun -N 1 -A "$SLURM_ACCOUNT" -p "$SLURM_PARTITION" \ + bash -s -- "$squash_file" "$image_ref" <<'IMPORT_SQUASH' +set -euo pipefail +squash_file="$1" +image_ref="$2" +lock_dir="$(dirname "$squash_file")/.locks" +mkdir -p "$lock_dir" +lock_file="$lock_dir/$(basename "$squash_file").lock" + +( + flock -w 3600 9 || { + echo "Failed to acquire lock for $squash_file" >&2 + exit 1 + } + if unsquashfs -l "$squash_file" > /dev/null 2>&1; then + echo "Squash file already exists and is valid, skipping import: $squash_file" + else + rm -f "$squash_file" + enroot import -o "$squash_file" "docker://$image_ref" + unsquashfs -l "$squash_file" > /dev/null 2>&1 || { + echo "enroot import did not produce a valid squash file: $squash_file" >&2 + exit 1 + } + chmod a+r "$squash_file" || true + fi +) 9>"$lock_file" +IMPORT_SQUASH +} + +import_squash "$SQUASH_FILE" "$IMAGE" +import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" export ISL="$ISL" export OSL="$OSL" @@ -171,6 +222,7 @@ containers: use_exclusive_sbatch_directive: true default_mounts: "/opt/ucx-no-ud": "/usr/local/ucx" + "${GITHUB_WORKSPACE}": "/infmax-workspace" EOF echo "Generated srtslurm.yaml:" @@ -200,6 +252,11 @@ fi # Override the job name in the recipe with the runner name. sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" + +# Throughput recipes opt into synthetic acceptance through the master config. +# Eval-only jobs leave the checked-in real-MTP recipe unchanged so generated +# tokens still pass target-model verification. +inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 if [[ "$MODEL_PREFIX" == "minimaxm3" && -n "$MINIMAX_M3_SLURM_EXCLUDED_NODELIST" ]]; then sed -i "/^name:.*/a sbatch_directives:\n exclude: \"${MINIMAX_M3_SLURM_EXCLUDED_NODELIST}\"" "$CONFIG_PATH" fi @@ -483,9 +540,17 @@ else export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-480}" - salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT -N 1 --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME" + if ! salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT -N 1 --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME"; then + echo "Failed to allocate B300 Slurm resources for account=$SLURM_ACCOUNT partition=$SLURM_PARTITION" >&2 + exit 1 + fi JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1) + if [[ -z "$JOB_ID" ]]; then + echo "Slurm allocation succeeded but no job ID was found for $RUNNER_NAME" >&2 + exit 1 + fi + srun --jobid=$JOB_ID \ --mpi=none \ --container-image=$SQUASH_FILE \ From e9c00d83ca6f26a31de2c9210e6b7b29e2b44f95 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:52:08 +0800 Subject: [PATCH 04/20] docs(perf): register B300 DSpark6 AgentX sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: the new B300 performance key was not yet registered in the append-only performance changelog.\n\nFix: append the B300 DSpark6 Dynamo-SGLang AgentX configuration and PR link.\n\nValidation: YAML parsing and git diff whitespace checks pass.\n\n根因：新的 B300 性能配置键尚未登记到只追加的性能变更日志。\n\n修复：追加 B300 DSpark6 Dynamo-SGLang AgentX 配置及 PR 链接。\n\n验证：YAML 解析及 git diff 空白检查均通过。 --- perf-changelog.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c8f3121ec..b2d718b28 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6030,3 +6030,12 @@ - "Refresh SGLang to nightly-dev-cu13-20260816-4a6dc267, which already contains the required DSpark performance and MegaMoE support" - "Retune the Pareto ladder to concurrency 1/4/8, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 + +- config-keys: + - dsv4-fp4-b300-dynamo-sglang-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Add B300 Dynamo-SGLang DSpark6 AgentX aggregate DEP4 and DEP8 configurations with session-affinity routing, HiCache host offload, an 8K prefill window, and golden synthetic acceptance length 3.82 for throughput; eval retains real verification." + - "Use the August 16 SGLang CUDA 13 nightly and srt-slurm v1.0.38, with DEP4 concurrency 48-64 and DEP8 concurrency 128-576." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2624 From e28fa42d5a5a544e90bcf4470e840fc6f155a6f6 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:31:33 +0800 Subject: [PATCH 05/20] fix(b300): make DSpark6 AgentX PR independent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: the B300 PR was stacked on the open GB300 DSpark6 branch, so its non-main base prevented the performance sweep workflow from triggering.\n\nFix: restore every GB300-only recipe, runner, golden curve, master entry, and changelog entry to main while retaining only the B300 DEP4/DEP8 configuration and support changes.\n\nValidation: the resulting diff against main contains only six B300-related files; Bash syntax, YAML parsing, and whitespace checks pass.\n\n根因：B300 PR 堆叠在尚未合并的 GB300 DSpark6 分支上，非 main base 导致性能 sweep 工作流无法触发。\n\n修复：将所有 GB300 专属 recipe、runner、golden curve、master 和 changelog 改动恢复为 main，仅保留 B300 DEP4/DEP8 配置及必要支持。\n\n验证：相对 main 的最终差异仅包含六个 B300 相关文件；Bash 语法、YAML 解析及空白检查均通过。 --- .../agentic/agg-gb300-tp4-mtp-kvoffload.yaml | 156 +++++++++++++ .../agentic/agg-gb300-tp4-mtp-lowlatency.yaml | 118 ---------- .../agentic/agg-gb300-tp8-mtp-lowlatency.yaml | 118 ---------- ...12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml} | 52 ++--- ...gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml | 216 ++++++++++++++++++ ...0-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml} | 42 ++-- ...0-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml} | 42 ++-- ...0-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml} | 42 ++-- configs/nvidia-master.yaml | 93 ++++---- golden_al_distribution/README.md | 1 - golden_al_distribution/README_zh.md | 1 - .../dsv4_dspark_probabilistic.yaml | 28 --- perf-changelog.yaml | 12 - runners/launch_gb300-nv.sh | 6 - runners/test_synthetic_acceptance.py | 134 ----------- 15 files changed, 489 insertions(+), 572 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml => disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml} (84%) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml => disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml} (86%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml => disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml} (86%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml => disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml} (86%) delete mode 100644 golden_al_distribution/dsv4_dspark_probabilistic.yaml delete mode 100644 runners/test_synthetic_acceptance.py diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml new file mode 100644 index 000000000..7bb82d2b5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml @@ -0,0 +1,156 @@ +name: "agg-gb300-tp4-mtp-kvoffload" + +# Agentic-coding SGLang aggregated recipe for DeepSeek-V4-Pro on GB300 +# (single aggregated worker, TP4, MTP + hierarchical-cache KV offload). +# +# Uses the flat single-variant schema the agentic CI flow expects. Concurrency is NOT +# a recipe field here: the GHA matrix fans out one job per concurrency from +# the master-config conc-list, exporting CONC into agentic_srt.sh. Modeled on +# the flat vllm/deepseek-v4/agentic recipes (agentic infra + benchmark wiring) +# and the flat sglang/deepseek-v4/8k1k recipes (sglang backend schema). + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev20260718" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + # Dedicated etcd/nats node — matches the vllm agentic recipes; under the + # dynamo router + multiple frontends the infra services need their own node. + etcd_nats_dedicated_node: true + # cc-traces prompts (~125K tokens) serialize to ~2.8MB; the 1 MiB NATS + # default drops them. 32 MiB matches the agentic vllm recipes (~10x headroom + # over the largest observed payload); schema recommends 24+ for long ISL. + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + # The sglang image is PEP 668 externally-managed; the runtime dynamo + # install (dynamo_wheels.py / source build) runs pip and fails with + # "externally-managed-environment" without this. Lets pip install into + # the system env. Applies to both dynamo.hash (source) and dynamo.wheel. + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + aggregated_environment: + SGLANG_DEFAULT_THINKING: '1' + # TP4/bs128 can spend over an hour capturing the DSV4 EAGLE draft-extend + # graph. Keep the target/decode graphs and use the upstream eager fallback. + SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. + PIP_BREAK_SYSTEM_PACKAGES: "1" + # from submission for B300 + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + # for kvcache offload + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + enable-metrics: true + trust-remote-code: true + stream-interval: 50 + watchdog-timeout: 1000000 + mem-fraction-static: 0.90 + chunked-prefill-size: 8192 # can only support 11000 for TP + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 128 + cuda-graph-max-bs: 128 + + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 4 + ep-size: 1 + # kv-cache offload + enable-hierarchical-cache: true + hicache-ratio: 6 + hicache-write-policy: write_through + hicache-io-backend: kernel + # mtp section + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user into the container as a non-root + # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to + # uid 0 inside the container. srt-slurm renders empty-string values as + # flag-only srun args. + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # Aggregated: one worker serves both prefill and decode, so GPU accounting + # is the single-worker form num_gpus = TP (not the disagg prefill+decode + # sum). The multinode post-processing path assumes disagg and would divide + # throughput by prefill_gpus + decode_gpus; force the single-node code path + # in process_agentic_result.py instead. TP must match sglang_config tp-size. + IS_MULTINODE: "false" + TP: "4" + # Stamp X-Dynamo-Session-ID on every turn so both nginx and Dynamo's + # router keep the session on the worker that owns its KV prefix. + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # Container-side path of the aiperf mmap dataset cache; host-side mount + # wired via launch_gb300-*.sh srtslurm.yaml default_mounts. Without it, + # aiperf re-tokenizes + re-writes the dataset mmap on every run. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also via default_mounts) so the trace dataset + # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE. + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml deleted file mode 100644 index 7cfd733f9..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml +++ /dev/null @@ -1,118 +0,0 @@ -name: "agg-gb300-tp4-mtp-lowlatency" - -# Low-latency AgentX aggregate topology: one TP4 worker occupies one -# four-GPU GB300 node and serves both prefill and decode with DSpark K=6. - -model: - path: "deepseek-v4-pro-dspark" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" - -dynamo: - install: true - hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" - -slurm: - time_limit: "4:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - aggregated_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - sglang_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - enable-metrics: true - trust-remote-code: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.94 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 32 - cuda-graph-max-bs-decode: 32 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 4 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_WARMUP_REQUESTS_PER_LANE: "1" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml deleted file mode 100644 index e6bc12780..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml +++ /dev/null @@ -1,118 +0,0 @@ -name: "agg-gb300-tp8-mtp-lowlatency" - -# Low-latency AgentX aggregate topology: one TP8 worker spans two -# four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. - -model: - path: "deepseek-v4-pro-dspark" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" - -dynamo: - install: true - hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" - -slurm: - time_limit: "4:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - aggregated_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - sglang_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - enable-metrics: true - trust-remote-code: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.94 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 4 - cuda-graph-max-bs-decode: 4 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 8 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_WARMUP_REQUESTS_PER_LANE: "1" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml index d587144c8..709962936 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload" +name: "disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1536. +# (12P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 1536. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,19 +9,13 @@ name: "disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro" container: "dynamo-sglang" precision: "fp4" -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" - dynamo: install: true - hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -33,9 +27,9 @@ health_check: resources: gpu_type: gb300 gpus_per_node: 4 - prefill_nodes: 8 + prefill_nodes: 12 decode_nodes: 4 - prefill_workers: 4 + prefill_workers: 6 decode_workers: 1 gpus_per_prefill: 8 gpus_per_decode: 16 @@ -66,8 +60,7 @@ backend: type: sglang prefill_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -100,8 +93,10 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -135,7 +130,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro enable-metrics: true model-path: /model/ trust-remote-code: true @@ -151,19 +146,18 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.85 + mem-fraction-static: 0.9 swa-full-tokens-ratio: 0.02 max-running-requests: 1024 cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 + chunked-prefill-size: 32768 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 3 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 + speculative-num-draft-tokens: 4 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -171,7 +165,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro enable-metrics: true model-path: /model/ trust-remote-code: true @@ -190,15 +184,14 @@ backend: mem-fraction-static: 0.9 swa-full-tokens-ratio: 0.02 max-running-requests: 3072 - cuda-graph-max-bs: 192 + cuda-graph-max-bs: 1024 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 3 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 + speculative-num-draft-tokens: 4 sbatch_directives: @@ -222,4 +215,3 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml new file mode 100644 index 000000000..e41438765 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml @@ -0,0 +1,216 @@ +name: "disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (2P x DEP8 / 1D x TP4, MTP + hierarchical-cache KV offload), tuned for concurrency 80. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev20260718" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + decode_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + # Match the TP4 aggregate worker: avoid the pathological DSV4 EAGLE + # draft-extend graph capture while retaining the target/decode graphs. + SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '2048' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + sglang_config: + prefill: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 32768 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 4 + dp-size: 1 + ep-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + moe-runner-backend: flashinfer_mxfp4 + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 128 + cuda-graph-max-bs: 128 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml similarity index 86% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml index 976a6c998..74b371e94 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload" +name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 192. +# (2P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 256. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,19 +9,13 @@ name: "disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro" container: "dynamo-sglang" precision: "fp4" -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" - dynamo: install: true - hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -63,7 +57,6 @@ backend: type: sglang prefill_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -97,8 +90,10 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -132,7 +127,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro enable-metrics: true model-path: /model/ trust-remote-code: true @@ -148,19 +143,18 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.85 + mem-fraction-static: 0.9 swa-full-tokens-ratio: 0.02 max-running-requests: 256 cuda-graph-max-bs: 256 - chunked-prefill-size: 65536 + chunked-prefill-size: 32768 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 3 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 + speculative-num-draft-tokens: 4 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -168,7 +162,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro enable-metrics: true model-path: /model/ trust-remote-code: true @@ -191,11 +185,10 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 3 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 + speculative-num-draft-tokens: 4 sbatch_directives: @@ -219,4 +212,3 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml similarity index 86% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml index 364cd582d..6b585fba8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload" +name: "disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload" # Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 -# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 400. +# (4P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 512. # # DEP8-prefill variant aligned with the measured Pareto point: prefill uses # tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. @@ -9,19 +9,13 @@ name: "disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload" # from the master-config conc-list. model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro" container: "dynamo-sglang" precision: "fp4" -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" - dynamo: install: true - hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -62,7 +56,6 @@ backend: type: sglang prefill_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -96,8 +89,10 @@ backend: SGLANG_DSV4_MHC_PREWARM: '1' decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -132,7 +127,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro enable-metrics: true model-path: /model/ trust-remote-code: true @@ -148,19 +143,18 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.85 + mem-fraction-static: 0.9 swa-full-tokens-ratio: 0.02 max-running-requests: 256 cuda-graph-max-bs: 256 - chunked-prefill-size: 65536 + chunked-prefill-size: 32768 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 3 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 + speculative-num-draft-tokens: 4 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -168,7 +162,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro enable-metrics: true model-path: /model/ trust-remote-code: true @@ -191,11 +185,10 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 3 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 + speculative-num-draft-tokens: 4 sbatch_directives: @@ -219,4 +212,3 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml similarity index 86% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml index 79ecbb704..b2cac21c7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload" +name: "disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 640. +# (6P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 768. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,19 +9,13 @@ name: "disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro" container: "dynamo-sglang" precision: "fp4" -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" - dynamo: install: true - hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -63,7 +57,6 @@ backend: type: sglang prefill_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -97,8 +90,10 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -132,7 +127,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro enable-metrics: true model-path: /model/ trust-remote-code: true @@ -148,19 +143,18 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.85 + mem-fraction-static: 0.9 swa-full-tokens-ratio: 0.02 max-running-requests: 512 cuda-graph-max-bs: 512 - chunked-prefill-size: 65536 + chunked-prefill-size: 32768 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 3 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 + speculative-num-draft-tokens: 4 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -168,7 +162,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro enable-metrics: true model-path: /model/ trust-remote-code: true @@ -191,11 +185,10 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 3 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 + speculative-num-draft-tokens: 4 sbatch_directives: @@ -219,4 +212,3 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index ea2f18935..3902856da 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7692,46 +7692,33 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: dp-attn: true dsv4-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 - model: deepseek-ai/DeepSeek-V4-Pro-DSpark + image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 + model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } + router: { name: dynamo-router, version: "1.3.0.dev20260718" } multinode: true disagg: false scenarios: agentic-coding: - - search-space: - - spec-decoding: draft_model - conc-list: [1, 4] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml" - decode: - num-worker: 0 - tp: 8 - ep: 1 - dp-attn: false - - search-space: - - spec-decoding: draft_model - conc-list: [8] + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [2, 4, 8, 16] + kv-offloading: dram + kv-offload-backend: { name: hicache } prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml" + # Aggregated worker: prefill's GPUs also serve decode, so decode has no + # separate allocation (num-worker: 0). Deployment is recipe-driven + # (agg_workers: 1); this only makes GPU accounting count 4, not 4+4. decode: num-worker: 0 tp: 4 @@ -7739,13 +7726,13 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: dp-attn: false dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 - model: deepseek-ai/DeepSeek-V4-Pro-DSpark + image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 + model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } + router: { name: dynamo-router, version: "1.3.0.dev20260718" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -7753,8 +7740,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: agentic-coding: - dram-utilization: 0.80 search-space: - - spec-decoding: draft_model - conc-list: [192] + - spec-decoding: mtp + conc-list: [80] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7763,16 +7750,30 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [256] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: draft_model - conc-list: [400] + - spec-decoding: mtp + conc-list: [512] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7781,16 +7782,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: draft_model - conc-list: [640] + - spec-decoding: mtp + conc-list: [768] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7799,27 +7798,23 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: draft_model + - spec-decoding: mtp conc-list: [1536] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: - num-worker: 4 + num-worker: 6 tp: 8 ep: 8 dp-attn: true additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 diff --git a/golden_al_distribution/README.md b/golden_al_distribution/README.md index 2713f39a6..95e4a5622 100644 --- a/golden_al_distribution/README.md +++ b/golden_al_distribution/README.md @@ -109,7 +109,6 @@ Before accepting an updated curve, reviewers should verify: | Model | Method | Golden YAML | Source run | | --- | --- | --- | --- | | DeepSeek V4 Pro | MTP | [`dsv4_mtp.yaml`](dsv4_mtp.yaml) | [27180633016](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27180633016) | -| DeepSeek V4 Pro DSpark | DSpark (probabilistic drafting) | [`dsv4_dspark_probabilistic.yaml`](dsv4_dspark_probabilistic.yaml) | [31279568355](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355) | | Qwen3.5 397B-A17B | MTP | [`qwen3.5_mtp.yaml`](qwen3.5_mtp.yaml) | [27317114007](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27317114007) | | Kimi K2.5 | EAGLE3 | [`kimik2.5_eagle3.yaml`](kimik2.5_eagle3.yaml) | [28122195822](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28122195822) | | MiniMax-M3 | EAGLE3 | [`minimaxm3_eagle3.yaml`](minimaxm3_eagle3.yaml) | [28061204145](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28061204145) | diff --git a/golden_al_distribution/README_zh.md b/golden_al_distribution/README_zh.md index baf680612..a56a005f2 100644 --- a/golden_al_distribution/README_zh.md +++ b/golden_al_distribution/README_zh.md @@ -109,7 +109,6 @@ gh workflow run speedbench-al.yml \ | 模型 | 方法 | 黄金 YAML | 源 run | | --- | --- | --- | --- | | DeepSeek V4 Pro | MTP | [`dsv4_mtp.yaml`](dsv4_mtp.yaml) | [27180633016](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27180633016) | -| DeepSeek V4 Pro DSpark | DSpark（概率式草稿采样） | [`dsv4_dspark_probabilistic.yaml`](dsv4_dspark_probabilistic.yaml) | [31279568355](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355) | | Qwen3.5 397B-A17B | MTP | [`qwen3.5_mtp.yaml`](qwen3.5_mtp.yaml) | [27317114007](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27317114007) | | Kimi K2.5 | EAGLE3 | [`kimik2.5_eagle3.yaml`](kimik2.5_eagle3.yaml) | [28122195822](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28122195822) | | MiniMax-M3 | EAGLE3 | [`minimaxm3_eagle3.yaml`](minimaxm3_eagle3.yaml) | [28061204145](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28061204145) | diff --git a/golden_al_distribution/dsv4_dspark_probabilistic.yaml b/golden_al_distribution/dsv4_dspark_probabilistic.yaml deleted file mode 100644 index 516aeb2ab..000000000 --- a/golden_al_distribution/dsv4_dspark_probabilistic.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Source GitHub Actions run: https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355 -# Acceptance Length (AL) reference values measured with SPEED-Bench. -# dataset: coding | temperature: 1.0 | output_len: 4096 -# thinking_on chat_template_kwargs: {"thinking": true, "reasoning_effort": "high"} -# speculative-config: method=dspark | draft_sample_method=probabilistic -# Measured on deepseek-v4-pro-dspark (B300, vLLM DSpark), per num_speculative_tokens. -# Auto-generated by benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh (speedbench-al.yml). -# -# key = num_speculative_tokens (DSpark level); value = golden AL -deepseek-v4-pro-dspark: - thinking_on: - 1: 1.86 - 2: 2.55 - 3: 3.08 - 4: 3.43 - 5: 3.74 - 6: 3.82 - 7: 3.86 - 8: 3.75 - thinking_off: - 1: 1.93 - 2: 2.76 - 3: 3.47 - 4: 4.10 - 5: 4.52 - 6: 4.87 - 7: 4.94 - 8: 4.94 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b2d718b28..cb295b344 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6019,18 +6019,6 @@ - "Refresh the MiniMax-M3 B200 AgentX submission: no-offload TP8 C1 and TP4 C1/C5/C10/C15/C20, plus a TP4 SimpleCPU KV-offload sweep at C15–C40 using the full-capacity DRAM budget." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2611 -- config-keys: - - dsv4-fp4-gb300-dynamo-sglang-agentic-agg - - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - "Replace the GB300 DeepSeek-V4-Pro AgentX EAGLE/MTP serving path with DSpark block size 6 across two aggregate and four disaggregated recipes" - - "Use DeepSeek-V4-Pro-DSpark with SGLANG_RAGGED_VERIFY_MODE=static, MegaMoE for DEP workers, and throughput-only thinking-on acceptance length 3.82 from the committed probabilistic DSpark curve (golden_al_distribution/dsv4_dspark_probabilistic.yaml, run 31279568355); eval keeps real DSpark verification" - - "Refresh SGLang to nightly-dev-cu13-20260816-4a6dc267, which already contains the required DSpark performance and MegaMoE support" - - "Retune the Pareto ladder to concurrency 1/4/8, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 - - config-keys: - dsv4-fp4-b300-dynamo-sglang-agentic-mtp-agg scenario-type: diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index cc583fb7f..83252461d 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -44,12 +44,6 @@ elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export SERVED_MODEL_NAME="deepseek-r1-fp8" export MODEL_PATH=/scratch/models/DeepSeek-R1-0528 export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" -elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $MODEL == "deepseek-ai/DeepSeek-V4-Pro-DSpark" ]]; then - # The DSpark checkpoint is staged in the shared runner cache visible from - # the GB300 compute nodes. Keep a distinct alias from the legacy MTP - # checkpoint so other DSv4 lanes retain their node-local model mapping. - export MODEL_PATH="$HF_HUB_CACHE_HOST_PATH/inferencex-models/DeepSeek-V4-Pro-DSpark" - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-dspark" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then # Use the node-local /scratch SSD for the 806 GB DSv4-Pro # checkpoint. Faster than the Vast NFS path, but this dir only diff --git a/runners/test_synthetic_acceptance.py b/runners/test_synthetic_acceptance.py deleted file mode 100644 index 874c87ec6..000000000 --- a/runners/test_synthetic_acceptance.py +++ /dev/null @@ -1,134 +0,0 @@ -import os -import subprocess -import sys -from pathlib import Path - -import yaml - -REPO_ROOT = Path(__file__).resolve().parents[1] -INJECTOR = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py" -LAUNCHER = REPO_ROOT / "runners" / "launch_gb300-nv.sh" -MASTER = REPO_ROOT / "configs" / "nvidia-master.yaml" -RECIPES = REPO_ROOT / "benchmarks" / "multi_node" / "srt-slurm-recipes" -GOLDEN_AL = REPO_ROOT / "golden_al_distribution" / "dsv4_dspark_probabilistic.yaml" -CONFIG_KEYS = ( - "dsv4-fp4-gb300-dynamo-sglang-agentic-agg", - "dsv4-fp4-gb300-dynamo-sglang-agentic-disagg", -) - - -def run_injector( - recipe: Path, *, eval_only: bool, acceptance_length: str = "3.82" -) -> subprocess.CompletedProcess[str]: - env = { - **os.environ, - "EVAL_ONLY": str(eval_only).lower(), - "SYNTHETIC_ACCEPTANCE": "true", - "SYNTHETIC_ACCEPTANCE_LENGTH": acceptance_length, - } - return subprocess.run( - [sys.executable, str(INJECTOR), str(recipe), "dynamo-sglang"], - check=False, - capture_output=True, - text=True, - env=env, - ) - - -def _configured_recipe_paths(master: dict) -> set[Path]: - recipe_paths = set() - for config_key in CONFIG_KEYS: - assert master[config_key]["model"] == "deepseek-ai/DeepSeek-V4-Pro-DSpark" - for group in master[config_key]["scenarios"]["agentic-coding"]: - for point in group["search-space"]: - assert point["spec-decoding"] == "draft_model" - settings = point["prefill"]["additional-settings"] - assert "SYNTHETIC_ACCEPTANCE=true" in settings - assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" in settings - config = next( - item for item in settings if item.startswith("CONFIG_FILE=") - ) - recipe_paths.add(RECIPES / config.removeprefix("CONFIG_FILE=recipes/")) - return recipe_paths - - -def test_sglang_injection_is_throughput_only(tmp_path: Path) -> None: - original = " prefill_environment:\n A: B\n decode_environment:\n C: D\n" - - throughput_recipe = tmp_path / "throughput.yaml" - throughput_recipe.write_text(original) - result = run_injector(throughput_recipe, eval_only=False) - assert result.returncode == 0, result.stderr - assert throughput_recipe.read_text().count("SGLANG_SIMULATE_ACC_LEN") == 2 - - eval_recipe = tmp_path / "eval.yaml" - eval_recipe.write_text(original) - result = run_injector(eval_recipe, eval_only=True) - assert result.returncode == 0, result.stderr - assert eval_recipe.read_text() == original - - -def test_gb300_dsv4_configs_inject_only_at_launch() -> None: - master = yaml.safe_load(MASTER.read_text()) - recipe_paths = _configured_recipe_paths(master) - - assert len(recipe_paths) == 6 - for recipe in recipe_paths: - text = recipe.read_text() - parsed = yaml.safe_load(text) - worker_count = 1 if recipe.name.startswith("agg-") else 2 - assert "SGLANG_SIMULATE_ACC_" not in text - assert text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count - assert text.count("speculative-algorithm: DSPARK") == worker_count - assert text.count("speculative-dspark-block-size: 6") == worker_count - assert text.count("speculative-num-draft-tokens: 7") == worker_count - assert "speculative-algorithm: EAGLE" not in text - assert 'path: "deepseek-v4-pro-dspark"' in text - assert parsed["benchmark"]["env"]["INFMAX_AIPERF_TOKENIZER"] == "/model" - assert "AIPERF_TOKENIZER" not in parsed["benchmark"]["env"] - - launcher = LAUNCHER.read_text() - source = 'source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh"' - inject = 'inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1' - apply = "SRTCTL_OUTPUT=$(srtctl apply" - assert launcher.index(source) < launcher.index(inject) < launcher.index(apply) - - -def test_gb300_dsv4_dspark6_matches_committed_golden_curve() -> None: - golden = yaml.safe_load(GOLDEN_AL.read_text()) - assert golden["deepseek-v4-pro-dspark"]["thinking_on"][6] == 3.82 - - master = yaml.safe_load(MASTER.read_text()) - for config_key in CONFIG_KEYS: - for group in master[config_key]["scenarios"]["agentic-coding"]: - for point in group["search-space"]: - settings = point["prefill"]["additional-settings"] - assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" in settings - - -def test_gb300_dsv4_dspark6_uses_golden_acceptance_only_for_throughput( - tmp_path: Path, -) -> None: - master = yaml.safe_load(MASTER.read_text()) - for recipe in _configured_recipe_paths(master): - original = recipe.read_text() - worker_count = 1 if recipe.name.startswith("agg-") else 2 - - throughput_recipe = tmp_path / recipe.name - throughput_recipe.write_text(original) - result = run_injector(throughput_recipe, eval_only=False) - assert result.returncode == 0, result.stderr - throughput_text = throughput_recipe.read_text() - assert throughput_text.count('SGLANG_SIMULATE_ACC_LEN: "3.82"') == worker_count - assert ( - throughput_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count - ) - - eval_recipe = tmp_path / f"eval-{recipe.name}" - eval_recipe.write_text(original) - result = run_injector(eval_recipe, eval_only=True) - assert result.returncode == 0, result.stderr - eval_text = eval_recipe.read_text() - assert eval_text == original - assert "SGLANG_SIMULATE_ACC_" not in eval_text - assert eval_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count From 1cbf9c35495f72382e87ff494a9f8d4b7b448452 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:34:09 +0800 Subject: [PATCH 06/20] perf(dsv4): calibrate DSpark6 acceptance to 3.77 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: the B300 DSpark6 sweep used acceptance length 3.82, but the DeepSeek-V4-Pro-0813 probabilistic golden curve measures thinking-on block size 6 at 3.77.\n\nFix: set both B300 DEP4 and DEP8 synthetic acceptance lengths to 3.77 and add the complete thinking-on/off block-size curve as the committed source of truth.\n\nValidation: YAML parsing passes and an automated consistency check confirms both master sweep arms equal the thinking-on block-size-6 golden value.\n\n根因：B300 DSpark6 sweep 使用了 3.82，但 DeepSeek-V4-Pro-0813 概率 golden 曲线中 thinking-on、block size 6 的测量值为 3.77。\n\n修复：将 B300 DEP4 和 DEP8 的合成 acceptance length 都改为 3.77，并提交完整的 thinking-on/off block-size 曲线作为依据。\n\n验证：YAML 解析通过，自动一致性检查确认两个 master sweep arm 都等于 thinking-on block-size-6 的 golden 值。 --- configs/nvidia-master.yaml | 4 ++-- .../dsv4_dspark_probabilistic.yaml | 19 +++++++++++++++++++ perf-changelog.yaml | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 golden_al_distribution/dsv4_dspark_probabilistic.yaml diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3902856da..b69895029 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8576,7 +8576,7 @@ dsv4-fp4-b300-dynamo-sglang-agentic-mtp-agg: dp-attn: true additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml" decode: num-worker: 0 @@ -8594,7 +8594,7 @@ dsv4-fp4-b300-dynamo-sglang-agentic-mtp-agg: dp-attn: true additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.82" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml" decode: num-worker: 0 diff --git a/golden_al_distribution/dsv4_dspark_probabilistic.yaml b/golden_al_distribution/dsv4_dspark_probabilistic.yaml new file mode 100644 index 000000000..f72fe9431 --- /dev/null +++ b/golden_al_distribution/dsv4_dspark_probabilistic.yaml @@ -0,0 +1,19 @@ +deepseek-v4-pro-0813: + thinking_on: + 1: 1.84 + 2: 2.51 + 3: 3.01 + 4: 3.36 + 5: 3.61 + 6: 3.77 + 7: 3.73 + 8: 3.47 + thinking_off: + 1: 1.94 + 2: 2.81 + 3: 3.58 + 4: 4.17 + 5: 4.74 + 6: 5.12 + 7: 5.24 + 8: 4.95 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cb295b344..b5df6a5c4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6024,6 +6024,6 @@ scenario-type: - agentic-coding description: - - "Add B300 Dynamo-SGLang DSpark6 AgentX aggregate DEP4 and DEP8 configurations with session-affinity routing, HiCache host offload, an 8K prefill window, and golden synthetic acceptance length 3.82 for throughput; eval retains real verification." + - "Add B300 Dynamo-SGLang DSpark6 AgentX aggregate DEP4 and DEP8 configurations with session-affinity routing, HiCache host offload, an 8K prefill window, and thinking-on block-size-6 golden synthetic acceptance length 3.77 for throughput; eval retains real verification." - "Use the August 16 SGLang CUDA 13 nightly and srt-slurm v1.0.38, with DEP4 concurrency 48-64 and DEP8 concurrency 128-576." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2624 From 4b038b9a716788adf96817dc5f2fb30ccb41873c Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:38:29 +0800 Subject: [PATCH 07/20] fix(ci): place B300 AgentX entry at YAML top level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: the B300 master entry was inserted between two fixed-sequence scenarios of the preceding configuration, so YAML parsed the trailing 8K scenario as a second AgentX scenario with forbidden isl/osl fields.\n\nFix: move the complete B300 entry after the preceding configuration's final scenario at the true top level.\n\nValidation: the repository master-config schema loader accepts the entry and reports only the agentic-coding scenario group; YAML and whitespace checks pass.\n\n根因：B300 master 条目被插入到前一个配置的两个 fixed-sequence scenario 之间，导致 YAML 将后续 8K scenario 解析成带有非法 isl/osl 字段的第二个 AgentX scenario。\n\n修复：将完整 B300 条目移动到前一配置最后一个 scenario 之后，作为真正的顶层配置。\n\n验证：仓库 master-config schema loader 已接受该条目，且只报告 agentic-coding scenario group；YAML 与空白检查通过。 --- configs/nvidia-master.yaml | 41 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b69895029..340f6797a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8551,6 +8551,27 @@ glm5.1-fp8-b200-tilert: additional-settings: - "DECODE_NODES=1" + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_IMAGE=vllm/vllm-openai:v0.26.0" + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + dsv4-fp4-b300-dynamo-sglang-agentic-mtp-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 model: deepseek-ai/DeepSeek-V4-Pro-DSpark @@ -8601,23 +8622,3 @@ dsv4-fp4-b300-dynamo-sglang-agentic-mtp-agg: tp: 8 ep: 8 dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_IMAGE=vllm/vllm-openai:v0.26.0" - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" From 432413264a1a4250753ea6acf75f5af6390e585f Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 05:00:53 +0800 Subject: [PATCH 08/20] fix(ci): fit B300 DEP8 HiCache within host DRAM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: DEP8 hicache-ratio 2.0 allocates about 1.545 TB of explicit host KV pools. Together with model loading, JIT, compiler, and page-cache overhead, startup exceeds the B300 x86 node host-memory budget and is OOM-killed before AgentX begins. Fix: cap the DEP8 HiCache ratio at 1.5, reducing the explicit host KV allocation by about 386 GB while retaining host prefix-cache offload. Validation: YAML parsing, whitespace checks, perf changelog validation, and exact-key matrix generation pass for all seven AgentX throughput points. The repository has no configured pre-commit executable in this checkout. 根因：DEP8 的 hicache-ratio 2.0 会分配约 1.545 TB 的显式主机 KV 池；叠加模型加载、JIT、编译器和页缓存开销后，启动阶段超过 B300 x86 节点的主机内存预算，在 AgentX 开始前被 OOM 杀死。 修复：将 DEP8 HiCache ratio 限制为 1.5，使显式主机 KV 分配减少约 386 GB，同时保留主机端 prefix cache offload。 验证：YAML 解析、空白检查、性能变更日志校验以及精确配置键的矩阵生成均通过，覆盖全部七个 AgentX 吞吐点；当前 checkout 未配置可执行的 pre-commit。 --- .../deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml index e5d9beaea..354db293d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -92,10 +92,9 @@ backend: load-balance-method: total_tokens moe-a2a-backend: megamoe enable-hierarchical-cache: true - # Ratio 1.75 completes c576, but its busiest DP rank reaches 99.4% host - # cache usage. Retry 2.0 with the smaller DSpark SWA pool (0.02) to add - # enough prefix-cache headroom without approaching the 2.5 startup OOM. - hicache-ratio: 2.0 + # Keep the explicit host KV pool within the x86 B300 node's 3 TB DRAM + # budget; ratio 2.0 OOMs during startup before the benchmark can run. + hicache-ratio: 1.5 hicache-write-policy: write_back hicache-io-backend: direct hicache-mem-layout: page_first_direct From 2be8f91dca7c1cc38432f21d865ba8e079e379c1 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 05:04:37 +0800 Subject: [PATCH 09/20] ci: retrigger B300 AgentX sweep after HiCache fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: the HiCache fix changed only the checked-in recipe, while the Run Sweep synchronize trigger is path-filtered to perf-changelog.yaml, so no workflow was created for the repaired commit. Fix: record the DEP8 ratio 1.5 host-DRAM constraint in the PR changelog entry, satisfying the benchmark-change invariant and triggering a fresh sweep. Validation: perf-changelog.yaml parses successfully, remains append-only relative to main, and whitespace checks pass. 根因：HiCache 修复只修改了提交进仓库的 recipe，而 Run Sweep 的 synchronize 触发器只监听 perf-changelog.yaml，因此修复 commit 没有创建 workflow。 修复：在本 PR 的 changelog 条目中记录 DEP8 ratio 1.5 的主机 DRAM 约束，满足 benchmark 变更规则并触发新一轮 sweep。 验证：perf-changelog.yaml 解析成功，相对 main 仍保持仅在末尾追加，且空白检查通过。 --- perf-changelog.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b5df6a5c4..c5f5eed65 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6026,4 +6026,5 @@ description: - "Add B300 Dynamo-SGLang DSpark6 AgentX aggregate DEP4 and DEP8 configurations with session-affinity routing, HiCache host offload, an 8K prefill window, and thinking-on block-size-6 golden synthetic acceptance length 3.77 for throughput; eval retains real verification." - "Use the August 16 SGLang CUDA 13 nightly and srt-slurm v1.0.38, with DEP4 concurrency 48-64 and DEP8 concurrency 128-576." + - "Cap the DEP8 HiCache ratio at 1.5 so the explicit host KV pool fits within the B300 x86 node's 3 TB DRAM budget during model loading and JIT startup." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2624 From 23662235ecc96bcd52fd48ba994648160e99b14b Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 08:46:20 +0800 Subject: [PATCH 10/20] fix(agentx): emit one staged tokenizer argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: build_replay_cmd always emitted --tokenizer $MODEL and the B300 DSpark recipe appended a second --tokenizer /model through INFMAX_AIPERF_TOKENIZER. AIPerf Click parsing rejects duplicate tokenizer options before profiling starts. Fix: select INFMAX_AIPERF_TOKENIZER when provided, otherwise fall back to $MODEL, and emit exactly one tokenizer option. Validation: Bash syntax passes; focused command-construction checks verify one --tokenizer /model for the B300 recipe and one --tokenizer org/model for the fallback path; perf changelog YAML parses successfully. 根因：build_replay_cmd 固定生成 --tokenizer $MODEL，而 B300 DSpark recipe 又通过 INFMAX_AIPERF_TOKENIZER 追加 --tokenizer /model。AIPerf 的 Click 参数解析在 profiling 开始前拒绝重复 tokenizer 选项。 修复：存在 INFMAX_AIPERF_TOKENIZER 时优先使用，否则回退到 $MODEL，并且只生成一个 tokenizer 参数。 验证：Bash 语法通过；聚焦命令构造测试确认 B300 recipe 只生成一个 --tokenizer /model，回退路径只生成一个 --tokenizer org/model；性能变更日志 YAML 解析成功。 --- benchmarks/benchmark_lib.sh | 13 ++++--------- perf-changelog.yaml | 1 + 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 15a07948e..f0452659c 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -2007,10 +2007,10 @@ build_replay_cmd() { REPLAY_CMD+=" --model ${SERVED_MODEL_NAME:-$MODEL}" # aiperf's dataset manager resolves the tokenizer from --model by # default, but a SERVED_MODEL_NAME override (above) is a wire name, not - # necessarily a valid HF repo id (e.g. "Qwen3.5-397B-A17B-NVFP4-V2" vs - # the real "nvidia/Qwen3.5-397B-A17B-NVFP4-V2"), which 404s tokenizer - # loading. Always pass the real HF id explicitly. - REPLAY_CMD+=" --tokenizer $MODEL" + # necessarily a valid HF repo id. Prefer a staged checkpoint when the + # recipe provides one; otherwise pass the real HF model ID. Emit exactly + # one --tokenizer option because Click rejects duplicate occurrences. + REPLAY_CMD+=" --tokenizer ${INFMAX_AIPERF_TOKENIZER:-$MODEL}" REPLAY_CMD+=" --concurrency $CONC" REPLAY_CMD+=" --benchmark-duration $duration" REPLAY_CMD+=" --stats-interval 30" @@ -2081,11 +2081,6 @@ build_replay_cmd() { # from vLLM) is unaffected by this flag and still gives us KV usage, # prefix cache hit rate, etc. REPLAY_CMD+=" --no-gpu-telemetry" - # A staged checkpoint can provide the tokenizer without touching the - # shared Hugging Face cache. This is required for local-dir model mounts. - if [[ -n "${INFMAX_AIPERF_TOKENIZER:-}" ]]; then - REPLAY_CMD+=" --tokenizer $INFMAX_AIPERF_TOKENIZER" - fi # aiperf's dataset manager (separate from the inference parser) loads # the model's tokenizer for trace-prompt tokenization regardless of # --use-server-token-count. Models like kimi (amd/Kimi-K2.5-MXFP4, diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 40e1e2986..493a2eacf 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6048,4 +6048,5 @@ - "Add B300 Dynamo-SGLang DSpark6 AgentX aggregate DEP4 and DEP8 configurations with session-affinity routing, HiCache host offload, an 8K prefill window, and thinking-on block-size-6 golden synthetic acceptance length 3.77 for throughput; eval retains real verification." - "Use the August 16 SGLang CUDA 13 nightly and srt-slurm v1.0.38, with DEP4 concurrency 48-64 and DEP8 concurrency 128-576." - "Cap the DEP8 HiCache ratio at 1.5 so the explicit host KV pool fits within the B300 x86 node's 3 TB DRAM budget during model loading and JIT startup." + - "Use the staged /model tokenizer as the single AIPerf tokenizer argument, avoiding a duplicate Click option while keeping AgentX setup independent of the shared Hugging Face cache." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2624 From 831b912f1978ff04037731a2f13a26ec0d3bfa21 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:23:51 +0800 Subject: [PATCH 11/20] Preserve B300 AgentX affinity across long sweeps Root cause: the c128 diagnostic run exceeded the 3600-second Dynamo session-affinity lease, and multiple frontend processes maintain separate affinity state.\n\nFix: use one Dynamo frontend for both B300 topologies and extend the affinity lease to four hours.\n\nValidation: parsed both changed YAML files and ran git diff --check. --- .../deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml | 5 ++--- .../deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml index 2909f6eb7..e199ad2b8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml @@ -37,13 +37,12 @@ frontend: type: dynamo nginx_session_affinity: true nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 + enable_multiple_frontends: false env: PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-session-affinity-ttl-secs: "3600" + router-session-affinity-ttl-secs: "14400" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml index 354db293d..6f8906c17 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -37,13 +37,12 @@ frontend: type: dynamo nginx_session_affinity: true nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 + enable_multiple_frontends: false env: PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-session-affinity-ttl-secs: "3600" + router-session-affinity-ttl-secs: "14400" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" From abf91552dd1ed76ac0a74c3f82e45100914e94ef Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:39:54 +0800 Subject: [PATCH 12/20] Match B300 DP-local prefill chunks to GB300 Root cause: chunked-prefill-size is DP-global. Values of 8192 resolved to only 2048 tokens per DEP4 rank and 1024 per DEP8 rank, delaying radix-cache materialization for concurrent AgentX branches. The GB300 reference resolves to 8192 tokens per rank and sustains near-theoretical cache hits.\n\nFix: set the B300 DEP4 and DEP8 global chunk budgets to 32768 and 65536 respectively, preserving an 8192-token local chunk on each DP rank.\n\nValidation: parsed both recipes, verified the configured values, and ran git diff --check. --- .../sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml | 2 +- .../sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml index e199ad2b8..a698fd5cb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml @@ -77,7 +77,7 @@ backend: # 0.9258 on B300; use the same 0.94 fraction as the GB300 TP4 recipe. mem-fraction-static: 0.94 swa-full-tokens-ratio: 0.02 - chunked-prefill-size: 8192 + chunked-prefill-size: 32768 max-prefill-tokens: 8192 max-running-requests: 128 cuda-graph-max-bs: 32 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml index 6f8906c17..3dde25656 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -75,7 +75,7 @@ backend: enable-metrics: true mem-fraction-static: 0.90 swa-full-tokens-ratio: 0.02 - chunked-prefill-size: 8192 + chunked-prefill-size: 65536 max-prefill-tokens: 8192 max-running-requests: 1152 cuda-graph-max-bs: 144 From 8d66f6cc12cb52bb14b9ac7b3c42dd06db729a88 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:14:03 +0800 Subject: [PATCH 13/20] perf: leave prefill activation headroom on B300 DEP8 --- .../deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml index 3dde25656..227956d6b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -73,7 +73,9 @@ backend: served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" trust-remote-code: true enable-metrics: true - mem-fraction-static: 0.90 + # Leave activation headroom for 8K/rank chunked prefill; 0.90 OOMs + # under the c128 AgentX warmup while 0.85 completes the full run. + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 chunked-prefill-size: 65536 max-prefill-tokens: 8192 From 3cf6a42dcd17d920b7050dd4ea7bc53138f7486b Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:28:49 +0800 Subject: [PATCH 14/20] fix: use staged B300 DSpark checkpoint --- runners/launch_b300-nv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 827c684ed..8262de6b4 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -44,7 +44,7 @@ elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo- export MODEL_PATH="${SELECTED_MODEL_PATH:-/data/models/dsv4-pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" && $MODEL == "deepseek-ai/DeepSeek-V4-Pro-DSpark" ]]; then - export MODEL_PATH="${MODEL_PATH:-/data/models/DeepSeek-V4-Pro-DSpark}" + export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro-0813}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-dspark" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" ]]; then export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro}" From b19ddadaa41bd55b3ed12f089e8420b45831c8db Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:30:19 +0800 Subject: [PATCH 15/20] cleanup: use native B300 DSpark model metadata --- .../sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml | 1 - .../sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml index a698fd5cb..aebf2bf42 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml @@ -71,7 +71,6 @@ backend: sglang_config: aggregated: served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - trust-remote-code: true enable-metrics: true # DSpark draft weights raise the measured minimum viable fraction to # 0.9258 on B300; use the same 0.94 fraction as the GB300 TP4 recipe. diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml index 227956d6b..943f83ccc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -71,7 +71,6 @@ backend: sglang_config: aggregated: served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - trust-remote-code: true enable-metrics: true # Leave activation headroom for 8K/rank chunked prefill; 0.90 OOMs # under the c128 AgentX warmup while 0.85 completes the full run. From b62a50c45a5274cee91478577b2dda2affcf82af Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:31:47 +0800 Subject: [PATCH 16/20] fix: retain mainline trust-remote-code setting --- .../sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml | 1 + .../sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml index aebf2bf42..a698fd5cb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml @@ -71,6 +71,7 @@ backend: sglang_config: aggregated: served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + trust-remote-code: true enable-metrics: true # DSpark draft weights raise the measured minimum viable fraction to # 0.9258 on B300; use the same 0.94 fraction as the GB300 TP4 recipe. diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml index 943f83ccc..227956d6b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -71,6 +71,7 @@ backend: sglang_config: aggregated: served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + trust-remote-code: true enable-metrics: true # Leave activation headroom for 8K/rank chunked prefill; 0.90 OOMs # under the c128 AgentX warmup while 0.85 completes the full run. From 8042c25be4604c06ab2b4bdaa7cdf4dc94ea11d7 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:47:49 +0800 Subject: [PATCH 17/20] fix(agentx): pin B300 reasoning effort high --- .../sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml | 1 + .../sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml index a698fd5cb..75e460663 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml @@ -57,6 +57,7 @@ backend: PIP_BREAK_SYSTEM_PACKAGES: "1" SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml index 227956d6b..e76b1bd30 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -57,6 +57,7 @@ backend: PIP_BREAK_SYSTEM_PACKAGES: "1" SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" From 8bdb32e4374fa2f4a395d76d830b0bb8225f0bed Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:49:44 +0800 Subject: [PATCH 18/20] fix(agentx): align B300 tokenizer with 0813 model --- benchmarks/benchmark_lib.sh | 8 ++++---- .../deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml | 7 +++---- .../deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml | 7 +++---- configs/nvidia-master.yaml | 2 +- runners/launch_b300-nv.sh | 4 ++-- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index f0452659c..68cd41221 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -2007,10 +2007,10 @@ build_replay_cmd() { REPLAY_CMD+=" --model ${SERVED_MODEL_NAME:-$MODEL}" # aiperf's dataset manager resolves the tokenizer from --model by # default, but a SERVED_MODEL_NAME override (above) is a wire name, not - # necessarily a valid HF repo id. Prefer a staged checkpoint when the - # recipe provides one; otherwise pass the real HF model ID. Emit exactly - # one --tokenizer option because Click rejects duplicate occurrences. - REPLAY_CMD+=" --tokenizer ${INFMAX_AIPERF_TOKENIZER:-$MODEL}" + # necessarily a valid HF repo id (e.g. "Qwen3.5-397B-A17B-NVFP4-V2" vs + # the real "nvidia/Qwen3.5-397B-A17B-NVFP4-V2"), which 404s tokenizer + # loading. Always pass the real HF id explicitly. + REPLAY_CMD+=" --tokenizer $MODEL" REPLAY_CMD+=" --concurrency $CONC" REPLAY_CMD+=" --benchmark-duration $duration" REPLAY_CMD+=" --stats-interval 30" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml index 75e460663..5ec9b3be6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml @@ -1,13 +1,13 @@ name: "agg-b300-dep4-mtp-kvoffload" model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" identity: model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" @@ -71,7 +71,7 @@ backend: SGLANG_OPT_USE_TOPK_V2: "1" sglang_config: aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" trust-remote-code: true enable-metrics: true # DSpark draft weights raise the measured minimum viable fraction to @@ -128,4 +128,3 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml index e76b1bd30..f794b3bfa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -1,13 +1,13 @@ name: "agg-b300-dep8-mtp-kvoffload" model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" identity: model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" @@ -71,7 +71,7 @@ backend: SGLANG_OPT_USE_TOPK_V2: "1" sglang_config: aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" trust-remote-code: true enable-metrics: true # Leave activation headroom for 8K/rank chunked prefill; 0.90 OOMs @@ -127,4 +127,3 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1f081f1ba..4b4dd5f6a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8666,7 +8666,7 @@ glm5.1-fp8-b200-tilert: dsv4-fp4-b300-dynamo-sglang-agentic-mtp-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 - model: deepseek-ai/DeepSeek-V4-Pro-DSpark + model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:b300-nv precision: fp4 diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 8262de6b4..d42ef5e66 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -43,9 +43,9 @@ elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo- fi export MODEL_PATH="${SELECTED_MODEL_PATH:-/data/models/dsv4-pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" -elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" && $MODEL == "deepseek-ai/DeepSeek-V4-Pro-DSpark" ]]; then +elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" && $MODEL == "deepseek-ai/DeepSeek-V4-Pro-0813" ]]; then export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro-0813}" - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-dspark" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-0813" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" ]]; then export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" From 78c1bd64a25b5d63e2ba7936cad2e8f48e1cb1fd Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:38:44 +0800 Subject: [PATCH 19/20] Enable B300 DP prefill phase alignment Root cause: independent DP schedulers can place some ranks in prefill while peers remain in decode, creating uneven expert-parallel work and idle time.\n\nFix: enable SGLang's prefill delayer with its bounded 30-pass delay on both B300 DEP4 and DEP8 AgentX recipes.\n\nValidation: parsed both YAML recipes, passed git diff checks, and successfully generated both recipes with srt-slurm dry-run on the B300 environment. --- .../sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml | 2 ++ .../sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml index 5ec9b3be6..8006c2251 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml @@ -90,6 +90,8 @@ backend: ep-size: 4 enable-dp-attention: true enable-dp-lm-head: true + enable-prefill-delayer: true + prefill-delayer-max-delay-passes: 30 moe-dense-tp-size: 1 load-balance-method: total_tokens moe-a2a-backend: megamoe diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml index f794b3bfa..15c491930 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -90,6 +90,8 @@ backend: ep-size: 8 enable-dp-attention: true enable-dp-lm-head: true + enable-prefill-delayer: true + prefill-delayer-max-delay-passes: 30 moe-dense-tp-size: 1 load-balance-method: total_tokens moe-a2a-backend: megamoe From 9da2042527e91bc45a3a05fefa150319354246e0 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 22:26:26 +0800 Subject: [PATCH 20/20] fix(agentx): handle B300 overlength traces --- .../deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml | 5 +++++ .../deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml index 8006c2251..fe5ba9d5a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml @@ -74,6 +74,11 @@ backend: served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" trust-remote-code: true enable-metrics: true + enable-cache-report: true + # The AgentX corpus contains a small number of traces longer than the + # runtime KV capacity. Match the existing native B300 launcher: truncate + # those requests instead of leaving Dynamo streams unresolved. + allow-auto-truncate: true # DSpark draft weights raise the measured minimum viable fraction to # 0.9258 on B300; use the same 0.94 fraction as the GB300 TP4 recipe. mem-fraction-static: 0.94 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml index 15c491930..a57e8c367 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -74,6 +74,11 @@ backend: served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" trust-remote-code: true enable-metrics: true + enable-cache-report: true + # The AgentX corpus contains a small number of traces longer than the + # runtime KV capacity. Match the existing native B300 launcher: truncate + # those requests instead of leaving Dynamo streams unresolved. + allow-auto-truncate: true # Leave activation headroom for 8K/rank chunked prefill; 0.90 OOMs # under the c128 AgentX warmup while 0.85 completes the full run. mem-fraction-static: 0.85