diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml deleted file mode 100644 index 7bb82d2b5..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "agg-gb300-tp4-mtp-kvoffload" - -# Agentic-coding SGLang aggregated recipe for DeepSeek-V4-Pro on GB300 -# (single aggregated worker, TP4, MTP + hierarchical-cache KV offload). -# -# Uses the flat single-variant schema the agentic CI flow expects. Concurrency is NOT -# a recipe field here: the GHA matrix fans out one job per concurrency from -# the master-config conc-list, exporting CONC into agentic_srt.sh. Modeled on -# the flat vllm/deepseek-v4/agentic recipes (agentic infra + benchmark wiring) -# and the flat sglang/deepseek-v4/8k1k recipes (sglang backend schema). - -model: - path: "deepseek-v4-pro" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - # Dedicated etcd/nats node — matches the vllm agentic recipes; under the - # dynamo router + multiple frontends the infra services need their own node. - etcd_nats_dedicated_node: true - # cc-traces prompts (~125K tokens) serialize to ~2.8MB; the 1 MiB NATS - # default drops them. 32 MiB matches the agentic vllm recipes (~10x headroom - # over the largest observed payload); schema recommends 24+ for long ISL. - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - # The sglang image is PEP 668 externally-managed; the runtime dynamo - # install (dynamo_wheels.py / source build) runs pip and fails with - # "externally-managed-environment" without this. Lets pip install into - # the system env. Applies to both dynamo.hash (source) and dynamo.wheel. - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - aggregated_environment: - SGLANG_DEFAULT_THINKING: '1' - # TP4/bs128 can spend over an hour capturing the DSV4 EAGLE draft-extend - # graph. Keep the target/decode graphs and use the upstream eager fallback. - SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. - PIP_BREAK_SYSTEM_PACKAGES: "1" - # from submission for B300 - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - # for kvcache offload - SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" - - sglang_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - enable-metrics: true - trust-remote-code: true - stream-interval: 50 - watchdog-timeout: 1000000 - mem-fraction-static: 0.90 - chunked-prefill-size: 8192 # can only support 11000 for TP - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 128 - cuda-graph-max-bs: 128 - - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 4 - ep-size: 1 - # kv-cache offload - enable-hierarchical-cache: true - hicache-ratio: 6 - hicache-write-policy: write_through - hicache-io-backend: kernel - # mtp section - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user into the container as a non-root - # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to - # uid 0 inside the container. srt-slurm renders empty-string values as - # flag-only srun args. - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - # Aggregated: one worker serves both prefill and decode, so GPU accounting - # is the single-worker form num_gpus = TP (not the disagg prefill+decode - # sum). The multinode post-processing path assumes disagg and would divide - # throughput by prefill_gpus + decode_gpus; force the single-node code path - # in process_agentic_result.py instead. TP must match sglang_config tp-size. - IS_MULTINODE: "false" - TP: "4" - # Stamp X-Dynamo-Session-ID on every turn so both nginx and Dynamo's - # router keep the session on the worker that owns its KV prefix. - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - # Container-side path of the aiperf mmap dataset cache; host-side mount - # wired via launch_gb300-*.sh srtslurm.yaml default_mounts. Without it, - # aiperf re-tokenizes + re-writes the dataset mmap on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also via default_mounts) so the trace dataset - # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml new file mode 100644 index 000000000..5f125e330 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml @@ -0,0 +1,120 @@ +name: "agg-gb300-tp4-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP4 worker occupies one +# four-GPU GB300 node and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + aggregated_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + # Avoid the TP4 DSpark startup stall in the shared-read-done fastpath. + SGLANG_FORCE_COARSE_WAR_BARRIER: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + enable-metrics: true + enable-cache-report: true + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 4 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml new file mode 100644 index 000000000..da765fd1e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml @@ -0,0 +1,120 @@ +name: "agg-gb300-tp8-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP8 worker spans two +# four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + aggregated_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + # Avoid aggregate DSpark startup stalls in the shared-read-done fastpath. + SGLANG_FORCE_COARSE_WAR_BARRIER: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + enable-metrics: true + enable-cache-report: true + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 8 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml deleted file mode 100644 index e41438765..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml +++ /dev/null @@ -1,216 +0,0 @@ -name: "disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 1D x TP4, MTP + hierarchical-cache KV offload), tuned for concurrency 80. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - - decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - # Match the TP4 aggregate worker: avoid the pathological DSV4 EAGLE - # draft-extend graph capture while retaining the target/decode graphs. - SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '2048' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - - sglang_config: - prefill: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - enable-metrics: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.9 - swa-full-tokens-ratio: 0.02 - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 32768 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - enable-metrics: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 4 - dp-size: 1 - ep-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - moe-runner-backend: flashinfer_mxfp4 - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - swa-full-tokens-ratio: 0.02 - max-running-requests: 128 - cuda-graph-max-bs: 128 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml similarity index 86% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml index 74b371e94..eb1ef5313 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" +name: "disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 256. +# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 192. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -57,6 +63,7 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -90,10 +97,8 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,8 +132,9 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -143,18 +149,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 256 cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,8 +169,9 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -185,10 +193,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml similarity index 86% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml index 6b585fba8..df487dd82 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload" +name: "disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload" # Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 -# (4P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 512. +# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 400. # # DEP8-prefill variant aligned with the measured Pareto point: prefill uses # tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. @@ -9,13 +9,19 @@ name: "disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload" # from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -56,6 +62,7 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -89,10 +96,8 @@ backend: SGLANG_DSV4_MHC_PREWARM: '1' decode_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,8 +132,9 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -143,18 +149,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 256 cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,8 +169,9 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -185,10 +193,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml similarity index 86% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml index b2cac21c7..d78f36153 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload" +name: "disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (6P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 768. +# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 640. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -57,6 +63,7 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -90,10 +97,8 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,8 +132,9 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -143,18 +149,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 512 cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,8 +169,9 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -185,10 +193,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml similarity index 83% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml index 709962936..798d96345 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload" +name: "disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (12P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 1536. +# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1152. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -27,9 +33,9 @@ health_check: resources: gpu_type: gb300 gpus_per_node: 4 - prefill_nodes: 12 + prefill_nodes: 8 decode_nodes: 4 - prefill_workers: 6 + prefill_workers: 4 decode_workers: 1 gpus_per_prefill: 8 gpus_per_decode: 16 @@ -42,6 +48,7 @@ frontend: type: dynamo nginx_session_affinity: true nginx_session_affinity_header: X-Dynamo-Session-ID + nginx_keepalive_timeout: "900s" enable_multiple_frontends: true num_additional_frontends: 4 env: @@ -60,7 +67,8 @@ backend: type: sglang prefill_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -93,10 +101,8 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -130,8 +136,9 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -146,18 +153,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 1024 cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -165,8 +173,9 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -184,14 +193,15 @@ backend: mem-fraction-static: 0.9 swa-full-tokens-ratio: 0.02 max-running-requests: 3072 - cuda-graph-max-bs: 1024 + cuda-graph-max-bs: 192 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -211,6 +221,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 2f73b1c64..7be3444cc 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7784,33 +7784,46 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: dp-attn: true dsv4-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 - model: deepseek-ai/DeepSeek-V4-Pro + image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 + model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev20260718" } + router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } multinode: true disagg: false scenarios: agentic-coding: - - dram-utilization: 0.80 - search-space: - - spec-decoding: mtp - conc-list: [2, 4, 8, 16] - kv-offloading: dram - kv-offload-backend: { name: hicache } + - search-space: + - spec-decoding: draft_model + conc-list: [1, 4] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + - search-space: + - spec-decoding: draft_model + conc-list: [8] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml" - # Aggregated worker: prefill's GPUs also serve decode, so decode has no - # separate allocation (num-worker: 0). Deployment is recipe-driven - # (agg_workers: 1); this only makes GPU accounting count 4, not 4+4. + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml" decode: num-worker: 0 tp: 4 @@ -7818,13 +7831,13 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: dp-attn: false dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 - model: deepseek-ai/DeepSeek-V4-Pro + image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 + model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev20260718" } + router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -7832,24 +7845,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: agentic-coding: - dram-utilization: 0.80 search-space: - - spec-decoding: mtp - conc-list: [80] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - conc-list: [256] + - spec-decoding: draft_model + conc-list: [192] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7858,14 +7855,16 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp - conc-list: [512] + - spec-decoding: draft_model + conc-list: [400] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7874,14 +7873,16 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp - conc-list: [768] + - spec-decoding: draft_model + conc-list: [640] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7890,23 +7891,27 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp - conc-list: [1536] + - spec-decoding: draft_model + conc-list: [1152] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: - num-worker: 6 + num-worker: 4 tp: 8 ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 diff --git a/golden_al_distribution/README.md b/golden_al_distribution/README.md index 95e4a5622..2993f9c1b 100644 --- a/golden_al_distribution/README.md +++ b/golden_al_distribution/README.md @@ -109,6 +109,8 @@ Before accepting an updated curve, reviewers should verify: | Model | Method | Golden YAML | Source run | | --- | --- | --- | --- | | DeepSeek V4 Pro | MTP | [`dsv4_mtp.yaml`](dsv4_mtp.yaml) | [27180633016](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27180633016) | +| DeepSeek V4 Pro DSpark | DSpark (probabilistic drafting) | [`dsv4_dspark_probabilistic.yaml`](dsv4_dspark_probabilistic.yaml) | [31279568355](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355) | +| DeepSeek V4 Pro 0813 | DSpark (probabilistic drafting) | [`dsv4_0813_dspark_probabilistic.yaml`](dsv4_0813_dspark_probabilistic.yaml) | Provided 2026-08-17; same SpeedBench AL method | | Qwen3.5 397B-A17B | MTP | [`qwen3.5_mtp.yaml`](qwen3.5_mtp.yaml) | [27317114007](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27317114007) | | Kimi K2.5 | EAGLE3 | [`kimik2.5_eagle3.yaml`](kimik2.5_eagle3.yaml) | [28122195822](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28122195822) | | MiniMax-M3 | EAGLE3 | [`minimaxm3_eagle3.yaml`](minimaxm3_eagle3.yaml) | [28061204145](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28061204145) | diff --git a/golden_al_distribution/README_zh.md b/golden_al_distribution/README_zh.md index a56a005f2..18b38be80 100644 --- a/golden_al_distribution/README_zh.md +++ b/golden_al_distribution/README_zh.md @@ -109,6 +109,8 @@ gh workflow run speedbench-al.yml \ | 模型 | 方法 | 黄金 YAML | 源 run | | --- | --- | --- | --- | | DeepSeek V4 Pro | MTP | [`dsv4_mtp.yaml`](dsv4_mtp.yaml) | [27180633016](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27180633016) | +| DeepSeek V4 Pro DSpark | DSpark(概率式草稿采样) | [`dsv4_dspark_probabilistic.yaml`](dsv4_dspark_probabilistic.yaml) | [31279568355](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355) | +| DeepSeek V4 Pro 0813 | DSpark(概率式草稿采样) | [`dsv4_0813_dspark_probabilistic.yaml`](dsv4_0813_dspark_probabilistic.yaml) | 2026-08-17 提供;使用相同 SpeedBench AL 测法 | | Qwen3.5 397B-A17B | MTP | [`qwen3.5_mtp.yaml`](qwen3.5_mtp.yaml) | [27317114007](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27317114007) | | Kimi K2.5 | EAGLE3 | [`kimik2.5_eagle3.yaml`](kimik2.5_eagle3.yaml) | [28122195822](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28122195822) | | MiniMax-M3 | EAGLE3 | [`minimaxm3_eagle3.yaml`](minimaxm3_eagle3.yaml) | [28061204145](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28061204145) | diff --git a/golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml b/golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml new file mode 100644 index 000000000..1333b4de8 --- /dev/null +++ b/golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml @@ -0,0 +1,24 @@ +# Acceptance Length (AL) reference values measured with the same SpeedBench +# setup as the existing DeepSeek-V4-Pro DSpark probabilistic curve, using the +# DeepSeek-V4-Pro-0813 model. Values provided on 2026-08-17. +# +# key = num_speculative_tokens (DSpark level); value = golden AL +deepseek-v4-pro-0813: + thinking_on: + 1: 1.84 + 2: 2.51 + 3: 3.01 + 4: 3.36 + 5: 3.61 + 6: 3.77 + 7: 3.73 + 8: 3.47 + thinking_off: + 1: 1.94 + 2: 2.81 + 3: 3.58 + 4: 4.17 + 5: 4.74 + 6: 5.12 + 7: 5.24 + 8: 4.95 diff --git a/golden_al_distribution/dsv4_dspark_probabilistic.yaml b/golden_al_distribution/dsv4_dspark_probabilistic.yaml new file mode 100644 index 000000000..516aeb2ab --- /dev/null +++ b/golden_al_distribution/dsv4_dspark_probabilistic.yaml @@ -0,0 +1,28 @@ +# Source GitHub Actions run: https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355 +# Acceptance Length (AL) reference values measured with SPEED-Bench. +# dataset: coding | temperature: 1.0 | output_len: 4096 +# thinking_on chat_template_kwargs: {"thinking": true, "reasoning_effort": "high"} +# speculative-config: method=dspark | draft_sample_method=probabilistic +# Measured on deepseek-v4-pro-dspark (B300, vLLM DSpark), per num_speculative_tokens. +# Auto-generated by benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh (speedbench-al.yml). +# +# key = num_speculative_tokens (DSpark level); value = golden AL +deepseek-v4-pro-dspark: + thinking_on: + 1: 1.86 + 2: 2.55 + 3: 3.08 + 4: 3.43 + 5: 3.74 + 6: 3.82 + 7: 3.86 + 8: 3.75 + thinking_off: + 1: 1.93 + 2: 2.76 + 3: 3.47 + 4: 4.10 + 5: 4.52 + 6: 4.87 + 7: 4.94 + 8: 4.94 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 085e9cb88..b8ea19f20 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6039,7 +6039,6 @@ - "EVAL_ONLY runs switch to real block verification by removing synthetic acceptance (TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS stripped from all agentx recipes before srtctl)." - "Add SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 to all 6 agentx recipe env blocks and make build_replay_cmd prefer SERVED_MODEL_NAME over $MODEL for the aiperf --model flag." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2612 - - config-keys: - glm5.2-fp4-gb200-dynamo-sglang-agentic-agg - glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg @@ -6049,3 +6048,35 @@ - "Add GB200 GLM-5.2 FP4 Dynamo-SGLang AgentX with EAGLE MTP, HiCache DRAM offload, and KV-aware correlation-ID affinity." - "Measure the TP8 aggregate curve and tuned 1P1D TP4 HiCache c10/c12 disaggregated frontier with every logical SGLang metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2620 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Replace the GB300 DeepSeek-V4-Pro AgentX EAGLE/MTP serving path with DSpark block size 6 across two aggregate and four disaggregated recipes" + - "Use the official DeepSeek-V4-Pro-0813 checkpoint with SGLANG_RAGGED_VERIFY_MODE=static, MegaMoE for DEP workers, and throughput-only thinking-on acceptance length 3.77 from the committed DeepSeek-V4-Pro-0813 probabilistic DSpark curve (golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml); eval keeps real DSpark verification" + - "Refresh SGLang to nightly-dev-cu13-20260816-4a6dc267, which already contains the required DSpark performance and MegaMoE support" + - "Retune the Pareto ladder to concurrency 1/4/8, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Restore concurrency 1152 as the validated 8P4D DSpark throughput endpoint after the concurrency-1536 formal AgentX warmup exceeded prefill activation-memory headroom" + - "Keep the 8P4D topology, DSpark block size 6, prefill mem-fraction-static 0.85, decode CUDA Graph batch size 192, and all other serving parameters unchanged" + - "Use the coarse WAR barrier only for the TP4 aggregate point to avoid the shared-read-done startup stall observed with the 20260816 nightly" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Enable SGLang cache reporting on every GB300 DeepSeek-V4-Pro DSpark worker and prefetch checkpoint shards on the two pure-TP aggregate recipes" + - "Move the DSv4 AgentX runner from srt-slurm v1.0.38 to v1.0.40; set the concurrency-1152 recipe's nginx client/upstream keepalive to 900 seconds and AIPerf TCP user timeout to 900 seconds so a pooled multi-turn connection is not closed between turns" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 88b964a8a..bea9f742b 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -44,6 +44,11 @@ elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export SERVED_MODEL_NAME="deepseek-r1-fp8" export MODEL_PATH=/scratch/models/DeepSeek-R1-0528 export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" +elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $MODEL == "deepseek-ai/DeepSeek-V4-Pro-0813" ]]; then + # The official 0813 checkpoint, including its attached DSpark weights, is + # staged on every GB300 node in batch_1. + export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro-0813" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-0813" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then # Use the node-local /scratch SSD for the 806 GB DSv4-Pro # checkpoint. Faster than the Vast NFS path, but this dir only @@ -224,15 +229,16 @@ elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX = cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" \ recipes/sglang/qwen3.5 elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then - # DSv4 GB300 SGLang agentic uses NVIDIA/srt-slurm v1.0.38. In addition to + # DSv4 GB300 SGLang agentic uses NVIDIA/srt-slurm v1.0.40. In addition to # the nginx body-size fix, session-affinity frontend, and custom benchmark # schema required by these recipes, this release injects every logical - # SGLang worker leader's /metrics URL into AIPERF_SERVER_METRICS_URLS. + # SGLang worker leader's /metrics URL into AIPERF_SERVER_METRICS_URLS and + # supports long-lived nginx keepalive for multi-turn AgentX replay. # AgentX forwards that list to aiperf's --server-metrics argument so its # trace artifacts include backend metrics for every engine. git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout v1.0.38 + git checkout v1.0.40 mkdir -p recipes/sglang/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ recipes/sglang/deepseek-v4/agentic diff --git a/runners/test_synthetic_acceptance.py b/runners/test_synthetic_acceptance.py new file mode 100644 index 000000000..6a5b910a6 --- /dev/null +++ b/runners/test_synthetic_acceptance.py @@ -0,0 +1,136 @@ +import os +import subprocess +import sys +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +INJECTOR = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py" +LAUNCHER = REPO_ROOT / "runners" / "launch_gb300-nv.sh" +MASTER = REPO_ROOT / "configs" / "nvidia-master.yaml" +RECIPES = REPO_ROOT / "benchmarks" / "multi_node" / "srt-slurm-recipes" +GOLDEN_AL = REPO_ROOT / "golden_al_distribution" / "dsv4_0813_dspark_probabilistic.yaml" +CONFIG_KEYS = ( + "dsv4-fp4-gb300-dynamo-sglang-agentic-agg", + "dsv4-fp4-gb300-dynamo-sglang-agentic-disagg", +) + + +def run_injector( + recipe: Path, *, eval_only: bool, acceptance_length: str = "3.77" +) -> subprocess.CompletedProcess[str]: + env = { + **os.environ, + "EVAL_ONLY": str(eval_only).lower(), + "SYNTHETIC_ACCEPTANCE": "true", + "SYNTHETIC_ACCEPTANCE_LENGTH": acceptance_length, + } + return subprocess.run( + [sys.executable, str(INJECTOR), str(recipe), "dynamo-sglang"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + +def _configured_recipe_paths(master: dict) -> set[Path]: + recipe_paths = set() + for config_key in CONFIG_KEYS: + assert master[config_key]["model"] == "deepseek-ai/DeepSeek-V4-Pro-0813" + for group in master[config_key]["scenarios"]["agentic-coding"]: + for point in group["search-space"]: + assert point["spec-decoding"] == "draft_model" + settings = point["prefill"]["additional-settings"] + assert "SYNTHETIC_ACCEPTANCE=true" in settings + assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" in settings + config = next( + item for item in settings if item.startswith("CONFIG_FILE=") + ) + recipe_paths.add(RECIPES / config.removeprefix("CONFIG_FILE=recipes/")) + return recipe_paths + + +def test_sglang_injection_is_throughput_only(tmp_path: Path) -> None: + original = " prefill_environment:\n A: B\n decode_environment:\n C: D\n" + + throughput_recipe = tmp_path / "throughput.yaml" + throughput_recipe.write_text(original) + result = run_injector(throughput_recipe, eval_only=False) + assert result.returncode == 0, result.stderr + assert throughput_recipe.read_text().count("SGLANG_SIMULATE_ACC_LEN") == 2 + + eval_recipe = tmp_path / "eval.yaml" + eval_recipe.write_text(original) + result = run_injector(eval_recipe, eval_only=True) + assert result.returncode == 0, result.stderr + assert eval_recipe.read_text() == original + + +def test_gb300_dsv4_configs_inject_only_at_launch() -> None: + master = yaml.safe_load(MASTER.read_text()) + recipe_paths = _configured_recipe_paths(master) + + assert len(recipe_paths) == 6 + for recipe in recipe_paths: + text = recipe.read_text() + parsed = yaml.safe_load(text) + worker_count = 1 if recipe.name.startswith("agg-") else 2 + assert "SGLANG_SIMULATE_ACC_" not in text + assert text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count + assert text.count("speculative-algorithm: DSPARK") == worker_count + assert text.count("speculative-dspark-block-size: 6") == worker_count + assert text.count("speculative-num-draft-tokens: 7") == worker_count + assert "speculative-algorithm: EAGLE" not in text + assert 'path: "deepseek-v4-pro-0813"' in text + assert "AIPERF_TOKENIZER" not in parsed["benchmark"]["env"] + + launcher = LAUNCHER.read_text() + assert 'MODEL == "deepseek-ai/DeepSeek-V4-Pro-0813"' in launcher + assert 'MODEL_PATH="/scratch/models/DeepSeek-V4-Pro-0813"' in launcher + assert 'SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-0813"' in launcher + source = 'source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh"' + inject = 'inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1' + apply = "SRTCTL_OUTPUT=$(srtctl apply" + assert launcher.index(source) < launcher.index(inject) < launcher.index(apply) + + +def test_gb300_dsv4_dspark6_matches_committed_golden_curve() -> None: + golden = yaml.safe_load(GOLDEN_AL.read_text()) + assert golden["deepseek-v4-pro-0813"]["thinking_on"][6] == 3.77 + + master = yaml.safe_load(MASTER.read_text()) + for config_key in CONFIG_KEYS: + for group in master[config_key]["scenarios"]["agentic-coding"]: + for point in group["search-space"]: + settings = point["prefill"]["additional-settings"] + assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" in settings + + +def test_gb300_dsv4_dspark6_uses_golden_acceptance_only_for_throughput( + tmp_path: Path, +) -> None: + master = yaml.safe_load(MASTER.read_text()) + for recipe in _configured_recipe_paths(master): + original = recipe.read_text() + worker_count = 1 if recipe.name.startswith("agg-") else 2 + + throughput_recipe = tmp_path / recipe.name + throughput_recipe.write_text(original) + result = run_injector(throughput_recipe, eval_only=False) + assert result.returncode == 0, result.stderr + throughput_text = throughput_recipe.read_text() + assert throughput_text.count('SGLANG_SIMULATE_ACC_LEN: "3.77"') == worker_count + assert ( + throughput_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count + ) + + eval_recipe = tmp_path / f"eval-{recipe.name}" + eval_recipe.write_text(original) + result = run_injector(eval_recipe, eval_only=True) + assert result.returncode == 0, result.stderr + eval_text = eval_recipe.read_text() + assert eval_text == original + assert "SGLANG_SIMULATE_ACC_" not in eval_text + assert eval_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count