diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml deleted file mode 100644 index 7bb82d2b5..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "agg-gb300-tp4-mtp-kvoffload" - -# Agentic-coding SGLang aggregated recipe for DeepSeek-V4-Pro on GB300 -# (single aggregated worker, TP4, MTP + hierarchical-cache KV offload). -# -# Uses the flat single-variant schema the agentic CI flow expects. Concurrency is NOT -# a recipe field here: the GHA matrix fans out one job per concurrency from -# the master-config conc-list, exporting CONC into agentic_srt.sh. Modeled on -# the flat vllm/deepseek-v4/agentic recipes (agentic infra + benchmark wiring) -# and the flat sglang/deepseek-v4/8k1k recipes (sglang backend schema). - -model: - path: "deepseek-v4-pro" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - # Dedicated etcd/nats node — matches the vllm agentic recipes; under the - # dynamo router + multiple frontends the infra services need their own node. - etcd_nats_dedicated_node: true - # cc-traces prompts (~125K tokens) serialize to ~2.8MB; the 1 MiB NATS - # default drops them. 32 MiB matches the agentic vllm recipes (~10x headroom - # over the largest observed payload); schema recommends 24+ for long ISL. - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - # The sglang image is PEP 668 externally-managed; the runtime dynamo - # install (dynamo_wheels.py / source build) runs pip and fails with - # "externally-managed-environment" without this. Lets pip install into - # the system env. Applies to both dynamo.hash (source) and dynamo.wheel. - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - aggregated_environment: - SGLANG_DEFAULT_THINKING: '1' - # TP4/bs128 can spend over an hour capturing the DSV4 EAGLE draft-extend - # graph. Keep the target/decode graphs and use the upstream eager fallback. - SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. - PIP_BREAK_SYSTEM_PACKAGES: "1" - # from submission for B300 - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - # for kvcache offload - SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" - - sglang_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - enable-metrics: true - trust-remote-code: true - stream-interval: 50 - watchdog-timeout: 1000000 - mem-fraction-static: 0.90 - chunked-prefill-size: 8192 # can only support 11000 for TP - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 128 - cuda-graph-max-bs: 128 - - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 4 - ep-size: 1 - # kv-cache offload - enable-hierarchical-cache: true - hicache-ratio: 6 - hicache-write-policy: write_through - hicache-io-backend: kernel - # mtp section - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user into the container as a non-root - # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to - # uid 0 inside the container. srt-slurm renders empty-string values as - # flag-only srun args. - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - # Aggregated: one worker serves both prefill and decode, so GPU accounting - # is the single-worker form num_gpus = TP (not the disagg prefill+decode - # sum). The multinode post-processing path assumes disagg and would divide - # throughput by prefill_gpus + decode_gpus; force the single-node code path - # in process_agentic_result.py instead. TP must match sglang_config tp-size. - IS_MULTINODE: "false" - TP: "4" - # Stamp X-Dynamo-Session-ID on every turn so both nginx and Dynamo's - # router keep the session on the worker that owns its KV prefix. - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - # Container-side path of the aiperf mmap dataset cache; host-side mount - # wired via launch_gb300-*.sh srtslurm.yaml default_mounts. Without it, - # aiperf re-tokenizes + re-writes the dataset mmap on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also via default_mounts) so the trace dataset - # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml new file mode 100644 index 000000000..b2aa4a88e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml @@ -0,0 +1,114 @@ +name: "agg-gb300-tp4-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP4 worker occupies one +# four-GPU GB300 node and serves both prefill and decode. + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" + frameworks: + dynamo: "1.4.0" + +dynamo: + install: true + wheel: "1.4.0" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + aggregated_environment: + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + enable-metrics: true + trust-remote-code: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 4 + ep-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_WARMUP_REQUESTS_PER_LANE: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml new file mode 100644 index 000000000..336b90ca5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml @@ -0,0 +1,114 @@ +name: "agg-gb300-tp8-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP8 worker spans two +# four-GPU GB300 nodes and serves both prefill and decode. + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" + frameworks: + dynamo: "1.4.0" + +dynamo: + install: true + wheel: "1.4.0" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + aggregated_environment: + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + enable-metrics: true + trust-remote-code: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 8 + ep-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_WARMUP_REQUESTS_PER_LANE: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml index 709962936..e305f8624 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml @@ -10,15 +10,20 @@ name: "disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload" model: path: "deepseek-v4-pro" - container: "dynamo-sglang" + container: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" + frameworks: + dynamo: "1.4.0" + dynamo: install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" + wheel: "1.4.0" health_check: max_attempts: 1440 @@ -94,9 +99,6 @@ backend: decode_environment: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml index e41438765..151fe388e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml @@ -10,15 +10,20 @@ name: "disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload" model: path: "deepseek-v4-pro" - container: "dynamo-sglang" + container: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" + frameworks: + dynamo: "1.4.0" + dynamo: install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" + wheel: "1.4.0" health_check: max_attempts: 1440 @@ -94,9 +99,6 @@ backend: # Match the TP4 aggregate worker: avoid the pathological DSV4 EAGLE # draft-extend graph capture while retaining the target/decode graphs. SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml index 74b371e94..fc0ef0a48 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -10,15 +10,20 @@ name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" model: path: "deepseek-v4-pro" - container: "dynamo-sglang" + container: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" + frameworks: + dynamo: "1.4.0" + dynamo: install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" + wheel: "1.4.0" health_check: max_attempts: 1440 @@ -91,9 +96,6 @@ backend: decode_environment: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml index 6b585fba8..c396796a2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml @@ -10,15 +10,20 @@ name: "disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload" model: path: "deepseek-v4-pro" - container: "dynamo-sglang" + container: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" + frameworks: + dynamo: "1.4.0" + dynamo: install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" + wheel: "1.4.0" health_check: max_attempts: 1440 @@ -90,9 +95,6 @@ backend: decode_environment: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml index b2cac21c7..803150680 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml @@ -10,15 +10,20 @@ name: "disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload" model: path: "deepseek-v4-pro" - container: "dynamo-sglang" + container: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf" + frameworks: + dynamo: "1.4.0" + dynamo: install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" + wheel: "1.4.0" health_check: max_attempts: 1440 @@ -91,9 +96,6 @@ backend: decode_environment: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index cb636bf90..12a86a809 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7955,33 +7955,46 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: dp-attn: true dsv4-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 + image: lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev20260718" } + router: { name: dynamo-router, version: "1.4.0" } multinode: true disagg: false scenarios: agentic-coding: - - dram-utilization: 0.80 - search-space: + - search-space: - spec-decoding: mtp - conc-list: [2, 4, 8, 16] - kv-offloading: dram - kv-offload-backend: { name: hicache } + conc-list: [1, 4] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + - search-space: + - spec-decoding: mtp + conc-list: [8] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml" - # Aggregated worker: prefill's GPUs also serve decode, so decode has no - # separate allocation (num-worker: 0). Deployment is recipe-driven - # (agg_workers: 1); this only makes GPU accounting count 4, not 4+4. + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml" decode: num-worker: 0 tp: 4 @@ -7989,13 +8002,13 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: dp-attn: false dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 + image: lmsysorg/sglang:nightly-dev-cu13-20260811-d59c1ddf model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev20260718" } + router: { name: dynamo-router, version: "1.4.0" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -8013,6 +8026,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -8029,6 +8044,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -8045,6 +8062,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -8061,6 +8080,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -8077,6 +8098,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" decode: num-worker: 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0559902cc..c8507db48 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6086,3 +6086,15 @@ - "Use TP8/EP1 concurrency 2 through 24, published FlashInfer 0.6.17 CUDA 13 wheels, and golden MTP acceptance length 3.39." - "Use the lmsysorg/sglang:nightly-dev-cu13-20260815-a5ba081f image." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2592 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Tune the GB300 SGLang AgentX aggregate low-concurrency frontier with TP8 concurrency 1/4 and TP4 concurrency 8 profiles." + - "Update the aggregate and disaggregated configurations to SGLang nightly-dev-cu13-20260811-d59c1ddf and the Dynamo 1.4.0 release." + - "Use phase-specific decode CUDA graph limits and throughput-only synthetic MTP acceptance; eval-only runs retain real target verification." + - "Use an eight-hour DSV4 AgentX allocation and pass the configured GB300 Slurm account to image-validation steps." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2644 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 88b964a8a..b682e97d2 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -118,7 +118,7 @@ NGINX_SQUASH_FILE="/data/home/sa-shared/gharunners/squash/$(echo "$NGINX_IMAGE" import_squash() { local squash="$1" image="$2" local lock="${squash}.lock" - srun --partition=$SLURM_PARTITION --exclusive --time=180 bash -c " + srun --account="$SLURM_ACCOUNT" --partition="$SLURM_PARTITION" --exclusive --time=180 bash -c " exec 9>\"$lock\" flock -w 600 9 || { echo 'Failed to acquire lock for $squash' >&2; exit 1; } if unsquashfs -l \"$squash\" > /dev/null 2>&1; then @@ -175,7 +175,7 @@ if [[ "$USES_DCGM_POWER" == "1" ]]; then # x86, nodes aarch64). import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE" test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } - srun --partition=$SLURM_PARTITION --exclusive --time=30 bash -c "unsquashfs -l \"$DCGM_EXPORTER_SQSH\" > /dev/null" || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; } + srun --account="$SLURM_ACCOUNT" --partition="$SLURM_PARTITION" --exclusive --time=30 bash -c "unsquashfs -l \"$DCGM_EXPORTER_SQSH\" > /dev/null" || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; } sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" fi @@ -383,13 +383,17 @@ echo "Configs available at: $SRT_REPO_DIR/" # Create srtslurm.yaml for srtctl (used by both frameworks) SRTCTL_ROOT="${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." +SRT_DEFAULT_TIME_LIMIT="4:00:00" +if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-sglang" && "$MODEL_PREFIX" == "dsv4" ]]; then + SRT_DEFAULT_TIME_LIMIT="8:00:00" +fi cat > srtslurm.yaml <