diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml new file mode 100644 index 000000000..fe5ba9d5a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml @@ -0,0 +1,137 @@ +name: "agg-b300-dep4-mtp-kvoffload" + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "b300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "14400" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + # Let srt-slurm allocate and register the worker-local publisher endpoint; + # Dynamo needs discovery metadata in addition to the SGLang CLI flag. + kv_events_config: + aggregated: true + aggregated_environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + trust-remote-code: true + enable-metrics: true + enable-cache-report: true + # The AgentX corpus contains a small number of traces longer than the + # runtime KV capacity. Match the existing native B300 launcher: truncate + # those requests instead of leaving Dynamo streams unresolved. + allow-auto-truncate: true + # DSpark draft weights raise the measured minimum viable fraction to + # 0.9258 on B300; use the same 0.94 fraction as the GB300 TP4 recipe. + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + max-running-requests: 128 + cuda-graph-max-bs: 32 + stream-interval: 20 + incremental-streaming-output: true + watchdog-timeout: 1800 + dp-size: 4 + tp-size: 4 + ep-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + enable-prefill-delayer: true + prefill-delayer-max-delay-passes: 30 + moe-dense-tp-size: 1 + load-balance-method: total_tokens + moe-a2a-backend: megamoe + enable-hierarchical-cache: true + # DEP4's smaller device KV pool otherwise leaves only ~28 GB of host + # prefix cache per node; ratio 75 raises it to ~700 GB, matching the + # effective per-session capacity of the validated DEP8 configuration. + hicache-ratio: 75 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + weight-loader-drop-cache-after-load: true + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml new file mode 100644 index 000000000..a57e8c367 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml @@ -0,0 +1,136 @@ +name: "agg-b300-dep8-mtp-kvoffload" + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "b300" + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "14400" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + # Let srt-slurm allocate and register the worker-local publisher endpoint; + # Dynamo needs discovery metadata in addition to the SGLang CLI flag. + kv_events_config: + aggregated: true + aggregated_environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + trust-remote-code: true + enable-metrics: true + enable-cache-report: true + # The AgentX corpus contains a small number of traces longer than the + # runtime KV capacity. Match the existing native B300 launcher: truncate + # those requests instead of leaving Dynamo streams unresolved. + allow-auto-truncate: true + # Leave activation headroom for 8K/rank chunked prefill; 0.90 OOMs + # under the c128 AgentX warmup while 0.85 completes the full run. + mem-fraction-static: 0.85 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + max-running-requests: 1152 + cuda-graph-max-bs: 144 + stream-interval: 20 + incremental-streaming-output: true + watchdog-timeout: 1800 + dp-size: 8 + tp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-prefill-delayer: true + prefill-delayer-max-delay-passes: 30 + moe-dense-tp-size: 1 + load-balance-method: total_tokens + moe-a2a-backend: megamoe + enable-hierarchical-cache: true + # Keep the explicit host KV pool within the x86 B300 node's 3 TB DRAM + # budget; ratio 2.0 OOMs during startup before the benchmark can run. + hicache-ratio: 1.5 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + weight-loader-drop-cache-after-load: true + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 2f73b1c64..ac7b96152 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8705,6 +8705,7 @@ glm5.1-fp8-b200-tilert: dp-attn: false additional-settings: - "DECODE_NODES=1" + - isl: 8192 osl: 1024 search-space: @@ -8725,3 +8726,54 @@ glm5.1-fp8-b200-tilert: dp-attn: false additional-settings: - "DECODE_NODES=1" + +dsv4-fp4-b300-dynamo-sglang-agentic-mtp-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:b300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.95 + search-space: + - spec-decoding: draft_model + conc-list: [48, 64] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-b300-dep4-mtp-kvoffload.yaml" + decode: + num-worker: 0 + tp: 4 + ep: 4 + dp-attn: true + - spec-decoding: draft_model + conc-list: [128, 256, 384, 512, 576] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-b300-dep8-mtp-kvoffload.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 8 + dp-attn: true diff --git a/golden_al_distribution/dsv4_dspark_probabilistic.yaml b/golden_al_distribution/dsv4_dspark_probabilistic.yaml new file mode 100644 index 000000000..f72fe9431 --- /dev/null +++ b/golden_al_distribution/dsv4_dspark_probabilistic.yaml @@ -0,0 +1,19 @@ +deepseek-v4-pro-0813: + thinking_on: + 1: 1.84 + 2: 2.51 + 3: 3.01 + 4: 3.36 + 5: 3.61 + 6: 3.77 + 7: 3.73 + 8: 3.47 + thinking_off: + 1: 1.94 + 2: 2.81 + 3: 3.58 + 4: 4.17 + 5: 4.74 + 6: 5.12 + 7: 5.24 + 8: 4.95 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 085e9cb88..f1b0b98d9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6039,7 +6039,6 @@ - "EVAL_ONLY runs switch to real block verification by removing synthetic acceptance (TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS stripped from all agentx recipes before srtctl)." - "Add SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 to all 6 agentx recipe env blocks and make build_replay_cmd prefer SERVED_MODEL_NAME over $MODEL for the aiperf --model flag." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2612 - - config-keys: - glm5.2-fp4-gb200-dynamo-sglang-agentic-agg - glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg @@ -6049,3 +6048,14 @@ - "Add GB200 GLM-5.2 FP4 Dynamo-SGLang AgentX with EAGLE MTP, HiCache DRAM offload, and KV-aware correlation-ID affinity." - "Measure the TP8 aggregate curve and tuned 1P1D TP4 HiCache c10/c12 disaggregated frontier with every logical SGLang metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2620 + +- config-keys: + - dsv4-fp4-b300-dynamo-sglang-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Add B300 Dynamo-SGLang DSpark6 AgentX aggregate DEP4 and DEP8 configurations with session-affinity routing, HiCache host offload, an 8K prefill window, and thinking-on block-size-6 golden synthetic acceptance length 3.77 for throughput; eval retains real verification." + - "Use the August 16 SGLang CUDA 13 nightly and srt-slurm v1.0.38, with DEP4 concurrency 48-64 and DEP8 concurrency 128-576." + - "Cap the DEP8 HiCache ratio at 1.5 so the explicit host KV pool fits within the B300 x86 node's 3 TB DRAM budget during model loading and JIT startup." + - "Use the official DeepSeek-V4-Pro-0813 model identity so AIPerf follows the standard --tokenizer $MODEL path and reuses the shared Hugging Face cache." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2624 diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index f7878a8ad..d42ef5e66 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -1,5 +1,7 @@ #!/usr/bin/bash +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" + # System-specific configuration for B300 NV Slurm cluster (sa-shared) SLURM_PARTITION="batch_1" SLURM_ACCOUNT="benchmark" @@ -41,6 +43,9 @@ elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo- fi export MODEL_PATH="${SELECTED_MODEL_PATH:-/data/models/dsv4-pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" +elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" && $MODEL == "deepseek-ai/DeepSeek-V4-Pro-0813" ]]; then + export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro-0813}" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-0813" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" ]]; then export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" @@ -70,7 +75,17 @@ if [ -d "$SRT_REPO_DIR" ]; then fi # TODO(CJQ): make first class upon srt-slurm upstream refactor -if [[ "$IS_AGENTIC" == "1" ]]; then +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then + # Match the GB300 DSV4 AgentX path: v1.0.38 provides --no-preflight, + # session-affinity frontend support, backend metric injection for custom + # benchmarks, and the current aggregated Dynamo-SGLang recipe schema. + git clone --branch v1.0.38 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + mkdir -p recipes/sglang/deepseek-v4/agentic + cp -R \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/." \ + recipes/sglang/deepseek-v4/agentic +elif [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then @@ -135,9 +150,45 @@ NGINX_IMAGE="nginx:1.27.4" SQUASH_FILE="/data/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" NGINX_SQUASH_FILE="/data/squash/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" -# Import containers via enroot -srun -N 1 -A $SLURM_ACCOUNT -p $SLURM_PARTITION bash -c "enroot import -o $SQUASH_FILE docker://$IMAGE" -srun -N 1 -A $SLURM_ACCOUNT -p $SLURM_PARTITION bash -c "enroot import -o $NGINX_SQUASH_FILE docker://$NGINX_IMAGE" +# Import containers via enroot. Perform the validity check on the allocated +# compute node, not before submitting srun: a job can wait in the queue while +# another runner creates the same shared squash file. Serialize imports so a +# concurrent cache miss cannot race on the output either. +import_squash() { + local squash_file="$1" + local image_ref="$2" + + srun -N 1 -A "$SLURM_ACCOUNT" -p "$SLURM_PARTITION" \ + bash -s -- "$squash_file" "$image_ref" <<'IMPORT_SQUASH' +set -euo pipefail +squash_file="$1" +image_ref="$2" +lock_dir="$(dirname "$squash_file")/.locks" +mkdir -p "$lock_dir" +lock_file="$lock_dir/$(basename "$squash_file").lock" + +( + flock -w 3600 9 || { + echo "Failed to acquire lock for $squash_file" >&2 + exit 1 + } + if unsquashfs -l "$squash_file" > /dev/null 2>&1; then + echo "Squash file already exists and is valid, skipping import: $squash_file" + else + rm -f "$squash_file" + enroot import -o "$squash_file" "docker://$image_ref" + unsquashfs -l "$squash_file" > /dev/null 2>&1 || { + echo "enroot import did not produce a valid squash file: $squash_file" >&2 + exit 1 + } + chmod a+r "$squash_file" || true + fi +) 9>"$lock_file" +IMPORT_SQUASH +} + +import_squash "$SQUASH_FILE" "$IMAGE" +import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" export ISL="$ISL" export OSL="$OSL" @@ -171,6 +222,7 @@ containers: use_exclusive_sbatch_directive: true default_mounts: "/opt/ucx-no-ud": "/usr/local/ucx" + "${GITHUB_WORKSPACE}": "/infmax-workspace" EOF echo "Generated srtslurm.yaml:" @@ -200,6 +252,11 @@ fi # Override the job name in the recipe with the runner name. sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" + +# Throughput recipes opt into synthetic acceptance through the master config. +# Eval-only jobs leave the checked-in real-MTP recipe unchanged so generated +# tokens still pass target-model verification. +inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 if [[ "$MODEL_PREFIX" == "minimaxm3" && -n "$MINIMAX_M3_SLURM_EXCLUDED_NODELIST" ]]; then sed -i "/^name:.*/a sbatch_directives:\n exclude: \"${MINIMAX_M3_SLURM_EXCLUDED_NODELIST}\"" "$CONFIG_PATH" fi @@ -483,9 +540,17 @@ else export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-480}" - salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT -N 1 --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME" + if ! salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT -N 1 --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME"; then + echo "Failed to allocate B300 Slurm resources for account=$SLURM_ACCOUNT partition=$SLURM_PARTITION" >&2 + exit 1 + fi JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1) + if [[ -z "$JOB_ID" ]]; then + echo "Slurm allocation succeeded but no job ID was found for $RUNNER_NAME" >&2 + exit 1 + fi + srun --jobid=$JOB_ID \ --mpi=none \ --container-image=$SQUASH_FILE \