diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml new file mode 100644 index 000000000..0704f959f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml @@ -0,0 +1,169 @@ +# Qwen3.5 NVFP4 V2 benchmark operating point. +# Topology and serving knobs are pinned for reproducibility. +name: qwen35-v2-gb200-c1-1p1d-tp4-mtp3-baseline +dynamo: + install: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + enforce-disagg: true +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 +backend: + type: sglang + prefill_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + decode_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 2048 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disaggregation-bootstrap-port: 31000 + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 128000 + max-running-requests: 128 + cuda-graph-max-bs: 4 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 10 + stream-interval: 30 + load-balance-method: round_robin + page-size: 64 + watchdog-timeout: 1000000 + log-level: info + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 128 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disaggregation-bootstrap-port: 31000 + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 1500000 + max-mamba-cache-size: 256 + max-running-requests: 128 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 10 + stream-interval: 30 + page-size: 64 + watchdog-timeout: 1000000 + decode-log-interval: 1 +health_check: + max_attempts: 240 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1' + req_rate: inf + random_range_ratio: 0.8 + num_prompts_mult: 10 + num_warmup_mult: 2 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml new file mode 100644 index 000000000..1f9e110f9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml @@ -0,0 +1,169 @@ +# Qwen3.5 NVFP4 V2 benchmark operating point. +# Topology and serving knobs are pinned for reproducibility. +name: qwen35-v2-gb200-c4-1p1d-tp4-mtp3-prefill-recv1 +dynamo: + install: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + enforce-disagg: true +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 +backend: + type: sglang + prefill_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + decode_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 2048 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disaggregation-bootstrap-port: 31000 + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 128000 + max-running-requests: 128 + cuda-graph-max-bs: 4 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 1 + stream-interval: 30 + load-balance-method: round_robin + page-size: 64 + watchdog-timeout: 1000000 + log-level: info + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 128 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disaggregation-bootstrap-port: 31000 + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 1500000 + max-mamba-cache-size: 256 + max-running-requests: 128 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 10 + stream-interval: 30 + page-size: 64 + watchdog-timeout: 1000000 + decode-log-interval: 1 +health_check: + max_attempts: 240 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '4' + req_rate: inf + random_range_ratio: 0.8 + num_prompts_mult: 10 + num_warmup_mult: 2 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml new file mode 100644 index 000000000..03cdcb9d8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml @@ -0,0 +1,169 @@ +# Qwen3.5 NVFP4 V2 benchmark operating point. +# Topology and serving knobs are pinned for reproducibility. +name: qwen35-v2-gb200-c8-1p1d-tp4-mtp3-decode-stream50 +dynamo: + install: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + enforce-disagg: true +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 +backend: + type: sglang + prefill_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + decode_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 2048 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disaggregation-bootstrap-port: 31000 + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 128000 + max-running-requests: 128 + cuda-graph-max-bs: 4 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 10 + stream-interval: 30 + load-balance-method: round_robin + page-size: 64 + watchdog-timeout: 1000000 + log-level: info + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 128 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disaggregation-bootstrap-port: 31000 + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 1500000 + max-mamba-cache-size: 256 + max-running-requests: 128 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 10 + stream-interval: 50 + page-size: 64 + watchdog-timeout: 1000000 + decode-log-interval: 1 +health_check: + max_attempts: 240 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '8' + req_rate: inf + random_range_ratio: 0.8 + num_prompts_mult: 10 + num_warmup_mult: 2 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml new file mode 100644 index 000000000..d1f13b24a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml @@ -0,0 +1,172 @@ +# Qwen3.5 NVFP4 V2 benchmark operating point. +# Topology and serving knobs are pinned for reproducibility. +name: qwen35-v2-gb200-c512-2p2d-tp2-mtp3-cap84 +dynamo: + install: false +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + args: + enforce-disagg: true +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 2 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 2 + spread_workers: true +backend: + type: sglang + prefill_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + ETCD_LEASE_TTL: '120' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + decode_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + ETCD_LEASE_TTL: '120' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 2048 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 64000 + max-running-requests: 64 + cuda-graph-max-bs: 4 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + scheduler-recv-interval: 10 + stream-interval: 30 + load-balance-method: round_robin + page-size: 64 + watchdog-timeout: 1000000 + log-level: info + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 128 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 750000 + max-mamba-cache-size: 128 + max-running-requests: 84 + cuda-graph-max-bs: 88 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 10 + stream-interval: 30 + page-size: 64 + watchdog-timeout: 1000000 + decode-log-interval: 1 +health_check: + max_attempts: 240 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '512' + req_rate: inf + random_range_ratio: 0.8 + num_prompts_mult: 10 + num_warmup_mult: 2 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml new file mode 100644 index 000000000..384a7c166 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml @@ -0,0 +1,168 @@ +# Qwen3.5 NVFP4 V2 benchmark operating point. +# Topology, collective settings, and serving knobs are pinned for reproducibility. +name: qwen35-v2-gb200-c768-2p3d-tp2-mtp3-cap86 +dynamo: + install: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + enforce-disagg: true +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 2 + prefill_nodes: 2 + decode_nodes: 3 + prefill_workers: 2 + decode_workers: 3 + spread_workers: true +backend: + type: sglang + prefill_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + decode_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 2048 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 64000 + max-running-requests: 64 + cuda-graph-max-bs: 4 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + scheduler-recv-interval: 10 + stream-interval: 30 + load-balance-method: round_robin + page-size: 64 + watchdog-timeout: 1000000 + log-level: info + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 128 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 750000 + max-mamba-cache-size: 128 + max-running-requests: 86 + cuda-graph-max-bs: 88 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 10 + stream-interval: 30 + page-size: 64 + watchdog-timeout: 1000000 + decode-log-interval: 1 +health_check: + max_attempts: 240 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '768' + req_rate: inf + random_range_ratio: 0.8 + num_prompts_mult: 10 + num_warmup_mult: 2 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml new file mode 100644 index 000000000..450e091ad --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml @@ -0,0 +1,169 @@ +# Qwen3.5 NVFP4 V2 benchmark operating point. +# Topology and serving knobs are pinned for reproducibility. +name: qwen35-v2-gb200-c128-3p5d-tp4-mtp3 +dynamo: + install: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + enforce-disagg: true +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 + prefill_nodes: 3 + decode_nodes: 5 + prefill_workers: 3 + decode_workers: 5 +backend: + type: sglang + prefill_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + decode_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 2048 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disaggregation-bootstrap-port: 31000 + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 128000 + max-running-requests: 128 + cuda-graph-max-bs: 4 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 10 + stream-interval: 30 + load-balance-method: round_robin + page-size: 64 + watchdog-timeout: 1000000 + log-level: info + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 128 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disaggregation-bootstrap-port: 31000 + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 1500000 + max-mamba-cache-size: 256 + max-running-requests: 128 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 10 + stream-interval: 30 + page-size: 64 + watchdog-timeout: 1000000 + decode-log-interval: 1 +health_check: + max_attempts: 240 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '128' + req_rate: inf + random_range_ratio: 0.8 + num_prompts_mult: 10 + num_warmup_mult: 2 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml new file mode 100644 index 000000000..c1c23b5be --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml @@ -0,0 +1,179 @@ +# V2 mixed quantization uses the CuteDSL MoE path at this saturated wide-EP point. +# Topology and serving knobs are pinned for reproducibility. +name: qwen35-v2-gb200-c5120-7p1d-dep4-dep16-mtp2-cap2048 +dynamo: + install: false +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + args: + enforce-disagg: true +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 + prefill_nodes: 7 + decode_nodes: 4 + prefill_workers: 7 + decode_workers: 1 +backend: + type: sglang + prefill_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + decode_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 2048 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 128000 + max-running-requests: 128 + cuda-graph-max-bs: 4 + chunked-prefill-size: 65536 + max-prefill-tokens: 65536 + scheduler-recv-interval: 1 + stream-interval: 50 + load-balance-method: round_robin + page-size: 64 + watchdog-timeout: 1000000 + log-level: info + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 128 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-a2a-backend: flashinfer + moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: none + speculative-moe-runner-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + linear-attn-decode-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + cuda-graph-max-bs: 256 + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 2200000 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + chunked-prefill-size: 4096 + max-prefill-tokens: 32768 + scheduler-recv-interval: 1 + stream-interval: 50 + page-size: 64 + watchdog-timeout: 1000000 + decode-log-interval: 50 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '5120' + req_rate: inf + random_range_ratio: 0.8 + num_prompts_mult: 10 + num_warmup_mult: 2 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml new file mode 100644 index 000000000..b5ba39378 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml @@ -0,0 +1,179 @@ +# V2 mixed quantization uses the CuteDSL MoE path at this saturated wide-EP point. +# Topology and serving knobs are pinned for reproducibility. +name: qwen35-v2-gb200-c8192-7p1d-dep4-dep16-mtp2-cap3413 +dynamo: + install: false +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + args: + enforce-disagg: true +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 + prefill_nodes: 7 + decode_nodes: 4 + prefill_workers: 7 + decode_workers: 1 +backend: + type: sglang + prefill_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + decode_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 2048 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 128000 + max-running-requests: 128 + cuda-graph-max-bs: 4 + chunked-prefill-size: 65536 + max-prefill-tokens: 65536 + scheduler-recv-interval: 1 + stream-interval: 50 + load-balance-method: round_robin + page-size: 64 + watchdog-timeout: 1000000 + log-level: info + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 128 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-a2a-backend: flashinfer + moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: none + speculative-moe-runner-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + linear-attn-decode-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + cuda-graph-max-bs: 256 + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 2200000 + max-mamba-cache-size: 3413 + max-running-requests: 3413 + chunked-prefill-size: 4096 + max-prefill-tokens: 32768 + scheduler-recv-interval: 1 + stream-interval: 50 + page-size: 64 + watchdog-timeout: 1000000 + decode-log-interval: 50 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '8192' + req_rate: inf + random_range_ratio: 0.8 + num_prompts_mult: 10 + num_warmup_mult: 2 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml new file mode 100644 index 000000000..bfa84d6cb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml @@ -0,0 +1,191 @@ +# Qwen3.5 NVFP4 V2 benchmark operating point. +# Seven DEP4 prefill workers feed four DEP4 MTP1 decode workers. +name: qwen35-v2-gb200-c12288-7p4d-dep4-mtp1-cap1056 +dynamo: + install: true + wheel: "1.2.1" + request_plane: "nats" +frontend: + type: dynamo + enable_multiple_frontends: false + args: + enforce-disagg: true +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 + prefill_nodes: 7 + decode_nodes: 4 + prefill_workers: 7 + decode_workers: 4 +backend: + type: sglang + prefill_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + decode_environment: + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600' + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600' + NCCL_DEBUG: INFO + NCCL_DEBUG_SUBSYS: COLL + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '3600' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + CUDA_ENABLE_COREDUMP_ON_EXCEPTION: '1' + CUDA_ENABLE_CPU_COREDUMP_ON_EXCEPTION: '0' + CUDA_COREDUMP_FILE: /logs/cuda_coredump_%h_%p + CUDA_COREDUMP_GENERATION_FLAGS: skip_nonrelocated_elf_images,skip_global_memory,skip_shared_memory,skip_local_memory,skip_constbank_memory + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-attention-local-control-broadcast: true + enable-dp-lm-head: true + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 2048 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 256000 + max-running-requests: 128 + cuda-graph-max-bs: 4 + chunked-prefill-size: 262144 + max-prefill-tokens: 131072 + scheduler-recv-interval: 1 + stream-interval: 50 + load-balance-method: round_robin + page-size: 64 + watchdog-timeout: 1000000 + log-level: info + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model + trust-remote-code: true + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + quantization: modelopt_mixed + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + mamba-scheduler-strategy: no_buffer + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 128 + attention-backend: trtllm_mha + mm-attention-backend: triton_attn + moe-a2a-backend: none + moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + speculative-moe-runner-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + linear-attn-decode-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + disable-flashinfer-autotune: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + cuda-graph-max-bs: 512 + mem-fraction-static: 0.94 + context-length: 9236 + max-total-tokens: 2052736 + max-mamba-cache-size: 1056 + max-running-requests: 1056 + chunked-prefill-size: 4096 + max-prefill-tokens: 32768 + scheduler-recv-interval: 1 + disaggregation-decode-polling-interval: 8 + stream-interval: 50 + page-size: 64 + soft-watchdog-timeout: 240 + watchdog-timeout: 300 + decode-log-interval: 50 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '12288' + req_rate: inf + random_range_ratio: 0.8 + num_prompts_mult: 10 + num_warmup_mult: 2 + use_chat_template: true + reuse_http_connections: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index caa0a7e08..6c3dea03c 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8084,6 +8084,148 @@ glm5.2-fp4-b200-sglang-agentic-mtp: search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } +qwen3.5-fp4-gb200-dynamo-sglang-mtp: + image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-prefix: qwen3.5 + runner: gb200 + precision: fp4 + framework: dynamo-sglang + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [4] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [8] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [512] + prefill: + num-worker: 2 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml" + decode: + num-worker: 2 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [128] + prefill: + num-worker: 3 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [768] + prefill: + num-worker: 2 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml" + decode: + num-worker: 3 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [5120] + prefill: + num-worker: 7 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [8192] + prefill: + num-worker: 7 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [12288] + prefill: + num-worker: 7 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + qwen3.5-fp8-gb200-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 04ba9feb5..deab9c100 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5953,3 +5953,14 @@ - "Use native EAGLE MTP (3 steps, top-k 1, 4 draft tokens) and golden synthetic acceptance length 2.49 for throughput; eval retains real verification." - "Follow the official SGLang DeepSeek-V4 Blackwell recipe, require nonempty SGLang server metrics, keep pooled AgentX connections alive, let AIPerf own HiCache warmup, and reserve transient MoE workspace at DEP8 c512." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2577 + +- config-keys: + - qwen3.5-fp4-gb200-dynamo-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Add a Qwen3.5 NVFP4 V2 GB200 Dynamo-SGLang disaggregated MTP configuration for the 8k/1k scenario" + - "Add nine checked-in srt-slurm recipes covering TP and data-parallel-attention topologies" + - "Use chat-formatted benchmark inputs with Mooncake KV transfer" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2594