Skip to content

Add Qwen3.5 NVFP4 V2 GB200 Dynamo-SGLang MTP / 新增 Qwen3.5 NVFP4 V2 GB200 Dynamo-SGLang MTP 配置 #2594

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
RohitNagraj wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Add Qwen3.5 NVFP4 V2 GB200 Dynamo-SGLang MTP / 新增 Qwen3.5 NVFP4 V2 GB200 Dynamo-SGLang MTP 配置 #2594

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
320cf7e
feat(config): add Qwen3.5 FP4 GB200 Dynamo-SGLang MTP
RohitNagraj Aug 13, 2026
0cab0a8
chore(changelog): link PR #2594
RohitNagraj Aug 13, 2026
19f0847
fix(changelog): link PR #2594 to Qwen3.5 entry
RohitNagraj Aug 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 169 additions & 0 deletions ...ks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
# Qwen3.5 NVFP4 V2 benchmark operating point.
# Topology and serving knobs are pinned for reproducibility.
name: qwen35-v2-gb200-c1-1p1d-tp4-mtp3-baseline
dynamo:
install: false
frontend:
type: dynamo
enable_multiple_frontends: false
args:
enforce-disagg: true
model:
path: qwen3.5-fp4
container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928
precision: fp4
resources:
gpu_type: gb200
gpus_per_node: 4
prefill_nodes: 1
decode_nodes: 1
prefill_workers: 1
decode_workers: 1
backend:
type: sglang
prefill_environment:
NO_COLOR: '1'
PYTHONUNBUFFERED: '1'
HF_HUB_OFFLINE: '1'
TRANSFORMERS_OFFLINE: '1'
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600'
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600'
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600'
NCCL_MNNVL_ENABLE: '1'
NCCL_NVLS_ENABLE: '0'
NCCL_CUMEM_ENABLE: '1'
MC_FORCE_MNNVL: '1'
SGLANG_ENABLE_SPEC_V2: '1'
SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache
FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache
SGLANG_ENABLE_JIT_DEEPGEMM: 'true'
SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
FLASHINFER_DISABLE_VERSION_CHECK: '1'
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
SGLANG_HEALTH_CHECK_TIMEOUT: '3600'
SGLANG_HEALTH_STARTING_OK: '1'
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0'
decode_environment:
NO_COLOR: '1'
PYTHONUNBUFFERED: '1'
HF_HUB_OFFLINE: '1'
TRANSFORMERS_OFFLINE: '1'
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600'
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600'
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600'
NCCL_MNNVL_ENABLE: '1'
NCCL_NVLS_ENABLE: '0'
NCCL_CUMEM_ENABLE: '1'
MC_FORCE_MNNVL: '1'
MC_TE_METRIC: 'true'
SGLANG_ENABLE_SPEC_V2: '1'
SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache
FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache
SGLANG_ENABLE_JIT_DEEPGEMM: 'true'
SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
FLASHINFER_DISABLE_VERSION_CHECK: '1'
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000'
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
SGLANG_HEALTH_CHECK_TIMEOUT: '3600'
SGLANG_HEALTH_STARTING_OK: '1'
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0'
sglang_config:
prefill:
served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
model-path: /model
trust-remote-code: true
reasoning-parser: qwen3
tool-call-parser: qwen3_coder
quantization: modelopt_mixed
fp4-gemm-backend: flashinfer_cutlass
kv-cache-dtype: fp8_e4m3
tensor-parallel-size: 4
data-parallel-size: 1
expert-parallel-size: 1
mamba-scheduler-strategy: no_buffer
mamba-ssm-dtype: bfloat16
mamba-track-interval: 2048
attention-backend: trtllm_mha
mm-attention-backend: triton_attn
moe-runner-backend: flashinfer_trtllm
linear-attn-decode-backend: flashinfer
disaggregation-mode: prefill
disaggregation-transfer-backend: mooncake
disaggregation-bootstrap-port: 31000
disable-radix-cache: true
mem-fraction-static: 0.8
context-length: 9236
max-total-tokens: 128000
max-running-requests: 128
cuda-graph-max-bs: 4
chunked-prefill-size: 32768
max-prefill-tokens: 32768
scheduler-recv-interval: 10
stream-interval: 30
load-balance-method: round_robin
page-size: 64
watchdog-timeout: 1000000
log-level: info
decode:
served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
model-path: /model
trust-remote-code: true
reasoning-parser: qwen3
tool-call-parser: qwen3_coder
quantization: modelopt_mixed
fp4-gemm-backend: flashinfer_cutlass
kv-cache-dtype: fp8_e4m3
tensor-parallel-size: 4
data-parallel-size: 1
expert-parallel-size: 1
mamba-scheduler-strategy: no_buffer
mamba-ssm-dtype: bfloat16
mamba-track-interval: 128
attention-backend: trtllm_mha
mm-attention-backend: triton_attn
moe-runner-backend: flashinfer_trtllm
linear-attn-decode-backend: flashinfer
speculative-algorithm: NEXTN
speculative-num-steps: 3
speculative-eagle-topk: 1
speculative-num-draft-tokens: 4
disaggregation-mode: decode
disaggregation-transfer-backend: mooncake
disaggregation-bootstrap-port: 31000
disable-radix-cache: true
mem-fraction-static: 0.8
context-length: 9236
max-total-tokens: 1500000
max-mamba-cache-size: 256
max-running-requests: 128
cuda-graph-max-bs: 256
chunked-prefill-size: 32768
max-prefill-tokens: 32768
scheduler-recv-interval: 10
stream-interval: 30
page-size: 64
watchdog-timeout: 1000000
decode-log-interval: 1
health_check:
max_attempts: 240
interval_seconds: 10
benchmark:
type: sa-bench
isl: 8192
osl: 1024
concurrencies: '1'
req_rate: inf
random_range_ratio: 0.8
num_prompts_mult: 10
num_warmup_mult: 2
use_chat_template: true
169 changes: 169 additions & 0 deletions ...ks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
# Qwen3.5 NVFP4 V2 benchmark operating point.
# Topology and serving knobs are pinned for reproducibility.
name: qwen35-v2-gb200-c4-1p1d-tp4-mtp3-prefill-recv1
dynamo:
install: false
frontend:
type: dynamo
enable_multiple_frontends: false
args:
enforce-disagg: true
model:
path: qwen3.5-fp4
container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928
precision: fp4
resources:
gpu_type: gb200
gpus_per_node: 4
prefill_nodes: 1
decode_nodes: 1
prefill_workers: 1
decode_workers: 1
backend:
type: sglang
prefill_environment:
NO_COLOR: '1'
PYTHONUNBUFFERED: '1'
HF_HUB_OFFLINE: '1'
TRANSFORMERS_OFFLINE: '1'
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600'
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600'
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600'
NCCL_MNNVL_ENABLE: '1'
NCCL_NVLS_ENABLE: '0'
NCCL_CUMEM_ENABLE: '1'
MC_FORCE_MNNVL: '1'
SGLANG_ENABLE_SPEC_V2: '1'
SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache
FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache
SGLANG_ENABLE_JIT_DEEPGEMM: 'true'
SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
FLASHINFER_DISABLE_VERSION_CHECK: '1'
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
SGLANG_HEALTH_CHECK_TIMEOUT: '3600'
SGLANG_HEALTH_STARTING_OK: '1'
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0'
decode_environment:
NO_COLOR: '1'
PYTHONUNBUFFERED: '1'
HF_HUB_OFFLINE: '1'
TRANSFORMERS_OFFLINE: '1'
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600'
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600'
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600'
NCCL_MNNVL_ENABLE: '1'
NCCL_NVLS_ENABLE: '0'
NCCL_CUMEM_ENABLE: '1'
MC_FORCE_MNNVL: '1'
MC_TE_METRIC: 'true'
SGLANG_ENABLE_SPEC_V2: '1'
SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache
FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache
SGLANG_ENABLE_JIT_DEEPGEMM: 'true'
SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
FLASHINFER_DISABLE_VERSION_CHECK: '1'
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000'
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
SGLANG_HEALTH_CHECK_TIMEOUT: '3600'
SGLANG_HEALTH_STARTING_OK: '1'
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0'
sglang_config:
prefill:
served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
model-path: /model
trust-remote-code: true
reasoning-parser: qwen3
tool-call-parser: qwen3_coder
quantization: modelopt_mixed
fp4-gemm-backend: flashinfer_cutlass
kv-cache-dtype: fp8_e4m3
tensor-parallel-size: 4
data-parallel-size: 1
expert-parallel-size: 1
mamba-scheduler-strategy: no_buffer
mamba-ssm-dtype: bfloat16
mamba-track-interval: 2048
attention-backend: trtllm_mha
mm-attention-backend: triton_attn
moe-runner-backend: flashinfer_trtllm
linear-attn-decode-backend: flashinfer
disaggregation-mode: prefill
disaggregation-transfer-backend: mooncake
disaggregation-bootstrap-port: 31000
disable-radix-cache: true
mem-fraction-static: 0.8
context-length: 9236
max-total-tokens: 128000
max-running-requests: 128
cuda-graph-max-bs: 4
chunked-prefill-size: 32768
max-prefill-tokens: 32768
scheduler-recv-interval: 1
stream-interval: 30
load-balance-method: round_robin
page-size: 64
watchdog-timeout: 1000000
log-level: info
decode:
served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
model-path: /model
trust-remote-code: true
reasoning-parser: qwen3
tool-call-parser: qwen3_coder
quantization: modelopt_mixed
fp4-gemm-backend: flashinfer_cutlass
kv-cache-dtype: fp8_e4m3
tensor-parallel-size: 4
data-parallel-size: 1
expert-parallel-size: 1
mamba-scheduler-strategy: no_buffer
mamba-ssm-dtype: bfloat16
mamba-track-interval: 128
attention-backend: trtllm_mha
mm-attention-backend: triton_attn
moe-runner-backend: flashinfer_trtllm
linear-attn-decode-backend: flashinfer
speculative-algorithm: NEXTN
speculative-num-steps: 3
speculative-eagle-topk: 1
speculative-num-draft-tokens: 4
disaggregation-mode: decode
disaggregation-transfer-backend: mooncake
disaggregation-bootstrap-port: 31000
disable-radix-cache: true
mem-fraction-static: 0.8
context-length: 9236
max-total-tokens: 1500000
max-mamba-cache-size: 256
max-running-requests: 128
cuda-graph-max-bs: 256
chunked-prefill-size: 32768
max-prefill-tokens: 32768
scheduler-recv-interval: 10
stream-interval: 30
page-size: 64
watchdog-timeout: 1000000
decode-log-interval: 1
health_check:
max_attempts: 240
interval_seconds: 10
benchmark:
type: sa-bench
isl: 8192
osl: 1024
concurrencies: '4'
req_rate: inf
random_range_ratio: 0.8
num_prompts_mult: 10
num_warmup_mult: 2
use_chat_template: true
Loading
Loading