From 5ab44aa23ec13334f41b3ba1d355256be1e0c6b2 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 13 Aug 2026 17:35:48 -0700 Subject: [PATCH 1/3] feat(config): add DSV4 FP4 B300 Dynamo TRT recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加 DSV4 FP4 B300 Dynamo TensorRT-LLM 配置及配套配方。 --- ...-b300-8k-10p1d-dep4-dep8-b256-mtp1-mm.yaml | 156 ++++++++++ ...-b300-8k-10p1d-dep4-dep8-b512-mtp0-mm.yaml | 182 +++++++++++ ...-b300-8k-12p1d-dep4-dep8-b512-mtp1-mm.yaml | 188 ++++++++++++ ...sagg-b300-8k-1p1d-dep4-tep8-b128-mtp0.yaml | 134 ++++++++ ...isagg-b300-8k-1p2d-dep4-tep8-b16-mtp3.yaml | 126 ++++++++ ...isagg-b300-8k-1p3d-dep4-tep8-b32-mtp0.yaml | 122 ++++++++ ...disagg-b300-8k-1p4d-dep4-tep8-b2-mtp0.yaml | 118 ++++++++ ...disagg-b300-8k-1p4d-dep4-tep8-b8-mtp0.yaml | 119 ++++++++ ...disagg-b300-8k-1p5d-dep4-tep4-b1-mtp3.yaml | 124 ++++++++ ...isagg-b300-8k-2p1d-dep4-dep8-b16-mtp3.yaml | 126 ++++++++ ...isagg-b300-8k-2p1d-dep4-dep8-b32-mtp0.yaml | 122 ++++++++ ...disagg-b300-8k-2p1d-dep4-dep8-b8-mtp3.yaml | 125 ++++++++ ...isagg-b300-8k-2p3d-dep4-tep8-b32-mtp3.yaml | 128 ++++++++ ...isagg-b300-8k-2p3d-dep4-tep8-b64-mtp0.yaml | 126 ++++++++ ...gg-b300-8k-3p1d-dep4-dep8-b32-mtp3-mm.yaml | 128 ++++++++ ...isagg-b300-8k-3p1d-dep4-dep8-b64-mtp0.yaml | 126 ++++++++ ...sagg-b300-8k-5p1d-dep4-dep8-b128-mtp0.yaml | 134 ++++++++ ...isagg-b300-8k-5p1d-dep4-dep8-b64-mtp1.yaml | 132 ++++++++ ...g-b300-8k-7p1d-dep4-dep8-b128-mtp3-mm.yaml | 140 +++++++++ ...g-b300-8k-7p1d-dep4-dep8-b256-mtp0-mm.yaml | 150 +++++++++ configs/nvidia-master.yaml | 286 ++++++++++++++++++ perf-changelog.yaml | 9 + runners/launch_b300-nv.sh | 54 +++- 23 files changed, 3049 insertions(+), 6 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-10p1d-dep4-dep8-b256-mtp1-mm.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-10p1d-dep4-dep8-b512-mtp0-mm.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-12p1d-dep4-dep8-b512-mtp1-mm.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p1d-dep4-tep8-b128-mtp0.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p2d-dep4-tep8-b16-mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p3d-dep4-tep8-b32-mtp0.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p4d-dep4-tep8-b2-mtp0.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p4d-dep4-tep8-b8-mtp0.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p5d-dep4-tep4-b1-mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b16-mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b32-mtp0.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b8-mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p3d-dep4-tep8-b32-mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p3d-dep4-tep8-b64-mtp0.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-3p1d-dep4-dep8-b32-mtp3-mm.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-3p1d-dep4-dep8-b64-mtp0.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-5p1d-dep4-dep8-b128-mtp0.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-5p1d-dep4-dep8-b64-mtp1.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-7p1d-dep4-dep8-b128-mtp3-mm.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-7p1d-dep4-dep8-b256-mtp0-mm.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-10p1d-dep4-dep8-b256-mtp1-mm.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-10p1d-dep4-dep8-b256-mtp1-mm.yaml new file mode 100644 index 0000000000..594e429b83 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-10p1d-dep4-dep8-b256-mtp1-mm.yaml @@ -0,0 +1,156 @@ +name: disagg-b300-8k-10p1d-dep4-dep8-b256-mtp1-mm +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 10 + prefill_workers: 10 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 256 + max_num_tokens: 512 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "2253" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-10p1d-dep4-dep8-b512-mtp0-mm.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-10p1d-dep4-dep8-b512-mtp0-mm.yaml new file mode 100644 index 0000000000..89d3bf2b65 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-10p1d-dep4-dep8-b512-mtp0-mm.yaml @@ -0,0 +1,182 @@ +name: disagg-b300-8k-10p1d-dep4-dep8-b512-mtp0-mm +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 10 + prefill_workers: 10 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 512 + max_num_tokens: 512 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "4301" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-12p1d-dep4-dep8-b512-mtp1-mm.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-12p1d-dep4-dep8-b512-mtp1-mm.yaml new file mode 100644 index 0000000000..ba970a9c4b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-12p1d-dep4-dep8-b512-mtp1-mm.yaml @@ -0,0 +1,188 @@ +name: disagg-b300-8k-12p1d-dep4-dep8-b512-mtp1-mm +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 12 + prefill_workers: 12 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 512 + max_num_tokens: 1024 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "4301" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p1d-dep4-tep8-b128-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p1d-dep4-tep8-b128-mtp0.yaml new file mode 100644 index 0000000000..c8daf6d3b4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p1d-dep4-tep8-b128-mtp0.yaml @@ -0,0 +1,134 @@ +name: disagg-b300-8k-1p1d-dep4-tep8-b128-mtp0 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "180" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p2d-dep4-tep8-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p2d-dep4-tep8-b16-mtp3.yaml new file mode 100644 index 0000000000..01aa5cedaa --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p2d-dep4-tep8-b16-mtp3.yaml @@ -0,0 +1,126 @@ +name: disagg-b300-8k-1p2d-dep4-tep8-b16-mtp3 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 2 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8320 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "48" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p3d-dep4-tep8-b32-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p3d-dep4-tep8-b32-mtp0.yaml new file mode 100644 index 0000000000..f87aa0a233 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p3d-dep4-tep8-b32-mtp0.yaml @@ -0,0 +1,122 @@ +name: disagg-b300-8k-1p3d-dep4-tep8-b32-mtp0 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 3 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "144" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p4d-dep4-tep8-b2-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p4d-dep4-tep8-b2-mtp0.yaml new file mode 100644 index 0000000000..620c44045e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p4d-dep4-tep8-b2-mtp0.yaml @@ -0,0 +1,118 @@ +name: disagg-b300-8k-1p4d-dep4-tep8-b2-mtp0 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 4 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 2 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "12" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p4d-dep4-tep8-b8-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p4d-dep4-tep8-b8-mtp0.yaml new file mode 100644 index 0000000000..97ff856615 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p4d-dep4-tep8-b8-mtp0.yaml @@ -0,0 +1,119 @@ +name: disagg-b300-8k-1p4d-dep4-tep8-b8-mtp0 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 4 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 8 + max_num_tokens: 8 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "44" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p5d-dep4-tep4-b1-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p5d-dep4-tep4-b1-mtp3.yaml new file mode 100644 index 0000000000..0dccd17d46 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p5d-dep4-tep4-b1-mtp3.yaml @@ -0,0 +1,124 @@ +name: disagg-b300-8k-1p5d-dep4-tep4-b1-mtp3 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 5 + decode_workers: 5 + gpus_per_decode: 4 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8320 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 4 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "10" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b16-mtp3.yaml new file mode 100644 index 0000000000..b3845024b4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b16-mtp3.yaml @@ -0,0 +1,126 @@ +name: disagg-b300-8k-2p1d-dep4-dep8-b16-mtp3 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8320 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "180" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b32-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b32-mtp0.yaml new file mode 100644 index 0000000000..a90afd46f7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b32-mtp0.yaml @@ -0,0 +1,122 @@ +name: disagg-b300-8k-2p1d-dep4-dep8-b32-mtp0 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "282" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b8-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b8-mtp3.yaml new file mode 100644 index 0000000000..933a4657e6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b8-mtp3.yaml @@ -0,0 +1,125 @@ +name: disagg-b300-8k-2p1d-dep4-dep8-b8-mtp3 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8320 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "90" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p3d-dep4-tep8-b32-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p3d-dep4-tep8-b32-mtp3.yaml new file mode 100644 index 0000000000..37638a6d3d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p3d-dep4-tep8-b32-mtp3.yaml @@ -0,0 +1,128 @@ +name: disagg-b300-8k-2p3d-dep4-tep8-b32-mtp3 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 3 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8320 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "144" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p3d-dep4-tep8-b64-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p3d-dep4-tep8-b64-mtp0.yaml new file mode 100644 index 0000000000..91bd8e2edb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p3d-dep4-tep8-b64-mtp0.yaml @@ -0,0 +1,126 @@ +name: disagg-b300-8k-2p3d-dep4-tep8-b64-mtp0 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 3 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "270" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-3p1d-dep4-dep8-b32-mtp3-mm.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-3p1d-dep4-dep8-b32-mtp3-mm.yaml new file mode 100644 index 0000000000..1c35c867e6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-3p1d-dep4-dep8-b32-mtp3-mm.yaml @@ -0,0 +1,128 @@ +name: disagg-b300-8k-3p1d-dep4-dep8-b32-mtp3-mm +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 3 + prefill_workers: 3 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8320 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "333" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-3p1d-dep4-dep8-b64-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-3p1d-dep4-dep8-b64-mtp0.yaml new file mode 100644 index 0000000000..13e9561277 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-3p1d-dep4-dep8-b64-mtp0.yaml @@ -0,0 +1,126 @@ +name: disagg-b300-8k-3p1d-dep4-dep8-b64-mtp0 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 3 + prefill_workers: 3 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "666" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-5p1d-dep4-dep8-b128-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-5p1d-dep4-dep8-b128-mtp0.yaml new file mode 100644 index 0000000000..48e06bc017 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-5p1d-dep4-dep8-b128-mtp0.yaml @@ -0,0 +1,134 @@ +name: disagg-b300-8k-5p1d-dep4-dep8-b128-mtp0 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 5 + prefill_workers: 5 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "1229" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-5p1d-dep4-dep8-b64-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-5p1d-dep4-dep8-b64-mtp1.yaml new file mode 100644 index 0000000000..ab2937092f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-5p1d-dep4-dep8-b64-mtp1.yaml @@ -0,0 +1,132 @@ +name: disagg-b300-8k-5p1d-dep4-dep8-b64-mtp1 +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 5 + prefill_workers: 5 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 64 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "666" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-7p1d-dep4-dep8-b128-mtp3-mm.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-7p1d-dep4-dep8-b128-mtp3-mm.yaml new file mode 100644 index 0000000000..02d13c4a97 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-7p1d-dep4-dep8-b128-mtp3-mm.yaml @@ -0,0 +1,140 @@ +name: disagg-b300-8k-7p1d-dep4-dep8-b128-mtp3-mm +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 7 + prefill_workers: 7 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8320 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "1229" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-7p1d-dep4-dep8-b256-mtp0-mm.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-7p1d-dep4-dep8-b256-mtp0-mm.yaml new file mode 100644 index 0000000000..3f71fe28cd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-7p1d-dep4-dep8-b256-mtp0-mm.yaml @@ -0,0 +1,150 @@ +name: disagg-b300-8k-7p1d-dep4-dep8-b256-mtp0-mm +model: + path: deepseek-v4-pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + precision: fp4 +dynamo: + install: true + wheel: "1.4.0.dev20260730" + request_plane: tcp +resources: + gpu_type: b300 + prefill_nodes: 7 + prefill_workers: 7 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + gpus_per_node: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_PROFILE_LOG_RANKS: all + TLLM_METRICS_ALL_RANKS: '1' + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_PROFILE_LOG_RANKS: all + UCX_TLS: rc,cuda_copy,sm,self + UCX_MAX_RNDV_RAILS: '1' + UCX_MAX_EAGER_RAILS: '1' + NIXL_PLUGIN_DIR: /opt/nvidia/nvda_nixl/lib/x86_64-linux-gnu/plugins + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "2253" + req_rate: inf + random_range_ratio: 0.8 + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +environment: + TRTLLM_SERVER_DISABLE_GC: '1' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index caa0a7e082..62d61a841f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8511,3 +8511,289 @@ glm5.1-fp8-b200-tilert: dp-attn: false additional-settings: - "DECODE_NODES=1" + +dsv4-fp4-b300-dynamo-trt: + image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: b300 + precision: fp4 + framework: dynamo-trt + multinode: true + disagg: true + kv-p2p-transfer: nixl + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [2253] + spec-decoding: "mtp" + prefill: + num-worker: 10 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-10p1d-dep4-dep8-b256-mtp1-mm.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [4301] + prefill: + num-worker: 10 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-10p1d-dep4-dep8-b512-mtp0-mm.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [4301] + spec-decoding: "mtp" + prefill: + num-worker: 12 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-12p1d-dep4-dep8-b512-mtp1-mm.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [180] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p1d-dep4-tep8-b128-mtp0.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [48] + spec-decoding: "mtp" + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p2d-dep4-tep8-b16-mtp3.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [144] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p3d-dep4-tep8-b32-mtp0.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [12] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p4d-dep4-tep8-b2-mtp0.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [44] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p4d-dep4-tep8-b8-mtp0.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [10] + spec-decoding: "mtp" + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-1p5d-dep4-tep4-b1-mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [180] + spec-decoding: "mtp" + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b16-mtp3.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [282] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b32-mtp0.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [90] + spec-decoding: "mtp" + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p1d-dep4-dep8-b8-mtp3.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [144] + spec-decoding: "mtp" + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p3d-dep4-tep8-b32-mtp3.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [270] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-2p3d-dep4-tep8-b64-mtp0.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [333] + spec-decoding: "mtp" + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-3p1d-dep4-dep8-b32-mtp3-mm.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [666] + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-3p1d-dep4-dep8-b64-mtp0.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [1229] + prefill: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-5p1d-dep4-dep8-b128-mtp0.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [666] + spec-decoding: "mtp" + prefill: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-5p1d-dep4-dep8-b64-mtp1.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [1229] + spec-decoding: "mtp" + prefill: + num-worker: 7 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-7p1d-dep4-dep8-b128-mtp3-mm.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [2253] + prefill: + num-worker: 7 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/deepseek-v4/8k1k/b300/disagg-b300-8k-7p1d-dep4-dep8-b256-mtp0-mm.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 04ba9feb56..a6e2932d91 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5953,3 +5953,12 @@ - "Use native EAGLE MTP (3 steps, top-k 1, 4 draft tokens) and golden synthetic acceptance length 2.49 for throughput; eval retains real verification." - "Follow the official SGLang DeepSeek-V4 Blackwell recipe, require nonempty SGLang server metrics, keep pooled AgentX connections alive, let AIPerf own HiCache warmup, and reserve transient MoE workspace at DEP8 c512." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2577 + +- config-keys: + - dsv4-fp4-b300-dynamo-trt + description: + - "Add the DeepSeek-V4-Pro FP4 B300 Dynamo TensorRT-LLM disaggregated configuration with STP and MTP recipe variants." + - "Run on b300 with the checkpoint at /scratch/models/DeepSeek-V4-Pro and batch_1/benchmark Slurm settings." + - "Stage the TensorRT-LLM recipes through launch_b300-nv.sh." + - "Image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index cad9ba30aa..92d6610595 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -27,6 +27,9 @@ elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/data/models/dsr1-fp8" export SERVED_MODEL_NAME="deepseek-r1-fp8" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" +elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-trt" ]]; then + export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro}" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then SELECTED_MODEL_PATH="" if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then @@ -57,7 +60,7 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" && $FRAMEWORK == "dy export MODEL_PATH="/data/models/MiniMax-M3-MXFP8" export SRT_SLURM_MODEL_PREFIX="MiniMaxAI/MiniMax-M3-MXFP8" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm or dynamo-sglang, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-trt, dynamo-vllm, or dynamo-sglang, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" exit 1 fi @@ -73,6 +76,19 @@ fi if [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 +elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then + git clone --branch v1.0.50 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout e4019633c9e2bc25f38c44b81edf52bb0504d937 || exit 1 + mkdir -p recipes/trtllm/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4" \ + recipes/trtllm/deepseek-v4 || exit 1 + # Dynamo installation enables enroot root remapping by default. Disable it for + # TensorRT-LLM MPI workers so the container can use the host PMIx socket. + sed -i 's/CONTAINER_REMAP_ROOT_EXPORT = {"ENROOT_REMAP_ROOT": "yes"}/CONTAINER_REMAP_ROOT_EXPORT = {"ENROOT_REMAP_ROOT": "no"}/' \ + src/srtctl/core/slurm.py + grep -Fq 'CONTAINER_REMAP_ROOT_EXPORT = {"ENROOT_REMAP_ROOT": "no"}' \ + src/srtctl/core/slurm.py || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 @@ -135,9 +151,28 @@ NGINX_IMAGE="nginx:1.27.4" SQUASH_FILE="/data/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" NGINX_SQUASH_FILE="/data/squash/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" -# Import containers via enroot -srun -N 1 -A $SLURM_ACCOUNT -p $SLURM_PARTITION bash -c "enroot import -o $SQUASH_FILE docker://$IMAGE" -srun -N 1 -A $SLURM_ACCOUNT -p $SLURM_PARTITION bash -c "enroot import -o $NGINX_SQUASH_FILE docker://$NGINX_IMAGE" +# Import containers via enroot. The DSV4 TensorRT-LLM matrix shares squash +# destinations across its jobs, so serialize imports and reuse valid images. +if [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" ]]; then + import_squash_b300() { + local squash="$1" image="$2" + srun -N 1 -A "$SLURM_ACCOUNT" -p "$SLURM_PARTITION" bash -c " + exec 9>\"${squash}.lock\" + flock -w 600 9 || { echo 'Failed to acquire lock for ${squash}'; exit 1; } + if unsquashfs -l \"${squash}\" >/dev/null 2>&1; then + echo 'Squash already exists and is valid, skipping import: ${squash}' + else + rm -f \"${squash}\" + enroot import -o \"${squash}\" \"docker://${image}\" + fi + " + } + import_squash_b300 "$SQUASH_FILE" "$IMAGE" + import_squash_b300 "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" +else + srun -N 1 -A "$SLURM_ACCOUNT" -p "$SLURM_PARTITION" bash -c "enroot import -o $SQUASH_FILE docker://$IMAGE" + srun -N 1 -A "$SLURM_ACCOUNT" -p "$SLURM_PARTITION" bash -c "enroot import -o $NGINX_SQUASH_FILE docker://$NGINX_IMAGE" +fi export ISL="$ISL" export OSL="$OSL" @@ -145,6 +180,11 @@ export EVAL_ONLY="${EVAL_ONLY:-false}" # Create srtslurm.yaml for srtctl SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" +DEFAULT_MOUNTS_BLOCK='default_mounts: + "/opt/ucx-no-ud": "/usr/local/ucx"' +if [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" ]]; then + DEFAULT_MOUNTS_BLOCK="" +fi echo "Creating srtslurm.yaml configuration..." cat > srtslurm.yaml < Date: Thu, 13 Aug 2026 17:36:55 -0700 Subject: [PATCH 2/3] chore(changelog): add pull request link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在变更日志中补充拉取请求链接。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a6e2932d91..682c94007e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2596 - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp From 598276b09a9761ffa4e8a11894af42afb953b5d8 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 13 Aug 2026 17:37:25 -0700 Subject: [PATCH 3/3] fix(changelog): correct pull request link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修正变更日志中的拉取请求链接。 --- perf-changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 682c94007e..735bf5646b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2596 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp @@ -5961,4 +5961,4 @@ - "Run on b300 with the checkpoint at /scratch/models/DeepSeek-V4-Pro and batch_1/benchmark Slurm settings." - "Stage the TensorRT-LLM recipes through launch_b300-nv.sh." - "Image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2596