Skip to content

feat(dsv4): add GB300 Dynamo-TensorRT-LLM AgentX recipes / 添加 GB300 Dynamo-TensorRT-LLM AgentX 配置 #2595

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
RohitNagraj wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat(dsv4): add GB300 Dynamo-TensorRT-LLM AgentX recipes / 添加 GB300 Dynamo-TensorRT-LLM AgentX 配置 #2595

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cb48e16
feat(config): add DSV4 GB300 Dynamo TRT AgentX recipes
RohitNagraj Aug 13, 2026
a2b79ed
chore(changelog): link PR 2595
RohitNagraj Aug 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 178 additions & 0 deletions ...lm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c206-b2-mtp.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
name: dynamo-disagg-gb300-1p1d-dep8-dep32-c206-b2-mtp
model:
path: deepseek-ai/DeepSeek-V4-Pro
container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24
precision: fp4
identity:
model:
repo: deepseek-ai/DeepSeek-V4-Pro
container:
image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24
frameworks:
dynamo: 1.4.0.dev20260807
tensorrt_llm: 1.3.0rc24
dynamo:
install: true
wheel: 1.4.0.dev20260807
request_plane: tcp
health_check:
max_attempts: 1440
interval_seconds: 10
resources:
gpu_type: gb300
gpus_per_node: 4
prefill_nodes: 2
prefill_workers: 1
gpus_per_prefill: 8
decode_nodes: 8
decode_workers: 1
gpus_per_decode: 32
backend:
type: trtllm
prefill_environment:
TLLM_LOG_LEVEL: INFO
TRTLLM_SERVER_DISABLE_GC: '1'
TRTLLM_WORKER_DISABLE_GC: '1'
NCCL_GRAPH_MIXING_SUPPORT: '0'
TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.97'
UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp
DYN_ENGINE_CONV_AFFINITY: '1'
MIMALLOC_PURGE_DELAY: '0'
TRTLLM_ENABLE_PDL: '1'
DYN_DEFAULT_THINKING_MODE: disabled
MIMALLOC_ARENA_RESERVE: '0'
TRTLLM_PINNED_WEIGHT_STAGING: '1'
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
CUDA_SCALE_LAUNCH_QUEUES: 4x
decode_environment:
TLLM_LOG_LEVEL: INFO
TRTLLM_SERVER_DISABLE_GC: '1'
TRTLLM_WORKER_DISABLE_GC: '1'
NCCL_GRAPH_MIXING_SUPPORT: '0'
TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.97'
UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp
DYN_ENGINE_CONV_AFFINITY: '1'
MIMALLOC_PURGE_DELAY: '0'
TRTLLM_ENABLE_PDL: '1'
DYN_DEFAULT_THINKING_MODE: disabled
MIMALLOC_ARENA_RESERVE: '0'
TRTLLM_PINNED_WEIGHT_STAGING: '1'
trtllm_config:
prefill:
attention_dp_config:
kv_cache_routing_conversation_affinity: true
cache_transceiver_config:
backend: NIXL
kv_transfer_timeout_ms: 600000
transceiver_runtime: PYTHON
cuda_graph_config: null
custom_tokenizer: deepseek_v4
disable_overlap_scheduler: false
enable_attention_dp: true
enable_chunked_prefill: true
kv_cache_config:
dtype: fp8
enable_block_reuse: true
event_buffer_max_size: 0
free_gpu_memory_fraction: 0.8
host_cache_size: 193273528320
pool_ratio:
- 0.55
- 0.22
- 0.23
tokens_per_block: 128
block_reuse_config:
policy: per_conversation
max_num_turns: 5
max_batch_size: 256
max_num_tokens: 16384
max_seq_len: 990016
moe_config:
backend: TRTLLM
load_balancer: /infmax-workspace/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/eplb_confs/moe_load_balancer_ctx_ep8_slots384.yaml
moe_expert_parallel_size: 8
pipeline_parallel_size: 1
print_iter_log: true
return_perf_metrics: true
scheduler_config:
capacity_scheduler_policy: MAX_UTILIZATION
speculative_config:
decoding_type: MTP
max_draft_len: 3
tensor_parallel_size: 8
decode:
cache_transceiver_config:
backend: NIXL
kv_transfer_timeout_ms: 600000
transceiver_runtime: PYTHON
cuda_graph_config:
batch_sizes:
- 1
- 2
- 4
enable_padding: true
custom_tokenizer: deepseek_v4
enable_attention_dp: true
enable_lm_head_tp_in_adp: true
kv_cache_config:
avg_seq_len: 200000
dtype: fp8
enable_block_reuse: false
event_buffer_max_size: 0
free_gpu_memory_fraction: 0.7
host_cache_size: 0
tokens_per_block: 128
max_batch_size: 2
max_num_tokens: 8
max_seq_len: 996595
moe_config:
backend: MEGAMOE_DEEPGEMM
use_low_precision_moe_combine: true
load_balancer: /infmax-workspace/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/eplb_confs/moe_load_balancer_gen_ep32_slots384.yaml
moe_expert_parallel_size: 32
num_postprocess_workers: 4
pipeline_parallel_size: 1
print_iter_log: true
return_perf_metrics: true
sparse_attention_config:
algorithm: deepseek_v4
enable_heuristic_topk: true
speculative_config:
decoding_type: MTP
max_draft_len: 3
stream_interval: 20
tensor_parallel_size: 32
publish_events_and_metrics: false
frontend:
type: dynamo
enable_multiple_frontends: false
env:
ETCD_LEASE_TTL: '120'
DYN_ROUTER_QUEUE_THRESHOLD: None
DYN_TOKENIZER_CACHE: '1'
DYN_TOKENIZER_CACHE_BYTES: '8000000000'
DYN_TOKENIZER: fastokens
DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400'
args:
router-mode: kv
no-kv-events: true
active-decode-blocks-threshold: None
active-prefill-tokens-threshold: None
active-prefill-tokens-threshold-frac: None
orchestrator_placement: first_decode
benchmark:
type: custom
use_chat_template: true
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
env:
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
RESULT_DIR: /logs/agentic
PORT: '8000'
IS_MULTINODE: 'true'
AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400'
AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
HF_HUB_CACHE: /hf_hub_cache
WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126
SERVED_MODEL_NAME: DeepSeek-V4-Pro
178 changes: 178 additions & 0 deletions ...lm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
name: dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp
model:
path: deepseek-ai/DeepSeek-V4-Pro
container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24
precision: fp4
identity:
model:
repo: deepseek-ai/DeepSeek-V4-Pro
container:
image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24
frameworks:
dynamo: 1.4.0.dev20260807
tensorrt_llm: 1.3.0rc24
dynamo:
install: true
wheel: 1.4.0.dev20260807
request_plane: tcp
health_check:
max_attempts: 1440
interval_seconds: 10
resources:
gpu_type: gb300
gpus_per_node: 4
prefill_nodes: 2
prefill_workers: 1
gpus_per_prefill: 8
decode_nodes: 8
decode_workers: 1
gpus_per_decode: 32
backend:
type: trtllm
prefill_environment:
TLLM_LOG_LEVEL: INFO
TRTLLM_SERVER_DISABLE_GC: '1'
TRTLLM_WORKER_DISABLE_GC: '1'
NCCL_GRAPH_MIXING_SUPPORT: '0'
TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.97'
UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp
DYN_ENGINE_CONV_AFFINITY: '1'
MIMALLOC_PURGE_DELAY: '0'
TRTLLM_ENABLE_PDL: '1'
DYN_DEFAULT_THINKING_MODE: disabled
MIMALLOC_ARENA_RESERVE: '0'
TRTLLM_PINNED_WEIGHT_STAGING: '1'
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
CUDA_SCALE_LAUNCH_QUEUES: 4x
decode_environment:
TLLM_LOG_LEVEL: INFO
TRTLLM_SERVER_DISABLE_GC: '1'
TRTLLM_WORKER_DISABLE_GC: '1'
NCCL_GRAPH_MIXING_SUPPORT: '0'
TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.97'
UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp
DYN_ENGINE_CONV_AFFINITY: '1'
MIMALLOC_PURGE_DELAY: '0'
TRTLLM_ENABLE_PDL: '1'
DYN_DEFAULT_THINKING_MODE: disabled
MIMALLOC_ARENA_RESERVE: '0'
TRTLLM_PINNED_WEIGHT_STAGING: '1'
trtllm_config:
prefill:
attention_dp_config:
kv_cache_routing_conversation_affinity: true
cache_transceiver_config:
backend: NIXL
kv_transfer_timeout_ms: 600000
transceiver_runtime: PYTHON
cuda_graph_config: null
custom_tokenizer: deepseek_v4
disable_overlap_scheduler: false
enable_attention_dp: true
enable_chunked_prefill: true
kv_cache_config:
dtype: fp8
enable_block_reuse: true
event_buffer_max_size: 0
free_gpu_memory_fraction: 0.8
host_cache_size: 193273528320
pool_ratio:
- 0.55
- 0.22
- 0.23
tokens_per_block: 128
block_reuse_config:
policy: per_conversation
max_num_turns: 5
max_batch_size: 256
max_num_tokens: 16384
max_seq_len: 990016
moe_config:
backend: TRTLLM
load_balancer: /infmax-workspace/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/eplb_confs/moe_load_balancer_ctx_ep8_slots384.yaml
moe_expert_parallel_size: 8
pipeline_parallel_size: 1
print_iter_log: true
return_perf_metrics: true
scheduler_config:
capacity_scheduler_policy: MAX_UTILIZATION
speculative_config:
decoding_type: MTP
max_draft_len: 3
tensor_parallel_size: 8
decode:
cache_transceiver_config:
backend: NIXL
kv_transfer_timeout_ms: 600000
transceiver_runtime: PYTHON
cuda_graph_config:
batch_sizes:
- 1
- 2
- 4
enable_padding: true
custom_tokenizer: deepseek_v4
enable_attention_dp: true
enable_lm_head_tp_in_adp: true
kv_cache_config:
avg_seq_len: 200000
dtype: fp8
enable_block_reuse: false
event_buffer_max_size: 0
free_gpu_memory_fraction: 0.7
host_cache_size: 0
tokens_per_block: 128
max_batch_size: 4
max_num_tokens: 16
max_seq_len: 996595
moe_config:
backend: MEGAMOE_DEEPGEMM
use_low_precision_moe_combine: true
load_balancer: /infmax-workspace/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/eplb_confs/moe_load_balancer_gen_ep32_slots384.yaml
moe_expert_parallel_size: 32
num_postprocess_workers: 4
pipeline_parallel_size: 1
print_iter_log: true
return_perf_metrics: true
sparse_attention_config:
algorithm: deepseek_v4
enable_heuristic_topk: true
speculative_config:
decoding_type: MTP
max_draft_len: 3
stream_interval: 20
tensor_parallel_size: 32
publish_events_and_metrics: false
frontend:
type: dynamo
enable_multiple_frontends: false
env:
ETCD_LEASE_TTL: '120'
DYN_ROUTER_QUEUE_THRESHOLD: None
DYN_TOKENIZER_CACHE: '1'
DYN_TOKENIZER_CACHE_BYTES: '8000000000'
DYN_TOKENIZER: fastokens
DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400'
args:
router-mode: kv
no-kv-events: true
active-decode-blocks-threshold: None
active-prefill-tokens-threshold: None
active-prefill-tokens-threshold-frac: None
orchestrator_placement: first_decode
benchmark:
type: custom
use_chat_template: true
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
env:
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
RESULT_DIR: /logs/agentic
PORT: '8000'
IS_MULTINODE: 'true'
AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400'
AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
HF_HUB_CACHE: /hf_hub_cache
WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126
SERVED_MODEL_NAME: DeepSeek-V4-Pro
Loading
Loading