diff --git a/benchmarks/multi_node/agentic/glm5.1_fp8_b200_tilert-disagg.sh b/benchmarks/multi_node/agentic/glm5.1_fp8_b200_tilert-disagg.sh new file mode 100755 index 000000000..6a903ca21 --- /dev/null +++ b/benchmarks/multi_node/agentic/glm5.1_fp8_b200_tilert-disagg.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars \ + CONC_LIST \ + DURATION \ + IMAGE \ + SPEC_DECODING \ + MODEL_PATH \ + PREFILL_NUM_WORKERS \ + PREFILL_TP \ + PREFILL_EP \ + PREFILL_DP_ATTN \ + DECODE_NUM_WORKERS \ + DECODE_TP \ + DECODE_EP \ + DECODE_DP_ATTN \ + PREFILL_NODES \ + DECODE_NODES \ + FRAMEWORK + +require_agentic_kv_offload_none + +export MODEL_NAME=glm5 +export TILERT_MODEL_TYPE=glm-5 + +export DECODE_KV_DTYPE=fp8 +export PREFILL_KV_DTYPE=fp8_ds_mla + +export TILERT_PARSER=none + +exec bash "$(dirname "$0")/../tilert_utils/submit.sh" diff --git a/benchmarks/multi_node/tilert_utils/run_node.sh b/benchmarks/multi_node/tilert_utils/run_node.sh index c6d3f0046..7a1e9cea3 100755 --- a/benchmarks/multi_node/tilert_utils/run_node.sh +++ b/benchmarks/multi_node/tilert_utils/run_node.sh @@ -23,6 +23,18 @@ PREFILL_KV_DTYPE=${PREFILL_KV_DTYPE:-fp8_ds_mla} PREFILL_SPEC=(--speculative-config '{"method":"mtp","num_speculative_tokens":1}') DECODE_MTP=(--with-mtp) +TILERT_IS_AGENTIC=0 +if [[ "${IS_AGENTIC:-0}" == "1" || "${SCENARIO_TYPE:-}" == "agentic-coding" ]]; then + TILERT_IS_AGENTIC=1 +fi + +if [[ "$TILERT_IS_AGENTIC" == "1" ]]; then + TILERT_QUEUE_TIMEOUT=${TILERT_QUEUE_TIMEOUT:-1800} +fi +TILERT_QUEUE_TIMEOUT=${TILERT_QUEUE_TIMEOUT:-0} + +AGENTIC_LOGS_DIR=${AGENTIC_LOGS_DIR:-$RESULT_DIR/LOGS/agentic} + : "${DECODE_HOST:?DECODE_HOST is unset -- submit.sh must export it}" : "${PREFILL_HOST:?PREFILL_HOST is unset -- submit.sh must export it}" : "${TILERT_ROLE:?TILERT_ROLE is unset -- submit.sh must set it to decode or prefill}" @@ -156,8 +168,10 @@ start_decode() { } start_prefill() { + local served=("$MODEL_NAME") + [[ -n "${MODEL:-}" && "$MODEL" != "$MODEL_NAME" ]] && served+=("$MODEL") local cmd=(vllm serve "$MODEL_PATH" - --served-model-name "$MODEL_NAME" --port "$PREFILL_PORT" + --served-model-name "${served[@]}" --port "$PREFILL_PORT" --tensor-parallel-size "$PREFILL_TP" --max-model-len "$MAX_MODEL_LEN" --enforce-eager --trust-remote-code --return-tokens-as-token-ids --gpu-memory-utilization "$GPU_MEM_UTIL" --kv-cache-dtype "$PREFILL_KV_DTYPE" @@ -171,7 +185,8 @@ start_router() { local cmd=(env CUDA_VISIBLE_DEVICES= "${PY:-python}" -m tilert.pd_vllm.pd_router --vllm-url "http://$PREFILL_HOST:$PREFILL_PORT" --decode "$DECODE_HOST:$DECODE_CTRL_PORT:$DECODE_HTTP_PORT" - --port "$ROUTER_PORT" --model-path "$MODEL_PATH" --parser "$TILERT_PARSER") + --port "$ROUTER_PORT" --model-path "$MODEL_PATH" --parser "$TILERT_PARSER" + --queue-timeout "$TILERT_QUEUE_TIMEOUT") log_and_run_bg router "$BENCHMARK_LOGS_DIR/tilert_router.log" "${cmd[@]}" ROUTER_PID=$LAST_BG_PID } @@ -211,16 +226,37 @@ run_bench_and_eval() { --result-filename "$(bench_result_stem "$conc")" --result-dir "$RESULT_DIR" \ || { rc=$?; echo "[bench] WARNING: conc=$conc failed/timed out (rc=$rc)"; } done - if [[ "${RUN_EVAL}" = "true" ]]; then - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="$EVAL_CONC" - else - export EVAL_CONCURRENT_REQUESTS="$(tr ' ' '\n' <<< "$CONC_LIST" | sort -n | tail -1)" - fi - export CONC="$EVAL_CONCURRENT_REQUESTS" - run_eval --framework lm-eval --port "$ROUTER_PORT" - append_lm_eval_summary + run_lm_eval + return $rc +} + +run_lm_eval() { + [[ "${RUN_EVAL}" = "true" ]] || return 0 + if [[ -n "${EVAL_CONC:-}" ]]; then + export EVAL_CONCURRENT_REQUESTS="$EVAL_CONC" + else + export EVAL_CONCURRENT_REQUESTS="$(tr ' ' '\n' <<< "$CONC_LIST" | sort -n | tail -1)" fi + export CONC="$EVAL_CONCURRENT_REQUESTS" + run_eval --framework lm-eval --port "$ROUTER_PORT" + append_lm_eval_summary +} + +run_agentic_replay() { + wait_for_server_ready --port "$ROUTER_PORT" \ + --server-log "$BENCHMARK_LOGS_DIR/tilert_router.log" --server-pid "$ROUTER_PID" + local rc=0 conc conc_result_dir + local result_filename_base="$RESULT_FILENAME" + for conc in $CONC_LIST; do + conc_result_dir="$AGENTIC_LOGS_DIR/conc_${conc}" + mkdir -p "$conc_result_dir" + export CONC="$conc" + export RESULT_FILENAME="${result_filename_base}_conc${conc}" + build_replay_cmd "$conc_result_dir" + run_agentic_replay_and_write_outputs "$conc_result_dir" \ + || { rc=$?; echo "[agentic] WARNING: conc=$conc failed/timed out (rc=$rc)"; } + done + export RESULT_FILENAME="$result_filename_base" return $rc } @@ -247,13 +283,21 @@ case "$TILERT_ROLE" in prefill) rdma_preflight || exit 1 rm -f "$DONE_SENTINEL" + if [[ "$TILERT_IS_AGENTIC" == "1" ]]; then + resolve_trace_source + install_agentic_deps + fi wait_for_tcp "$DECODE_HOST" "$DECODE_CTRL_PORT" "$DECODE_WAIT" \ || echo "[prefill] WARNING: timed out waiting for the decode ctrl port ($DECODE_HOST:$DECODE_CTRL_PORT), starting anyway" start_prefill wait_for_tcp "$PREFILL_HOST" "$PREFILL_PORT" "${PREFILL_WAIT:-3600}" \ || echo "[prefill] WARNING: timed out waiting for the vLLM port ($PREFILL_HOST:$PREFILL_PORT), continuing (see $BENCHMARK_LOGS_DIR/tilert_prefill.log)" start_router - run_bench_and_eval; BENCH_RC=$? + if [[ "$TILERT_IS_AGENTIC" == "1" ]]; then + run_agentic_replay; BENCH_RC=$? + else + run_bench_and_eval; BENCH_RC=$? + fi touch "$DONE_SENTINEL" kill "$ROUTER_PID" "$PREFILL_PID" 2>/dev/null || true exit $BENCH_RC diff --git a/benchmarks/multi_node/tilert_utils/setup_deps.sh b/benchmarks/multi_node/tilert_utils/setup_deps.sh index 047087af2..645e81af8 100644 --- a/benchmarks/multi_node/tilert_utils/setup_deps.sh +++ b/benchmarks/multi_node/tilert_utils/setup_deps.sh @@ -1,6 +1,6 @@ #!/bin/bash -TILERT_VERSION="${TILERT_VERSION:-0.1.5.post2}" +TILERT_VERSION="${TILERT_VERSION:-0.1.5.post3}" TILERT_PIP_INDEX_URL="${TILERT_PIP_INDEX_URL:-}" TILERT_HTTP_DEPS="${TILERT_HTTP_DEPS:-fastapi uvicorn httpx}" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c8d55ad5d..a7845343a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8834,7 +8834,7 @@ glm5.1-fp8-b200-tilert: runner: b200-multinode precision: fp8 framework: tilert - router: { name: tilert-pd-router, version: "0.1.5.post2" } + router: { name: tilert-pd-router, version: "0.1.5.post3" } multinode: true disagg: true kv-p2p-transfer: nixl @@ -8880,3 +8880,35 @@ glm5.1-fp8-b200-tilert: dp-attn: false additional-settings: - "DECODE_NODES=1" + +glm5.1-fp8-b200-tilert-agentic: + image: ghcr.io/tile-ai/tilert:0.1.5 + model: zai-org/GLM-5.1-FP8 + model-prefix: glm5.1 + runner: cluster:b200-dgxc-slurm + precision: fp8 + framework: tilert + router: { name: tilert-pd-router, version: "0.1.5.post3" } + multinode: true + disagg: true + kv-p2p-transfer: nixl + scenarios: + agentic-coding: + - search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_IMAGE=vllm/vllm-openai:v0.26.0" + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" diff --git a/configs/runners.yaml b/configs/runners.yaml index 6f0b35e9c..7b8cda49c 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -80,6 +80,10 @@ labels: - b200-dgxc-slurm_7 - b200-dgxc-slurm_8 - b200-dgxc-slurm_9 + cluster:b200-dgxc-slurm: + - b200-dgxc-slurm_7 + - b200-dgxc-slurm_8 + - b200-dgxc-slurm_9 b200-new: - b200-nscale-slurm_0 - b200-nscale-slurm_1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 73648d9b9..76db2f4c5 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6076,3 +6076,14 @@ - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM AgentX mirroring the GB300 PR #2571 MTP tuning, with every GB300 4-GPU worker sized to 8 GPUs on GB200." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2636 +- config-keys: + - glm5.1-fp8-b200-tilert-agentic + scenario-type: + - agentic-coding + description: + - "Add the agentic-coding scenario for GLM-5.1 FP8 B200 TileRT PD-disaggregation, reusing the fixed-seq-len topology (1 prefill node TP8 + 1 decode node TP8, NIXL KV transfer, MTP) with an AIPerf trace replay in place of benchmark_serving.py" + - "Separate config key rather than a second scenario on glm5.1-fp8-b200-tilert: agentic master configs must declare an exact cluster: runner, so this entry adds cluster:b200-dgxc-slurm to configs/runners.yaml over the same three b200-dgxc-slurm nodes the b200-multinode label already uses" + - "TileRT decode serves one sequence at a time, so conc-list is the single point [1]; the router waits for a free decode node (--queue-timeout) instead of answering 429 when a session fans out into concurrent sub-conversations" + - "kv-offloading: none -- TileRT keeps all KV state on the GPU and exposes no DRAM offload tier" + - "GLM-5.1 has 202752 native positions, so AIPerf's --max-context-length drops the corpus traces whose peak context exceeds it: 175 of the 393 traces in cc-traces-weka-062126-256k remain eligible" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2645 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 8c2b357b0..498ad7d1c 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -138,7 +138,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then export UCX_NET_DEVICES="${UCX_NET_DEVICES:-mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1}" export UCX_MEMTYPE_CACHE="${UCX_MEMTYPE_CACHE:-n}" export UCX_MEMTYPE_REG_WHOLE="${UCX_MEMTYPE_REG_WHOLE:-n}" - TILERT_DISAGG="$GITHUB_WORKSPACE/benchmarks/multi_node/${EXP_NAME%%_*}_${PRECISION}_b200_${FRAMEWORK}-disagg.sh" + TILERT_SUBDIR="multi_node" + [[ "${SCENARIO_SUBDIR}" == "agentic/" ]] && TILERT_SUBDIR="multi_node/agentic" + TILERT_DISAGG="$GITHUB_WORKSPACE/benchmarks/${TILERT_SUBDIR}/${EXP_NAME%%_*}_${PRECISION}_b200_${FRAMEWORK}-disagg.sh" [[ -f "$TILERT_DISAGG" ]] || { echo "tilert disagg script not found: $TILERT_DISAGG"; exit 1; } exec bash "$TILERT_DISAGG" exit 1