diff --git a/benchmarks/multi_node/glm5.1_fp8_b200_tilert-disagg.sh b/benchmarks/multi_node/glm5.1_fp8_b200_tilert-disagg.sh deleted file mode 100755 index b7fffba9fc..0000000000 --- a/benchmarks/multi_node/glm5.1_fp8_b200_tilert-disagg.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -export MODEL_NAME=glm5 -export TILERT_MODEL_TYPE=glm-5 -export MAX_MODEL_LEN="${MAX_MODEL_LEN:-202752}" - -export DECODE_KV_DTYPE=fp8 -export PREFILL_KV_DTYPE=fp8_ds_mla - -export TILERT_PARSER=none - -exec bash "$(dirname "$0")/tilert_utils/submit.sh" diff --git a/benchmarks/multi_node/srt-slurm-recipes/tilert/glm5.1/b200-fp8/fixed-seq-len/disagg-1p1d-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/tilert/glm5.1/b200-fp8/fixed-seq-len/disagg-1p1d-tp8-mtp.yaml new file mode 100644 index 0000000000..bee0916642 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/tilert/glm5.1/b200-fp8/fixed-seq-len/disagg-1p1d-tp8-mtp.yaml @@ -0,0 +1,113 @@ +name: "b200-glm51-tilert-fixed-disagg-1p1d-tp8-mtp" + +model: + path: "hf:zai-org/GLM-5.1-FP8" + container: "tilert-decode" + precision: "fp8" + +identity: + model: + repo: "zai-org/GLM-5.1-FP8" + container: + image: "ghcr.io/tile-ai/tilert:0.1.5" + frameworks: + tilert: "0.1.5.post3" + vllm: "0.26.0" + nixl: "1.3.1" + +dynamo: + install: false + +setup_script: tilert_setup.sh + +slurm: + time_limit: "04:00:00" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: tilert-router + enable_multiple_frontends: false + env: + TILERT_ROLE: router + args: + parser: none + queue-timeout: 0 + +backend: + type: tilert + prefill_container: "tilert-prefill" + model_profile: "glm5" + weight_model_type: "glm-5" + weights_dir: "/data/home/sa-shared/gharunners/tilert-cache/glm5.1-fp8-8shard" + max_seq_len: 202752 + kv_cache_dtype: "fp8" + prefill_kv_cache_dtype: "fp8_ds_mla" + transport: nixl + with_mtp: true + speculative_tokens: 1 + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + decode_environment: + <<: *common_environment + tilert_config: + prefill: + enforce-eager: true + trust-remote-code: true + gpu-memory-utilization: 0.75 + +srun_options: + container-writable: "" + container-remap-root: "" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_dir="/logs/${ISL}isl${OSL}osl" + mkdir -p "$result_dir" + for concurrency in $CONC_LIST; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then + num_prompts=16 + fi + python /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --dataset-name random \ + --model zai-org/GLM-5.1-FP8 \ + --tokenizer zai-org/GLM-5.1-FP8 \ + --random-input-len "$ISL" \ + --random-output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$num_prompts" \ + --max-concurrency "$concurrency" \ + --ignore-eos \ + --use-chat-template \ + --trust-remote-code \ + --save-result \ + --result-dir "$result_dir" \ + --result-filename "results_concurrency_${concurrency}_gpus_16_ctx_8_gen_8.json" + done + env: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 78fa30f657..366aa53be1 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9832,6 +9832,7 @@ glm5.1-fp8-b200-tilert: additional-settings: - "PREFILL_IMAGE=vllm/vllm-openai:v0.26.0" - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/tilert/glm5.1/b200-fp8/fixed-seq-len/disagg-1p1d-tp8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -9852,6 +9853,7 @@ glm5.1-fp8-b200-tilert: additional-settings: - "PREFILL_IMAGE=vllm/vllm-openai:v0.26.0" - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/tilert/glm5.1/b200-fp8/fixed-seq-len/disagg-1p1d-tp8-mtp.yaml" decode: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9fcda3df36..6c2044e331 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6547,3 +6547,12 @@ - "Filter AgentX traces at the same 202,752-token context limit used by both TileRT roles so oversized Weka trajectories are excluded before replay." - "Pin SemiAnalysisAI/srt-slurm PR #10 commit d1e6c97b3baf3e87103b6d83189544c3c7d61c38, stacked on the AMD/native-router PR #7 and base runtime PR #1, including explicit native HTTP dependencies, GLM-5.1-compatible Transformers v5 router tokenization, incomplete-snapshot recovery, backend-declared conversion GPU resources, pre-container NVIDIA driver-hook activation, and lossless Slurm container-environment exports." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2750 + +- config-keys: + - glm5.1-fp8-b200-tilert + description: + - "Port the B200 GLM-5.1 FP8 TileRT fixed-sequence MTP 1P1D configurations from the manual multi-node launcher to native srt-slurm orchestration." + - "Preserve the existing 1k/1k and 8k/1k concurrency-1 search points while using the native vLLM-prefill, TileRT-decode, and TileRT-router topology." + - "Run the existing InferenceX benchmark_serving.py directly through srt-slurm's custom benchmark interface instead of maintaining a downstream benchmark implementation." + - "Remove the legacy fixed-sequence TileRT launcher entrypoint and pin SemiAnalysisAI/srt-slurm PR #10 commit d1e6c97b3baf3e87103b6d83189544c3c7d61c38, including explicit native HTTP dependencies, GLM-5.1-compatible Transformers v5 router tokenization, incomplete-snapshot recovery, GPU-scoped TileRT preparation, NVIDIA Enroot/Pyxis driver-hook activation, and lossless Slurm container-environment exports." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2755 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index 1e613d7ed1..33d98eeb0f 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -108,7 +108,7 @@ if [[ "$USES_DCGM_POWER" == "1" ]]; then mkdir -p recipes/vllm/kimi-k2.6 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6" recipes/vllm/kimi-k2.6 fi -elif [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "glm5.1" && $FRAMEWORK == "tilert" ]]; then +elif [[ $MODEL_PREFIX == "glm5.1" && $FRAMEWORK == "tilert" ]]; then git clone "$TILERT_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 git checkout "$TILERT_SRT_SLURM_PIN" || exit 1 @@ -116,9 +116,9 @@ elif [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "glm5.1" && $FRAMEWORK == "tile echo "Error: srt-slurm HEAD does not match TILERT_SRT_SLURM_PIN=$TILERT_SRT_SLURM_PIN" >&2 exit 1 } - mkdir -p recipes/tilert/glm5.1/b200-fp8/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/tilert/glm5.1/b200-fp8/agentic" \ - recipes/tilert/glm5.1/b200-fp8/agentic || exit 1 + mkdir -p recipes/tilert/glm5.1/b200-fp8 || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/tilert/glm5.1/b200-fp8" \ + recipes/tilert/glm5.1/b200-fp8 || exit 1 elif [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then # Pin the tested renderer so branch movement cannot change generated rank # commands between sweep points. @@ -180,7 +180,6 @@ NGINX_SQUASH_FILE="$SQUASH_DIR/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh enroot_uri_for_image() { local image_ref="$1" local first_component="${image_ref%%/*}" - if [[ "$image_ref" == */* && ( "$first_component" == *.* || "$first_component" == *:* || @@ -262,10 +261,18 @@ if [[ "$IS_AGENTIC" == "1" ]]; then fi if [[ $FRAMEWORK == "tilert" ]]; then TILERT_WEIGHTS_HOST_PATH="/data/home/sa-shared/gharunners/tilert-cache" - mkdir -p "$TILERT_WEIGHTS_HOST_PATH" - DEFAULT_MOUNTS_BLOCK="${DEFAULT_MOUNTS_BLOCK} + mkdir -p "$HF_HUB_CACHE_HOST_PATH" "$TILERT_WEIGHTS_HOST_PATH" + chmod 777 "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true + if [[ -n "$DEFAULT_MOUNTS_BLOCK" ]]; then + DEFAULT_MOUNTS_BLOCK="${DEFAULT_MOUNTS_BLOCK} ${GITHUB_WORKSPACE}: /infmax-workspace ${TILERT_WEIGHTS_HOST_PATH}: ${TILERT_WEIGHTS_HOST_PATH}" + else + DEFAULT_MOUNTS_BLOCK="default_mounts: + ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache + ${GITHUB_WORKSPACE}: /infmax-workspace + ${TILERT_WEIGHTS_HOST_PATH}: ${TILERT_WEIGHTS_HOST_PATH}" + fi fi SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}"