diff --git a/benchmarks/multi_node/agentic/glm5.1_fp8_b200_tilert-disagg.sh b/benchmarks/multi_node/agentic/glm5.1_fp8_b200_tilert-disagg.sh new file mode 100755 index 0000000000..6a903ca218 --- /dev/null +++ b/benchmarks/multi_node/agentic/glm5.1_fp8_b200_tilert-disagg.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars \ + CONC_LIST \ + DURATION \ + IMAGE \ + SPEC_DECODING \ + MODEL_PATH \ + PREFILL_NUM_WORKERS \ + PREFILL_TP \ + PREFILL_EP \ + PREFILL_DP_ATTN \ + DECODE_NUM_WORKERS \ + DECODE_TP \ + DECODE_EP \ + DECODE_DP_ATTN \ + PREFILL_NODES \ + DECODE_NODES \ + FRAMEWORK + +require_agentic_kv_offload_none + +export MODEL_NAME=glm5 +export TILERT_MODEL_TYPE=glm-5 + +export DECODE_KV_DTYPE=fp8 +export PREFILL_KV_DTYPE=fp8_ds_mla + +export TILERT_PARSER=none + +exec bash "$(dirname "$0")/../tilert_utils/submit.sh" diff --git a/benchmarks/multi_node/tilert_utils/build_queue_wheel.py b/benchmarks/multi_node/tilert_utils/build_queue_wheel.py new file mode 100644 index 0000000000..5f05680345 --- /dev/null +++ b/benchmarks/multi_node/tilert_utils/build_queue_wheel.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Build InferenceX's queueing backport from the official TileRT post2 wheel.""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +import subprocess +import sys +import tempfile +import urllib.request +import zipfile +from pathlib import Path + + +UPSTREAM_VERSION = "0.1.5.post2" +PATCHED_VERSION = "0.1.5.post2+inferencex.1" +UPSTREAM_WHEEL = "tilert-0.1.5.post2-cp312-cp312-manylinux_2_28_x86_64.whl" +UPSTREAM_URL = ( + "https://github.com/tile-ai/TileRT/releases/download/" + f"v{UPSTREAM_VERSION}/{UPSTREAM_WHEEL}" +) +UPSTREAM_SHA256 = "e65b876ccfc1a419b0047a6d6b395f619ea35c15194ad1892f171c78476fe407" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def download_upstream(destination: Path) -> None: + with urllib.request.urlopen(UPSTREAM_URL) as response: # noqa: S310 (fixed URL) + with destination.open("wb") as output: + shutil.copyfileobj(response, output) + actual = sha256(destination) + if actual != UPSTREAM_SHA256: + raise RuntimeError( + f"upstream wheel SHA256 mismatch: expected {UPSTREAM_SHA256}, got {actual}" + ) + + +def update_metadata(unpacked: Path) -> None: + old_dist_info = unpacked / f"tilert-{UPSTREAM_VERSION}.dist-info" + new_dist_info = unpacked / f"tilert-{PATCHED_VERSION}.dist-info" + old_dist_info.rename(new_dist_info) + metadata = new_dist_info / "METADATA" + text = metadata.read_text() + old_version = f"Version: {UPSTREAM_VERSION}\n" + if text.count(old_version) != 1: + raise RuntimeError("expected exactly one upstream Version field in METADATA") + metadata.write_text(text.replace(old_version, f"Version: {PATCHED_VERSION}\n")) + + +def build(output_dir: Path) -> Path: + patch = Path(__file__).with_name("patches") / "tilert-0.1.5.post2-queue.patch" + output_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="tilert-queue-wheel-") as temporary: + work = Path(temporary) + upstream = work / UPSTREAM_WHEEL + unpacked = work / "unpacked" + download_upstream(upstream) + with zipfile.ZipFile(upstream) as archive: + archive.extractall(unpacked) + subprocess.run( + ["patch", "-p1", "--batch", "--forward", "-i", str(patch)], + cwd=unpacked, + check=True, + ) + update_metadata(unpacked) + subprocess.run( + [sys.executable, "-m", "wheel", "pack", "--dest-dir", str(output_dir), "."], + cwd=unpacked, + check=True, + ) + wheels = list(output_dir.glob("tilert-0.1.5.post2+inferencex.1-*.whl")) + if len(wheels) != 1: + raise RuntimeError(f"expected one patched wheel, found {len(wheels)}") + return wheels[0] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("output_dir", type=Path) + args = parser.parse_args() + wheel = build(args.output_dir.resolve()) + print(f"{sha256(wheel)} {wheel}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/multi_node/tilert_utils/patches/tilert-0.1.5.post2-queue.patch b/benchmarks/multi_node/tilert_utils/patches/tilert-0.1.5.post2-queue.patch new file mode 100644 index 0000000000..29f3316561 --- /dev/null +++ b/benchmarks/multi_node/tilert_utils/patches/tilert-0.1.5.post2-queue.patch @@ -0,0 +1,133 @@ +--- a/tilert/pd_vllm/pd_router.py ++++ b/tilert/pd_vllm/pd_router.py +@@ -4,7 +4,8 @@ + non-streaming. + + Flow per request (phase-1 hybrid, see design doc): +- 1. pick a free decode node (in-memory busy tracking; all busy -> 429) ++ 1. wait up to --queue-timeout for a free decode node (in-memory busy ++ tracking; timeout -> 429) + 2. forward to vLLM with max_tokens=1 + logprobs and inject + kv_transfer_params {tilert_host, tilert_ctrl_port} — the connector + claims the request and RDMA-sends state to the decode node +@@ -56,19 +57,27 @@ + class Pool: + def __init__(self, nodes: list[DecodeNode]): + self.nodes = nodes +- self._lock = threading.Lock() ++ self._available = threading.Condition() + +- def acquire(self) -> DecodeNode | None: +- with self._lock: +- for n in self.nodes: +- if not n.busy: +- n.busy = True +- return n +- return None ++ def acquire(self, timeout: float = 0.0) -> DecodeNode | None: ++ deadline = time.monotonic() + timeout ++ with self._available: ++ while True: ++ for n in self.nodes: ++ if not n.busy: ++ n.busy = True ++ return n ++ if timeout <= 0: ++ return None ++ remaining = deadline - time.monotonic() ++ if remaining <= 0: ++ return None ++ self._available.wait(timeout=remaining) + + def release(self, node: DecodeNode) -> None: +- with self._lock: ++ with self._available: + node.busy = False ++ self._available.notify() + + + def first_token_from_logprobs(resp: dict, is_chat: bool) -> int: +@@ -123,11 +134,19 @@ + class RouterCtx: + """Immutable per-process context (tokenizer, parser factory, config).""" + +- def __init__(self, vllm_url: str, pool: Pool, tokenizer, parser_name: str): ++ def __init__( ++ self, ++ vllm_url: str, ++ pool: Pool, ++ tokenizer, ++ parser_name: str, ++ queue_timeout: float = 0.0, ++ ): + self.vllm_url = vllm_url + self.pool = pool + self.tokenizer = tokenizer + self.parser_name = parser_name ++ self.queue_timeout = queue_timeout + self._parsers = {} + if parser_name != "none": + if tokenizer is None: +@@ -178,9 +197,12 @@ + # ── non-streaming ──────────────────────────────────────────────────── + def _handle(path: str, body: dict): + is_chat = path.endswith("chat/completions") +- node = pool.acquire() ++ node = pool.acquire(ctx.queue_timeout) + if node is None: +- return JSONResponse({"error": "all decode nodes busy"}, status_code=429) ++ return JSONResponse( ++ {"error": f"all decode nodes busy after {ctx.queue_timeout:g}s"}, ++ status_code=429, ++ ) + t0 = time.time() + try: + prefill = _prefill(path, body, node) +@@ -257,9 +280,12 @@ + async def _handle_stream(path: str, body: dict, request: Request): + from starlette.concurrency import run_in_threadpool + +- node = pool.acquire() ++ node = await run_in_threadpool(pool.acquire, ctx.queue_timeout) + if node is None: +- return JSONResponse({"error": "all decode nodes busy"}, status_code=429) ++ return JSONResponse( ++ {"error": f"all decode nodes busy after {ctx.queue_timeout:g}s"}, ++ status_code=429, ++ ) + + try: + prefill = await run_in_threadpool(_prefill, path, body, node) +@@ -445,7 +472,15 @@ + default="glm47", + help="output parser (reasoning + tool calls)", + ) ++ ap.add_argument( ++ "--queue-timeout", ++ type=float, ++ default=0.0, ++ help="seconds to wait for a free decode node before returning HTTP 429", ++ ) + args = ap.parse_args() ++ if args.queue_timeout < 0: ++ ap.error("--queue-timeout must be non-negative") + + nodes = [] + for spec in args.decode: +@@ -460,14 +495,15 @@ + args.model_path, trust_remote_code=True + ) # nosec B615 + +- ctx = RouterCtx(args.vllm_url, Pool(nodes), tokenizer, args.parser) ++ ctx = RouterCtx(args.vllm_url, Pool(nodes), tokenizer, args.parser, args.queue_timeout) + app = build_app(ctx) + logger.info( +- "router on :%d -> vllm=%s, %d decode node(s), parser=%s", ++ "router on :%d -> vllm=%s, %d decode node(s), parser=%s, queue_timeout=%gs", + args.port, + args.vllm_url, + len(nodes), + args.parser, ++ args.queue_timeout, + ) + uvicorn.run(app, host=args.host, port=args.port, log_level="warning") diff --git a/benchmarks/multi_node/tilert_utils/run_node.sh b/benchmarks/multi_node/tilert_utils/run_node.sh index c6d3f0046c..7a1e9cea3e 100755 --- a/benchmarks/multi_node/tilert_utils/run_node.sh +++ b/benchmarks/multi_node/tilert_utils/run_node.sh @@ -23,6 +23,18 @@ PREFILL_KV_DTYPE=${PREFILL_KV_DTYPE:-fp8_ds_mla} PREFILL_SPEC=(--speculative-config '{"method":"mtp","num_speculative_tokens":1}') DECODE_MTP=(--with-mtp) +TILERT_IS_AGENTIC=0 +if [[ "${IS_AGENTIC:-0}" == "1" || "${SCENARIO_TYPE:-}" == "agentic-coding" ]]; then + TILERT_IS_AGENTIC=1 +fi + +if [[ "$TILERT_IS_AGENTIC" == "1" ]]; then + TILERT_QUEUE_TIMEOUT=${TILERT_QUEUE_TIMEOUT:-1800} +fi +TILERT_QUEUE_TIMEOUT=${TILERT_QUEUE_TIMEOUT:-0} + +AGENTIC_LOGS_DIR=${AGENTIC_LOGS_DIR:-$RESULT_DIR/LOGS/agentic} + : "${DECODE_HOST:?DECODE_HOST is unset -- submit.sh must export it}" : "${PREFILL_HOST:?PREFILL_HOST is unset -- submit.sh must export it}" : "${TILERT_ROLE:?TILERT_ROLE is unset -- submit.sh must set it to decode or prefill}" @@ -156,8 +168,10 @@ start_decode() { } start_prefill() { + local served=("$MODEL_NAME") + [[ -n "${MODEL:-}" && "$MODEL" != "$MODEL_NAME" ]] && served+=("$MODEL") local cmd=(vllm serve "$MODEL_PATH" - --served-model-name "$MODEL_NAME" --port "$PREFILL_PORT" + --served-model-name "${served[@]}" --port "$PREFILL_PORT" --tensor-parallel-size "$PREFILL_TP" --max-model-len "$MAX_MODEL_LEN" --enforce-eager --trust-remote-code --return-tokens-as-token-ids --gpu-memory-utilization "$GPU_MEM_UTIL" --kv-cache-dtype "$PREFILL_KV_DTYPE" @@ -171,7 +185,8 @@ start_router() { local cmd=(env CUDA_VISIBLE_DEVICES= "${PY:-python}" -m tilert.pd_vllm.pd_router --vllm-url "http://$PREFILL_HOST:$PREFILL_PORT" --decode "$DECODE_HOST:$DECODE_CTRL_PORT:$DECODE_HTTP_PORT" - --port "$ROUTER_PORT" --model-path "$MODEL_PATH" --parser "$TILERT_PARSER") + --port "$ROUTER_PORT" --model-path "$MODEL_PATH" --parser "$TILERT_PARSER" + --queue-timeout "$TILERT_QUEUE_TIMEOUT") log_and_run_bg router "$BENCHMARK_LOGS_DIR/tilert_router.log" "${cmd[@]}" ROUTER_PID=$LAST_BG_PID } @@ -211,16 +226,37 @@ run_bench_and_eval() { --result-filename "$(bench_result_stem "$conc")" --result-dir "$RESULT_DIR" \ || { rc=$?; echo "[bench] WARNING: conc=$conc failed/timed out (rc=$rc)"; } done - if [[ "${RUN_EVAL}" = "true" ]]; then - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="$EVAL_CONC" - else - export EVAL_CONCURRENT_REQUESTS="$(tr ' ' '\n' <<< "$CONC_LIST" | sort -n | tail -1)" - fi - export CONC="$EVAL_CONCURRENT_REQUESTS" - run_eval --framework lm-eval --port "$ROUTER_PORT" - append_lm_eval_summary + run_lm_eval + return $rc +} + +run_lm_eval() { + [[ "${RUN_EVAL}" = "true" ]] || return 0 + if [[ -n "${EVAL_CONC:-}" ]]; then + export EVAL_CONCURRENT_REQUESTS="$EVAL_CONC" + else + export EVAL_CONCURRENT_REQUESTS="$(tr ' ' '\n' <<< "$CONC_LIST" | sort -n | tail -1)" fi + export CONC="$EVAL_CONCURRENT_REQUESTS" + run_eval --framework lm-eval --port "$ROUTER_PORT" + append_lm_eval_summary +} + +run_agentic_replay() { + wait_for_server_ready --port "$ROUTER_PORT" \ + --server-log "$BENCHMARK_LOGS_DIR/tilert_router.log" --server-pid "$ROUTER_PID" + local rc=0 conc conc_result_dir + local result_filename_base="$RESULT_FILENAME" + for conc in $CONC_LIST; do + conc_result_dir="$AGENTIC_LOGS_DIR/conc_${conc}" + mkdir -p "$conc_result_dir" + export CONC="$conc" + export RESULT_FILENAME="${result_filename_base}_conc${conc}" + build_replay_cmd "$conc_result_dir" + run_agentic_replay_and_write_outputs "$conc_result_dir" \ + || { rc=$?; echo "[agentic] WARNING: conc=$conc failed/timed out (rc=$rc)"; } + done + export RESULT_FILENAME="$result_filename_base" return $rc } @@ -247,13 +283,21 @@ case "$TILERT_ROLE" in prefill) rdma_preflight || exit 1 rm -f "$DONE_SENTINEL" + if [[ "$TILERT_IS_AGENTIC" == "1" ]]; then + resolve_trace_source + install_agentic_deps + fi wait_for_tcp "$DECODE_HOST" "$DECODE_CTRL_PORT" "$DECODE_WAIT" \ || echo "[prefill] WARNING: timed out waiting for the decode ctrl port ($DECODE_HOST:$DECODE_CTRL_PORT), starting anyway" start_prefill wait_for_tcp "$PREFILL_HOST" "$PREFILL_PORT" "${PREFILL_WAIT:-3600}" \ || echo "[prefill] WARNING: timed out waiting for the vLLM port ($PREFILL_HOST:$PREFILL_PORT), continuing (see $BENCHMARK_LOGS_DIR/tilert_prefill.log)" start_router - run_bench_and_eval; BENCH_RC=$? + if [[ "$TILERT_IS_AGENTIC" == "1" ]]; then + run_agentic_replay; BENCH_RC=$? + else + run_bench_and_eval; BENCH_RC=$? + fi touch "$DONE_SENTINEL" kill "$ROUTER_PID" "$PREFILL_PID" 2>/dev/null || true exit $BENCH_RC diff --git a/benchmarks/multi_node/tilert_utils/setup_deps.sh b/benchmarks/multi_node/tilert_utils/setup_deps.sh index 047087af2d..796fb1b4f9 100644 --- a/benchmarks/multi_node/tilert_utils/setup_deps.sh +++ b/benchmarks/multi_node/tilert_utils/setup_deps.sh @@ -1,7 +1,15 @@ #!/bin/bash -TILERT_VERSION="${TILERT_VERSION:-0.1.5.post2}" TILERT_PIP_INDEX_URL="${TILERT_PIP_INDEX_URL:-}" +if [[ "${IS_AGENTIC:-0}" == "1" || "${SCENARIO_TYPE:-}" == "agentic-coding" ]]; then + TILERT_VERSION="${TILERT_VERSION:-0.1.5.post2+inferencex.1}" + TILERT_WHEEL_URL="${TILERT_WHEEL_URL:-https://github.com/SemiAnalysisAI/InferenceX/releases/download/tilert-v0.1.5.post2-inferencex.1/tilert-0.1.5.post2+inferencex.1-cp312-cp312-manylinux_2_28_x86_64.whl}" + TILERT_WHEEL_SHA256="${TILERT_WHEEL_SHA256:-4c0a4330b96d3cb96536d761197e978bde18030bb1e3d14f2a39472053ab4b7b}" +else + TILERT_VERSION="${TILERT_VERSION:-0.1.5.post2}" + TILERT_WHEEL_URL="${TILERT_WHEEL_URL:-}" + TILERT_WHEEL_SHA256="${TILERT_WHEEL_SHA256:-}" +fi TILERT_HTTP_DEPS="${TILERT_HTTP_DEPS:-fastapi uvicorn httpx}" TILERT_NIXL_VERSION="${TILERT_NIXL_VERSION:-1.3.1}" @@ -42,6 +50,18 @@ _pip_args() { printf '%s\n' "${a[@]}" } +_tilert_install_spec() { + if [[ -n "$TILERT_WHEEL_URL" ]]; then + [[ -n "$TILERT_WHEEL_SHA256" ]] || { + echo "[SETUP] ERROR: TILERT_WHEEL_SHA256 is required with TILERT_WHEEL_URL" >&2 + return 1 + } + printf '%s#sha256=%s\n' "$TILERT_WHEEL_URL" "$TILERT_WHEEL_SHA256" + else + printf 'tilert==%s\n' "$TILERT_VERSION" + fi +} + install_tilert_decode() { mapfile -t _pa < <(_pip_args) local have; have="$(_installed_version tilert)" @@ -49,9 +69,13 @@ install_tilert_decode() { echo "[SETUP] tilert $have already installed, skipping" else [[ -n "$have" ]] && echo "[SETUP] tilert $have installed, switching to pinned $TILERT_VERSION" - echo "[SETUP] installing tilert==$TILERT_VERSION (official PyPI release wheel)" - "$PY" -m pip install "${_pa[@]}" "tilert==$TILERT_VERSION" || { - echo "[SETUP] ERROR: failed to install tilert==$TILERT_VERSION"; exit 1; } + if [[ -n "$TILERT_WHEEL_URL" ]]; then + echo "[SETUP] installing tilert==$TILERT_VERSION (InferenceX queueing backport; SHA256 pinned)" + else + echo "[SETUP] installing tilert==$TILERT_VERSION (official package-index wheel)" + fi + "$PY" -m pip install "${_pa[@]}" "$(_tilert_install_spec)" || { + echo "[SETUP] ERROR: failed to install tilert==$TILERT_VERSION from $TILERT_WHEEL_URL"; exit 1; } have="$(_installed_version tilert)" [[ "$have" == "$TILERT_VERSION" ]] || { echo "[SETUP] ERROR: still not $TILERT_VERSION after install (actual: ${have:-not installed})"; exit 1; } @@ -104,8 +128,8 @@ install_tilert_prefill() { else echo "[SETUP] installing tilert==$TILERT_VERSION --no-deps (connector plugin only; leaves transformers untouched)" mapfile -t _pa < <(_pip_args) - "$PY" -m pip install "${_pa[@]}" --no-deps "tilert==$TILERT_VERSION" || { - echo "[SETUP] ERROR: failed to install tilert==$TILERT_VERSION (--no-deps)"; exit 1; } + "$PY" -m pip install "${_pa[@]}" --no-deps "$(_tilert_install_spec)" || { + echo "[SETUP] ERROR: failed to install tilert==$TILERT_VERSION from $TILERT_WHEEL_URL (--no-deps)"; exit 1; } have="$(_installed_version tilert)" [[ "$have" == "$TILERT_VERSION" ]] || { echo "[SETUP] ERROR: still not $TILERT_VERSION after install (actual: ${have:-not installed})"; exit 1; } diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a995310234..76a3f6002e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9041,3 +9041,35 @@ glm5.1-fp8-b200-tilert: dp-attn: false additional-settings: - "DECODE_NODES=1" + +glm5.1-fp8-b200-tilert-agentic: + image: ghcr.io/tile-ai/tilert:0.1.5 + model: zai-org/GLM-5.1-FP8 + model-prefix: glm5.1 + runner: cluster:b200-dgxc + precision: fp8 + framework: tilert + router: { name: tilert-pd-router, version: "0.1.5.post2+inferencex.1" } + multinode: true + disagg: true + kv-p2p-transfer: nixl + scenarios: + agentic-coding: + - search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_IMAGE=vllm/vllm-openai:v0.26.0" + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index a3afb3076a..120fff7fa4 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -175,6 +175,8 @@ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L1824-L1848)). +Keep non-index engine or router wheels reproducible and immutable: check in the source patch and builder beside the launcher, verify the upstream wheel's digest before patching, assign an explicit local version, and install the published artifact through an exact URL with a SHA256 fragment. A local backport must not use an unreleased upstream version number. + Targeted canonical run (configured duration and warmup, with fast and duration overrides omitted): ```bash diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 669ed290c3..ecc7f06212 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -175,6 +175,8 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[Fast replay 设置](../benchmarks/benchmark_lib.sh#L1824-L1848)）。 +对于未发布到 package index 的 engine 或 router wheel，必须保证构建可复现且 artifact 不可变：在 launcher 旁签入源码 patch 与构建器，打 patch 前校验上游 wheel 的 digest，分配明确的 local version，并通过带 SHA256 fragment 的精确 URL 安装已发布 artifact。本地 backport 不得冒用尚未发布的上游版本号。 + 目标 canonical 运行（使用配置的 duration 和 warmup；不要加 fast 或 duration override）： ```bash diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6b0d955301..dbfee78c99 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6171,3 +6171,24 @@ - "Use AITER INT4 quick-reduce, ptpc_fp8 online quantization excluding embeddings, lm_head, gates, and experts, an FP8 KV cache, and three-token MTP with golden synthetic acceptance length 2.99 for benchmark runs." - "Set max-num-seqs to twice the concurrency, select CUDA graph capture sizes by concurrency, and cap max-num-batched-tokens at 16384." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2576 + +- config-keys: + - glm5.1-fp8-b200-tilert-agentic + scenario-type: + - agentic-coding + description: + - "Add the agentic-coding scenario for GLM-5.1 FP8 B200 TileRT PD-disaggregation, reusing the fixed-seq-len topology (1 prefill node TP8 + 1 decode node TP8, NIXL KV transfer, MTP) with an AIPerf trace replay in place of benchmark_serving.py" + - "Separate config key rather than a second scenario on glm5.1-fp8-b200-tilert: agentic master configs must declare an exact cluster: runner, so this entry uses the existing cluster:b200-dgxc runner label" + - "Pin TileRT to the published 0.1.5.post2 wheel; decode serves one sequence at a time, so conc-list is the single point [1]" + - "kv-offloading: none -- TileRT keeps all KV state on the GPU and exposes no DRAM offload tier" + - "GLM-5.1 has 202752 native positions, so AIPerf's --max-context-length drops the corpus traces whose peak context exceeds it: 175 of the 393 traces in cc-traces-weka-062126-256k remain eligible" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2650 + +- config-keys: + - glm5.1-fp8-b200-tilert-agentic + scenario-type: + - agentic-coding + description: + - "Replace the unavailable upstream TileRT post3 dependency with the reproducible 0.1.5.post2+inferencex.1 queueing backport: preserve the official post2 native libraries, patch only pd_router.py, and pin the internal release wheel by URL and SHA256" + - "Wait up to 1800 seconds for the single decode node instead of returning HTTP 429 immediately; this addresses AgentX intra-session fan-out while keeping the benchmark concurrency point at 1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2650 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 8c2b357b00..498ad7d1c1 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -138,7 +138,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then export UCX_NET_DEVICES="${UCX_NET_DEVICES:-mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1}" export UCX_MEMTYPE_CACHE="${UCX_MEMTYPE_CACHE:-n}" export UCX_MEMTYPE_REG_WHOLE="${UCX_MEMTYPE_REG_WHOLE:-n}" - TILERT_DISAGG="$GITHUB_WORKSPACE/benchmarks/multi_node/${EXP_NAME%%_*}_${PRECISION}_b200_${FRAMEWORK}-disagg.sh" + TILERT_SUBDIR="multi_node" + [[ "${SCENARIO_SUBDIR}" == "agentic/" ]] && TILERT_SUBDIR="multi_node/agentic" + TILERT_DISAGG="$GITHUB_WORKSPACE/benchmarks/${TILERT_SUBDIR}/${EXP_NAME%%_*}_${PRECISION}_b200_${FRAMEWORK}-disagg.sh" [[ -f "$TILERT_DISAGG" ]] || { echo "tilert disagg script not found: $TILERT_DISAGG"; exit 1; } exec bash "$TILERT_DISAGG" exit 1