From 42692ef7774a5c874154aab0ad746ff899d6203c Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:25:40 -0400 Subject: [PATCH 1/3] feat(qwen3.5-fp4-mi355x-sglang-agentic-mtp): add HiCache DRAM offload arms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror every resident TP2/EP2 and TP4 EAGLE MTP concurrency point with a HiCache DRAM KV-offload arm, so each measured resident point has a host-tier counterpart on MI355X. The HiCache arms run at page size 64 with the kernel io-backend and page_first layout -- the transfer path Qwen3.5's hybrid attention/Mamba host pools already build on gfx950 -- sized to 80% of the workflow DRAM budget under a 180 GB per-rank ceiling. Resident arms keep page size 16 and are untouched. They also patch sgl-project/sglang#34560 into the container. Qwen3.5 keeps its language-model attributes in the nested text_config and SGLang normalizes the MTP draft depth only on the parent HF config, so ModelConfig.num_nextn_predict_layers stays None, the packed/sidecar routing added in sgl-project/sglang#30393 misroutes the draft cache to the sidecar path, and the scheduler aborts with "AttributeError: 'HybridLinearKVPool' object has no attribute 'layer_num'". The patch is scoped to the HiCache arms and is a no-op once the fix ships in an MI355X image. 为 MI355X Qwen3.5 MXFP4 AgentX EAGLE MTP 配方新增 HiCache DRAM KV 卸载分支， 与现有 TP2/EP2 与 TP4 常驻并发点一一对应，使每个已测常驻点都有对应的主机层数据点。 HiCache 分支使用 page size 64、kernel io-backend 与 page_first 布局，即 Qwen3.5 混合注意力/Mamba 主机池在 gfx950 上已经能够构建的传输路径；容量按工作流 DRAM 预算的 80% 计算，并设置每 rank 180 GB 上限。常驻分支仍为 page size 16， 保持不变。 该分支同时在容器内应用 sgl-project/sglang#34560 的修复。Qwen3.5 将语言模型属性 存放在嵌套的 text_config 中，而 SGLang 仅在父 HF config 上归一化 MTP draft 深度，导致 ModelConfig.num_nextn_predict_layers 保持为 None； sgl-project/sglang#30393 引入的 packed/sidecar 路由因此将 draft cache 误判到 sidecar 路径，调度器在启动时抛出 "AttributeError: 'HybridLinearKVPool' object has no attribute 'layer_num'"。 该补丁仅作用于 HiCache 分支，待修复进入 MI355X 镜像后即为空操作。 Co-Authored-By: Claude Opus 5 (1M context) --- .../agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh | 99 ++++++++++++++++++- configs/amd-master.yaml | 2 + perf-changelog.yaml | 10 ++ 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh index 804d65b08..6091369d2 100644 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -11,7 +11,8 @@ source "$(dirname "$0")/../../benchmark_lib.sh" export EVAL_FRAMEWORK="lm-eval" check_env_vars \ - MODEL TP CONC EP_SIZE RESULT_DIR DURATION + MODEL TP CONC EP_SIZE KV_OFFLOADING \ + TOTAL_CPU_DRAM_GB RESULT_DIR DURATION SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} @@ -53,6 +54,99 @@ trap cleanup_agentic_services EXIT trap 'exit 130' INT trap 'exit 143' TERM +# Resident arms keep the page size the TP2/EP2 and TP4 sweeps were measured +# with. HiCache arms move to 64 because Qwen3.5's hybrid attention/Mamba host +# pools transfer page-first, and page size 1 fails the EAGLE verify-graph +# compile on gfx950. +CACHE_ARGS=() +PAGE_SIZE=16 +if require_agentic_kv_offload_backend hicache; then + PAGE_SIZE=64 + + # sgl-project/sglang#30393 (merged 2026-08-06) routes an MTP draft KV cache + # to either a packed or a sidecar HiCache pool. Qwen3.5 conditional- + # generation checkpoints keep their language-model attributes in the nested + # text_config, and SGLang normalizes the draft depth only on the parent HF + # config, so ModelConfig.num_nextn_predict_layers stays None, the draft is + # misrouted to the sidecar path, and the scheduler dies during startup with + # AttributeError: 'HybridLinearKVPool' object has no attribute 'layer_num' + # Apply the one-line fix from sgl-project/sglang#34560 until it ships in an + # MI355X image. Resident runs never reach this routing, so the patch stays + # scoped to the HiCache arms and leaves the measured resident data alone. + python3 - /sgl-workspace/sglang/python/sglang/srt/configs/model_config.py <<'PYPATCH' +import sys + +path = sys.argv[1] +anchor = 'self.hf_config.architectures[0] = "Qwen3_5ForCausalLMMTP"' +assign = "self.hf_config.num_nextn_predict_layers = 1" +fix = "self.hf_text_config.num_nextn_predict_layers = 1" + +with open(path) as fh: + lines = fh.readlines() + +matches = [i for i, line in enumerate(lines) if line.strip() == anchor] +if len(matches) != 1: + sys.exit(f"sglang#34560: expected 1 Qwen3.5 MTP anchor in {path}, found {len(matches)}") + +i = matches[0] +if lines[i + 1].strip() != assign: + sys.exit(f"sglang#34560: unexpected line after anchor in {path}: {lines[i + 1]!r}") +if lines[i + 2].strip() == fix: + print("sglang#34560 already applied") + sys.exit(0) + +indent = lines[i + 1][: len(lines[i + 1]) - len(lines[i + 1].lstrip())] +lines.insert(i + 2, f"{indent}{fix}\n") +with open(path, "w") as fh: + fh.writelines(lines) +print("sglang#34560 applied") +PYPATCH + + # --hicache-size is the per-rank budget SGLang splits across Qwen3.5's two + # hybrid host pools, not a per-pool figure: on this image at TP4 with + # --hicache-size 144 the ranks allocated 93.37 GB target KV + 50.65 GB Mamba + # = 144.02 GB each. Packed NEXTN then adds its single draft layer on top of + # the 60 transferred target layers. Hold the node to 80% of the workflow + # DRAM budget so the draft layer, page alignment, and the trace-replay + # client cannot walk the host into the OOM killer mid-storm. + HICACHE_ALIGNMENT_RESERVE_GB=$TP + HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) + if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then + echo "Error: insufficient DRAM after HiCache alignment reserve." >&2 + exit 1 + fi + MAX_HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 80 / 100 * 60 / 61 / TP)) + # 144 GB/rank is the largest pool observed to allocate on this image; 180 + # is a bounded step up from it. Raise once a run confirms the larger pinned + # allocation stays inside the watchdog. + HICACHE_MAX_SIZE_GB_PER_RANK=${HICACHE_MAX_SIZE_GB_PER_RANK:-180} + if [ "$MAX_HICACHE_SIZE_GB" -gt "$HICACHE_MAX_SIZE_GB_PER_RANK" ]; then + MAX_HICACHE_SIZE_GB="$HICACHE_MAX_SIZE_GB_PER_RANK" + fi + HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" + if [ "$HICACHE_SIZE_GB" -lt 1 ] || [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then + echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB outside 1..$MAX_HICACHE_SIZE_GB." >&2 + exit 1 + fi + PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 61 + 59) / 60 + HICACHE_ALIGNMENT_RESERVE_GB)) + if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then + echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured ${TOTAL_CPU_DRAM_GB} GB." >&2 + exit 1 + fi + echo "HiCache pools: ${HICACHE_SIZE_GB} GB per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB of ${TOTAL_CPU_DRAM_GB} GB." + + # kernel + page_first is the transfer path the hybrid KV/Mamba stack already + # builds on gfx950 (both host pools and the pool-stack attach complete under + # it). write_through_selective matches the B300 Qwen3.5 MTP sibling. + CACHE_ARGS=( + --enable-hierarchical-cache + --hicache-size "$HICACHE_SIZE_GB" + --hicache-io-backend kernel + --hicache-mem-layout page_first + --hicache-write-policy write_through_selective + ) +fi + PARALLEL_ARGS=( --tp "$TP" --dp 1 @@ -93,7 +187,7 @@ SGLANG_CMD=( --mem-fraction-static 0.80 --model-loader-extra-config '{"enable_multithread_load": true}' --watchdog-timeout 1200 - --page-size 16 + --page-size "$PAGE_SIZE" --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" --max-running-requests "$MAX_RUNNING_REQUESTS" --max-prefill-tokens 32768 @@ -110,6 +204,7 @@ SGLANG_CMD=( --speculative-num-draft-tokens 4 --enable-metrics --enable-cache-report + "${CACHE_ARGS[@]}" ) printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 07f3defd9..b79bd24a0 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -387,7 +387,9 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: - dram-utilization: 0.80 search-space: - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20] } - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a1875d61b..96d4b19d0 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5894,3 +5894,13 @@ - "Cover the measured resident TP2/EP2 and TP4 Pareto ranges through their HBM capacity knees, with required SGLang metrics exports." - "Use SGLang v0.5.17 and disable unstable AITER all-reduce fusion for TP2/EP2 EAGLE rank consistency." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2562 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add HiCache DRAM KV-offload arms mirroring the resident TP2/EP2 and TP4 EAGLE MTP concurrency points, so every measured resident point has a host-tier counterpart." + - "Run the HiCache arms at page size 64 with kernel io-backend and page_first layout, sized to 80 percent of the workflow DRAM budget at a 180 GB per-rank ceiling." + - "Patch sgl-project/sglang#34560 into the container for the HiCache arms only: Qwen3.5 leaves num_nextn_predict_layers unset on hf_text_config, which misroutes the MTP draft cache to the sidecar path and aborts startup." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING From d309134b66d1cfe780688a075f697c24d67095b5 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:28:04 -0400 Subject: [PATCH 2/3] chore: link Qwen3.5 MI355X HiCache changelog to PR #2582 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 Qwen3.5 MI355X HiCache 变更日志条目关联至 PR #2582。 Co-Authored-By: Claude Opus 5 (1M context) --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 96d4b19d0..e825f755f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5903,4 +5903,4 @@ - "Add HiCache DRAM KV-offload arms mirroring the resident TP2/EP2 and TP4 EAGLE MTP concurrency points, so every measured resident point has a host-tier counterpart." - "Run the HiCache arms at page size 64 with kernel io-backend and page_first layout, sized to 80 percent of the workflow DRAM budget at a 180 GB per-rank ceiling." - "Patch sgl-project/sglang#34560 into the container for the HiCache arms only: Qwen3.5 leaves num_nextn_predict_layers unset on hf_text_config, which misroutes the MTP draft cache to the sidecar path and aborts startup." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2582 From 24facf0a60297e405274ea93678e61c9113e2b0f Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:02:50 -0400 Subject: [PATCH 3/3] test: isolate one TP4 HiCache point to validate sglang#34560 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment out the merged resident arms and the remaining HiCache concurrency points, leaving TP4/EP1 conc 16 with kv-offloading dram + hicache. This proves whether the sgl-project/sglang#34560 container patch clears the HybridLinearKVPool draft-sidecar crash on gfx950 without spending MI355X node time on already-measured resident points. 注释掉已合入的常驻分支与其余 HiCache 并发点，仅保留 TP4/EP1 并发 16 的 kv-offloading dram + hicache 配置，用于验证 sgl-project/sglang#34560 容器补丁 能否在 gfx950 上消除 HybridLinearKVPool draft sidecar 崩溃，同时避免在已测的 常驻点上占用 MI355X 节点时间。 Co-Authored-By: Claude Opus 5 (1M context) --- configs/amd-master.yaml | 14 ++++++++++---- perf-changelog.yaml | 4 ++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b79bd24a0..fbbe1ba64 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -386,10 +386,16 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } + # Isolating one HiCache point to prove sgl-project/sglang#34560 clears the + # HybridLinearKVPool draft-sidecar crash on gfx950. The resident arms are + # already merged and measured (#2562), so they stay commented out to keep + # this iteration off the MI355X node; restore all four rows once the + # HiCache path boots. + # - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } + # - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20] } + # - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } + # - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16] } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e825f755f..a49d20db8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5900,7 +5900,7 @@ scenario-type: - agentic-coding description: - - "Add HiCache DRAM KV-offload arms mirroring the resident TP2/EP2 and TP4 EAGLE MTP concurrency points, so every measured resident point has a host-tier counterpart." - - "Run the HiCache arms at page size 64 with kernel io-backend and page_first layout, sized to 80 percent of the workflow DRAM budget at a 180 GB per-rank ceiling." + - "Add HiCache DRAM KV-offload support to the recipe and isolate a single TP4 concurrency 16 point to prove the path boots on gfx950; the resident arms and the remaining HiCache points stay commented out for this iteration." + - "Run the HiCache arm at page size 64 with kernel io-backend and page_first layout, sized to 80 percent of the workflow DRAM budget at a 180 GB per-rank ceiling." - "Patch sgl-project/sglang#34560 into the container for the HiCache arms only: Qwen3.5 leaves num_nextn_predict_layers unset on hf_text_config, which misroutes the MTP draft cache to the sidecar path and aborts startup." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2582