From cd71b9e510fd4e97bed91a8a66ac9c29614ff2da Mon Sep 17 00:00:00 2001 From: Samuel Shen Date: Thu, 13 Aug 2026 11:35:02 -0700 Subject: [PATCH 1/5] kimik3-fp4-b300-vllm-agentic-dspark: add LMCache DRAM KV-offload arm Add a dedicated config key kimik3-fp4-b300-vllm-agentic-dspark-lmcache with an LMCache 0.5.4rc2 DRAM KV-offload arm at TP8 conc 4/8/16, on top of the unchanged DSpark MTP serving stack of kimik3-fp4-b300-vllm-agentic-dspark (same image, script, and topology). A separate key means the changelog selects only the LMCache points; the resident and vllm-simple arms of the base key are not re-run. B300 sister of the MI355X arm in #2583. --- .../agentic/kimik3_fp4_b300_vllm_mtp.sh | 92 ++++++++++++++++++- configs/nvidia-master.yaml | 26 ++++++ perf-changelog.yaml | 9 ++ 3 files changed, 126 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh index 1d9bff132..07fc81b80 100755 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh @@ -120,6 +120,20 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" +# The LMCache arm starts a second long-lived process that must not outlive this +# job (its L1 holds the whole host-DRAM budget). The vLLM server's lifecycle is +# left exactly as it was -- the job wrapper still owns it. +LMCACHE_PID="" + +cleanup_lmcache_server() { + local exit_code=$? + trap - EXIT + set +e + stop_background_process_tree "$LMCACHE_PID" "LMCache server" + exit "$exit_code" +} +trap cleanup_lmcache_server EXIT + # ---- KV offloading ---------------------------------------------------------- # The generated TOTAL_CPU_DRAM_GB budget is the aggregate host-DRAM pool for the # node; SimpleCPUOffloadConnector is sized per rank. At dram-utilization 0.63 on @@ -142,8 +156,84 @@ case "${KV_OFFLOAD_BACKEND:-}" in "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":${CPU_BYTES_PER_RANK},\"lazy_offload\":false}}" ) ;; + lmcache) + require_agentic_kv_offload_backend lmcache + + # Install LMCache's runtime dependencies that this image does not + # already ship, then LMCache itself with --no-deps so pip never + # re-resolves the image's tested torch/CUDA stack. None of the three + # dependency packages depends on torch, so they resolve safely. + # cupy-cuda13x is the CUDA counterpart of the cupy-rocm-7-0 pin in the + # MI355X sister arm; LMCache lists it as a hard requirement. + LMCACHE_VERSION="0.5.4rc2" + agentic_pip_install --quiet --no-cache-dir \ + "sortedcontainers==2.4.0" \ + "opentelemetry-exporter-prometheus==0.61b0" \ + "cupy-cuda13x==14.1.1" + agentic_pip_install --quiet --no-cache-dir --no-deps \ + "lmcache==${LMCACHE_VERSION}" + python3 -c \ + "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ + >/dev/null + + # One MP server for the node, per the Kimi-K3 recipe + # (docs.lmcache.ai/recipes/kimi_k3.html). --chunk-size 768 is that + # recipe's CUDA-path value; the connector requires the chunk to be a + # multiple of every engine KV group's tokens_per_block, so if the + # server log reports a mismatch, raise it to the least common multiple + # of the block sizes vLLM prints at startup ("Setting attention block + # size to N" plus the KDA state group's own size) -- the ROCm sister + # arm needs 3072 for exactly this reason. K3's hybrid KDA/MLA layout + # registers more than one KV-cache group under MTP, which additionally + # requires one object group per sliding-window size: + # --separate-object-groups. + LMCACHE_PORT=6555 + LMCACHE_HTTP_PORT=8090 + LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" + + # Consume the generated aggregate budget verbatim, per + # benchmarks/single_node/agentic/README.md. --shm-name "" keeps L1 in + # ordinary process memory so the budget is not silently capped by the + # size of the node's /dev/shm mount. + LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB" + + LMCACHE_CMD=( + lmcache server + --host 127.0.0.1 + --port "$LMCACHE_PORT" + --http-host 127.0.0.1 + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$LMCACHE_L1_SIZE_GB" + --l1-init-size-gb 10 + --chunk-size 768 + --separate-object-groups + --enable-extra-logging + --extra-logging-interval 30 + --max-cpu-workers 8 + --max-gpu-workers 1 + --eviction-policy LRU + --supported-transfer-mode lmcache_driven + --shm-name "" + ) + append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + wait_for_ready \ + --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \ + --log "$LMCACHE_LOG" \ + --pid "$LMCACHE_PID" \ + --sleep-interval 1 \ + --timeout 600 + + # 100k-330k-token agentic prefixes make single retrieves large; use the + # same MQ timeout headroom as the MI355X arm. + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}" + ) + ;; *) - echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or vllm-simple)" >&2 + echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty, vllm-simple, or lmcache)" >&2 exit 1 ;; esac diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b95f59618..9e27ba365 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1546,6 +1546,32 @@ kimik3-fp4-b300-vllm-agentic-dspark: # TP8 SimpleCPUOffload (host DRAM) - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16] } +# LMCache MP-server DRAM offload on top of the same DSpark MTP serving stack as +# kimik3-fp4-b300-vllm-agentic-dspark (same image, script, and topology). A +# dedicated key so LMCache points can be selected and swept without re-running +# the resident and vllm-simple arms of the base key. +kimik3-fp4-b300-vllm-agentic-dspark-lmcache: + image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b300-nv + precision: fp4 + framework: vllm + multinode: false + scenarios: + # Agentic-coding only, and 0.63 matches the base key so the LMCache L1 gets + # exactly the host-DRAM budget the SimpleCPUOffload arm gets (~1,889 GB + # aggregate at TP8). The LMCache server runs with --shm-name "" so that L1 + # lives in ordinary process memory and is not capped by /dev/shm. + agentic-coding: + - dram-utilization: 0.63 + search-space: + # conc 4/8/16 overlaps the base key's vllm-simple ladder at its top three + # offload points, so LMCache is directly comparable there. TP8-only for + # the same reason as the base key: a ~1.5 TB MXFP4 checkpoint does not + # fit below 8 GPUs. + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.4rc2" }, conc-list: [4, 8, 16] } + dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 model: deepseek-ai/DeepSeek-R1-0528 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6082f2b86..e296a3998 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5918,3 +5918,12 @@ - "Add a TEP2 arm (tp 2, ep 2) to the qwen3.5-fp4-b200-sglang-mtp 8k/1k sweep at concurrency 16, 32, and 64" - "Rides on the NVFP4-V2 checkpoint switch from #2205" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2550 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark-lmcache + scenario-type: + - agentic-coding + description: + - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config." + - "Run one LMCache MP server per node with --separate-object-groups for the hybrid KDA/MLA multi-group layout, keeping the L1 in process memory (--shm-name \"\") so the DRAM budget is not capped by /dev/shm." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PR_NUMBER From 57a9b8863062f874f8fb1132e0a80e699b7a0072 Mon Sep 17 00:00:00 2001 From: Samuel Shen Date: Thu, 13 Aug 2026 11:36:29 -0700 Subject: [PATCH 2/5] perf-changelog: point the LMCache B300 entry at #2593 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e296a3998..556968fbe 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5926,4 +5926,4 @@ description: - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config." - "Run one LMCache MP server per node with --separate-object-groups for the hybrid KDA/MLA multi-group layout, keeping the L1 in process memory (--shm-name \"\") so the DRAM budget is not capped by /dev/shm." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PR_NUMBER + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2593 From dcf60b51050a9c7d70cee56671bd6e736f80ea1a Mon Sep 17 00:00:00 2001 From: Samuel Shen Date: Thu, 13 Aug 2026 11:38:05 -0700 Subject: [PATCH 3/5] kimik3-fp4-b300-vllm-agentic-dspark-lmcache: sweep conc 4/8/10/16 --- configs/nvidia-master.yaml | 11 ++++++----- perf-changelog.yaml | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 9e27ba365..32ab2dfec 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1566,11 +1566,12 @@ kimik3-fp4-b300-vllm-agentic-dspark-lmcache: agentic-coding: - dram-utilization: 0.63 search-space: - # conc 4/8/16 overlaps the base key's vllm-simple ladder at its top three - # offload points, so LMCache is directly comparable there. TP8-only for - # the same reason as the base key: a ~1.5 TB MXFP4 checkpoint does not - # fit below 8 GPUs. - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.4rc2" }, conc-list: [4, 8, 16] } + # 4/8/16 land on the base key's vllm-simple ladder at its top three + # offload points, so LMCache is directly comparable there; 10 fills the + # 8->16 gap, where the MI355X sister arm's ladder is densest. TP8-only + # for the same reason as the base key: a ~1.5 TB MXFP4 checkpoint does + # not fit below 8 GPUs. + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.4rc2" }, conc-list: [4, 8, 10, 16] } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 556968fbe..84513f45b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5924,6 +5924,6 @@ scenario-type: - agentic-coding description: - - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config." + - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/10/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config." - "Run one LMCache MP server per node with --separate-object-groups for the hybrid KDA/MLA multi-group layout, keeping the L1 in process memory (--shm-name \"\") so the DRAM budget is not capped by /dev/shm." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2593 From b529bef46f60f4e2a267ae9a94c5421e8f596372 Mon Sep 17 00:00:00 2001 From: Samuel Shen Date: Thu, 13 Aug 2026 11:39:15 -0700 Subject: [PATCH 4/5] kimik3 b300 lmcache: install the published cu129 wheel --- .../agentic/kimik3_fp4_b300_vllm_mtp.sh | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh index 07fc81b80..9a9c4ce89 100755 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh @@ -159,19 +159,28 @@ case "${KV_OFFLOAD_BACKEND:-}" in lmcache) require_agentic_kv_offload_backend lmcache - # Install LMCache's runtime dependencies that this image does not - # already ship, then LMCache itself with --no-deps so pip never - # re-resolves the image's tested torch/CUDA stack. None of the three - # dependency packages depends on torch, so they resolve safely. - # cupy-cuda13x is the CUDA counterpart of the cupy-rocm-7-0 pin in the - # MI355X sister arm; LMCache lists it as a hard requirement. + # LMCache's own CUDA 12.9 build, not the generic PyPI wheel -- the + # release publishes a dedicated -cu129 asset set, the CUDA counterpart + # of the -rocm assets the MI355X sister arm installs. This is the + # upstream install line, which reads in uv form as: + # + # uv pip install lmcache==v$VERSION \ + # --extra-index-url https://download.pytorch.org/whl/cu129 \ + # --find-links .../expanded_assets/v$VERSION-cu129 \ + # --index-strategy unsafe-best-match + # + # --index-strategy has no pip counterpart because it does not need one: + # pip already resolves best-match across every configured index, which + # is exactly what unsafe-best-match restores in uv. torch is an + # unpinned LMCache requirement that the image already satisfies, so + # its tested build is left alone and the cu129 torch index is only + # consulted for CUDA wheels pip would otherwise miss. LMCACHE_VERSION="0.5.4rc2" + LMCACHE_CUDA_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-cu129" agentic_pip_install --quiet --no-cache-dir \ - "sortedcontainers==2.4.0" \ - "opentelemetry-exporter-prometheus==0.61b0" \ - "cupy-cuda13x==14.1.1" - agentic_pip_install --quiet --no-cache-dir --no-deps \ - "lmcache==${LMCACHE_VERSION}" + "lmcache==${LMCACHE_VERSION}" \ + --extra-index-url https://download.pytorch.org/whl/cu129 \ + --find-links "$LMCACHE_CUDA_INDEX" python3 -c \ "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ >/dev/null From 07e114d287a4180a16f210acba887755375bfd78 Mon Sep 17 00:00:00 2001 From: Samuel Shen Date: Thu, 13 Aug 2026 11:45:57 -0700 Subject: [PATCH 5/5] kimik3 b300 lmcache: chunk size 3072 to match the 1536 attention block --- .../agentic/kimik3_fp4_b300_vllm_mtp.sh | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh index 9a9c4ce89..823405965 100755 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh @@ -186,15 +186,19 @@ case "${KV_OFFLOAD_BACKEND:-}" in >/dev/null # One MP server for the node, per the Kimi-K3 recipe - # (docs.lmcache.ai/recipes/kimi_k3.html). --chunk-size 768 is that - # recipe's CUDA-path value; the connector requires the chunk to be a - # multiple of every engine KV group's tokens_per_block, so if the - # server log reports a mismatch, raise it to the least common multiple - # of the block sizes vLLM prints at startup ("Setting attention block - # size to N" plus the KDA state group's own size) -- the ROCm sister - # arm needs 3072 for exactly this reason. K3's hybrid KDA/MLA layout - # registers more than one KV-cache group under MTP, which additionally - # requires one object group per sliding-window size: + # (docs.lmcache.ai/recipes/kimi_k3.html), but NOT that recipe's + # --chunk-size 768: the connector requires the chunk to be a multiple + # of every engine KV group's tokens_per_block, and this stack's blocks + # are far larger than 768. On this exact image and script, vLLM pins + # the attention group to 1536 (run 31404943911, interface.py:911, + # "Setting attention block size to 1536 tokens to ensure that + # attention page size is >= mamba page size") and pads the KDA/mamba + # page to match (interface.py:935). That alignment is generic vLLM + # code, not platform-specific, which is why the MI355X sister arm + # lands on the same 1536 plus a 3072-token KDA state group and also + # needs 3072. 768 is smaller than the attention block, so it is not a + # multiple of it and fails at connector init. The multi-group layout + # additionally requires one object group per sliding-window size: # --separate-object-groups. LMCACHE_PORT=6555 LMCACHE_HTTP_PORT=8090 @@ -214,7 +218,7 @@ case "${KV_OFFLOAD_BACKEND:-}" in --http-port "$LMCACHE_HTTP_PORT" --l1-size-gb "$LMCACHE_L1_SIZE_GB" --l1-init-size-gb 10 - --chunk-size 768 + --chunk-size 3072 --separate-object-groups --enable-extra-logging --extra-logging-interval 30