From cd71b9e510fd4e97bed91a8a66ac9c29614ff2da Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Thu, 13 Aug 2026 11:35:02 -0700
Subject: [PATCH 01/18] kimik3-fp4-b300-vllm-agentic-dspark: add LMCache DRAM
 KV-offload arm

Add a dedicated config key kimik3-fp4-b300-vllm-agentic-dspark-lmcache with
an LMCache 0.5.4rc2 DRAM KV-offload arm at TP8 conc 4/8/16, on top of the
unchanged DSpark MTP serving stack of kimik3-fp4-b300-vllm-agentic-dspark
(same image, script, and topology). A separate key means the changelog
selects only the LMCache points; the resident and vllm-simple arms of the
base key are not re-run.

B300 sister of the MI355X arm in #2583.
---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh       | 92 ++++++++++++++++++-
 configs/nvidia-master.yaml                    | 26 ++++++
 perf-changelog.yaml                           |  9 ++
 3 files changed, 126 insertions(+), 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 1d9bff132..07fc81b80 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -120,6 +120,20 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
 SERVER_LOG="$RESULT_DIR/server.log"
 mkdir -p "$RESULT_DIR"
 
+# The LMCache arm starts a second long-lived process that must not outlive this
+# job (its L1 holds the whole host-DRAM budget). The vLLM server's lifecycle is
+# left exactly as it was -- the job wrapper still owns it.
+LMCACHE_PID=""
+
+cleanup_lmcache_server() {
+    local exit_code=$?
+    trap - EXIT
+    set +e
+    stop_background_process_tree "$LMCACHE_PID" "LMCache server"
+    exit "$exit_code"
+}
+trap cleanup_lmcache_server EXIT
+
 # ---- KV offloading ----------------------------------------------------------
 # The generated TOTAL_CPU_DRAM_GB budget is the aggregate host-DRAM pool for the
 # node; SimpleCPUOffloadConnector is sized per rank. At dram-utilization 0.63 on
@@ -142,8 +156,84 @@ case "${KV_OFFLOAD_BACKEND:-}" in
             "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":${CPU_BYTES_PER_RANK},\"lazy_offload\":false}}"
         )
         ;;
+    lmcache)
+        require_agentic_kv_offload_backend lmcache
+
+        # Install LMCache's runtime dependencies that this image does not
+        # already ship, then LMCache itself with --no-deps so pip never
+        # re-resolves the image's tested torch/CUDA stack. None of the three
+        # dependency packages depends on torch, so they resolve safely.
+        # cupy-cuda13x is the CUDA counterpart of the cupy-rocm-7-0 pin in the
+        # MI355X sister arm; LMCache lists it as a hard requirement.
+        LMCACHE_VERSION="0.5.4rc2"
+        agentic_pip_install --quiet --no-cache-dir \
+            "sortedcontainers==2.4.0" \
+            "opentelemetry-exporter-prometheus==0.61b0" \
+            "cupy-cuda13x==14.1.1"
+        agentic_pip_install --quiet --no-cache-dir --no-deps \
+            "lmcache==${LMCACHE_VERSION}"
+        python3 -c \
+            "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
+            >/dev/null
+
+        # One MP server for the node, per the Kimi-K3 recipe
+        # (docs.lmcache.ai/recipes/kimi_k3.html). --chunk-size 768 is that
+        # recipe's CUDA-path value; the connector requires the chunk to be a
+        # multiple of every engine KV group's tokens_per_block, so if the
+        # server log reports a mismatch, raise it to the least common multiple
+        # of the block sizes vLLM prints at startup ("Setting attention block
+        # size to N" plus the KDA state group's own size) -- the ROCm sister
+        # arm needs 3072 for exactly this reason. K3's hybrid KDA/MLA layout
+        # registers more than one KV-cache group under MTP, which additionally
+        # requires one object group per sliding-window size:
+        # --separate-object-groups.
+        LMCACHE_PORT=6555
+        LMCACHE_HTTP_PORT=8090
+        LMCACHE_LOG="$RESULT_DIR/lmcache_server.log"
+
+        # Consume the generated aggregate budget verbatim, per
+        # benchmarks/single_node/agentic/README.md. --shm-name "" keeps L1 in
+        # ordinary process memory so the budget is not silently capped by the
+        # size of the node's /dev/shm mount.
+        LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB"
+
+        LMCACHE_CMD=(
+            lmcache server
+            --host 127.0.0.1
+            --port "$LMCACHE_PORT"
+            --http-host 127.0.0.1
+            --http-port "$LMCACHE_HTTP_PORT"
+            --l1-size-gb "$LMCACHE_L1_SIZE_GB"
+            --l1-init-size-gb 10
+            --chunk-size 768
+            --separate-object-groups
+            --enable-extra-logging
+            --extra-logging-interval 30
+            --max-cpu-workers 8
+            --max-gpu-workers 1
+            --eviction-policy LRU
+            --supported-transfer-mode lmcache_driven
+            --shm-name ""
+        )
+        append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}"
+        "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 &
+        LMCACHE_PID=$!
+        wait_for_ready \
+            --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \
+            --log "$LMCACHE_LOG" \
+            --pid "$LMCACHE_PID" \
+            --sleep-interval 1 \
+            --timeout 600
+
+        # 100k-330k-token agentic prefixes make single retrieves large; use the
+        # same MQ timeout headroom as the MI355X arm.
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}"
+        )
+        ;;
     *)
-        echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or vllm-simple)" >&2
+        echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty, vllm-simple, or lmcache)" >&2
         exit 1
         ;;
 esac
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index b95f59618..9e27ba365 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1546,6 +1546,32 @@ kimik3-fp4-b300-vllm-agentic-dspark:
       # TP8 SimpleCPUOffload (host DRAM)
       - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16] }
 
+# LMCache MP-server DRAM offload on top of the same DSpark MTP serving stack as
+# kimik3-fp4-b300-vllm-agentic-dspark (same image, script, and topology). A
+# dedicated key so LMCache points can be selected and swept without re-running
+# the resident and vllm-simple arms of the base key.
+kimik3-fp4-b300-vllm-agentic-dspark-lmcache:
+  image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21
+  model: moonshotai/Kimi-K3
+  model-prefix: kimik3
+  runner: cluster:b300-nv
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    # Agentic-coding only, and 0.63 matches the base key so the LMCache L1 gets
+    # exactly the host-DRAM budget the SimpleCPUOffload arm gets (~1,889 GB
+    # aggregate at TP8). The LMCache server runs with --shm-name "" so that L1
+    # lives in ordinary process memory and is not capped by /dev/shm.
+    agentic-coding:
+    - dram-utilization: 0.63
+      search-space:
+      # conc 4/8/16 overlaps the base key's vllm-simple ladder at its top three
+      # offload points, so LMCache is directly comparable there. TP8-only for
+      # the same reason as the base key: a ~1.5 TB MXFP4 checkpoint does not
+      # fit below 8 GPUs.
+      - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.4rc2" }, conc-list: [4, 8, 16] }
+
 dsr1-fp8-b200-trt:
   image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14
   model: deepseek-ai/DeepSeek-R1-0528
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 6082f2b86..e296a3998 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -5918,3 +5918,12 @@
     - "Add a TEP2 arm (tp 2, ep 2) to the qwen3.5-fp4-b200-sglang-mtp 8k/1k sweep at concurrency 16, 32, and 64"
     - "Rides on the NVFP4-V2 checkpoint switch from #2205"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2550
+
+- config-keys:
+    - kimik3-fp4-b300-vllm-agentic-dspark-lmcache
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config."
+    - "Run one LMCache MP server per node with --separate-object-groups for the hybrid KDA/MLA multi-group layout, keeping the L1 in process memory (--shm-name \"\") so the DRAM budget is not capped by /dev/shm."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PR_NUMBER

From 57a9b8863062f874f8fb1132e0a80e699b7a0072 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Thu, 13 Aug 2026 11:36:29 -0700
Subject: [PATCH 02/18] perf-changelog: point the LMCache B300 entry at #2593

---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index e296a3998..556968fbe 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -5926,4 +5926,4 @@
   description:
     - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config."
     - "Run one LMCache MP server per node with --separate-object-groups for the hybrid KDA/MLA multi-group layout, keeping the L1 in process memory (--shm-name \"\") so the DRAM budget is not capped by /dev/shm."
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PR_NUMBER
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2593

From dcf60b51050a9c7d70cee56671bd6e736f80ea1a Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Thu, 13 Aug 2026 11:38:05 -0700
Subject: [PATCH 03/18] kimik3-fp4-b300-vllm-agentic-dspark-lmcache: sweep conc
 4/8/10/16

---
 configs/nvidia-master.yaml | 11 ++++++-----
 perf-changelog.yaml        |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 9e27ba365..32ab2dfec 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1566,11 +1566,12 @@ kimik3-fp4-b300-vllm-agentic-dspark-lmcache:
     agentic-coding:
     - dram-utilization: 0.63
       search-space:
-      # conc 4/8/16 overlaps the base key's vllm-simple ladder at its top three
-      # offload points, so LMCache is directly comparable there. TP8-only for
-      # the same reason as the base key: a ~1.5 TB MXFP4 checkpoint does not
-      # fit below 8 GPUs.
-      - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.4rc2" }, conc-list: [4, 8, 16] }
+      # 4/8/16 land on the base key's vllm-simple ladder at its top three
+      # offload points, so LMCache is directly comparable there; 10 fills the
+      # 8->16 gap, where the MI355X sister arm's ladder is densest. TP8-only
+      # for the same reason as the base key: a ~1.5 TB MXFP4 checkpoint does
+      # not fit below 8 GPUs.
+      - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.4rc2" }, conc-list: [4, 8, 10, 16] }
 
 dsr1-fp8-b200-trt:
   image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 556968fbe..84513f45b 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -5924,6 +5924,6 @@
   scenario-type:
     - agentic-coding
   description:
-    - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config."
+    - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/10/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config."
     - "Run one LMCache MP server per node with --separate-object-groups for the hybrid KDA/MLA multi-group layout, keeping the L1 in process memory (--shm-name \"\") so the DRAM budget is not capped by /dev/shm."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2593

From b529bef46f60f4e2a267ae9a94c5421e8f596372 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Thu, 13 Aug 2026 11:39:15 -0700
Subject: [PATCH 04/18] kimik3 b300 lmcache: install the published cu129 wheel

---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh       | 31 ++++++++++++-------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 07fc81b80..9a9c4ce89 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -159,19 +159,28 @@ case "${KV_OFFLOAD_BACKEND:-}" in
     lmcache)
         require_agentic_kv_offload_backend lmcache
 
-        # Install LMCache's runtime dependencies that this image does not
-        # already ship, then LMCache itself with --no-deps so pip never
-        # re-resolves the image's tested torch/CUDA stack. None of the three
-        # dependency packages depends on torch, so they resolve safely.
-        # cupy-cuda13x is the CUDA counterpart of the cupy-rocm-7-0 pin in the
-        # MI355X sister arm; LMCache lists it as a hard requirement.
+        # LMCache's own CUDA 12.9 build, not the generic PyPI wheel -- the
+        # release publishes a dedicated -cu129 asset set, the CUDA counterpart
+        # of the -rocm assets the MI355X sister arm installs. This is the
+        # upstream install line, which reads in uv form as:
+        #
+        #   uv pip install lmcache==v$VERSION \
+        #     --extra-index-url https://download.pytorch.org/whl/cu129 \
+        #     --find-links .../expanded_assets/v$VERSION-cu129 \
+        #     --index-strategy unsafe-best-match
+        #
+        # --index-strategy has no pip counterpart because it does not need one:
+        # pip already resolves best-match across every configured index, which
+        # is exactly what unsafe-best-match restores in uv. torch is an
+        # unpinned LMCache requirement that the image already satisfies, so
+        # its tested build is left alone and the cu129 torch index is only
+        # consulted for CUDA wheels pip would otherwise miss.
         LMCACHE_VERSION="0.5.4rc2"
+        LMCACHE_CUDA_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-cu129"
         agentic_pip_install --quiet --no-cache-dir \
-            "sortedcontainers==2.4.0" \
-            "opentelemetry-exporter-prometheus==0.61b0" \
-            "cupy-cuda13x==14.1.1"
-        agentic_pip_install --quiet --no-cache-dir --no-deps \
-            "lmcache==${LMCACHE_VERSION}"
+            "lmcache==${LMCACHE_VERSION}" \
+            --extra-index-url https://download.pytorch.org/whl/cu129 \
+            --find-links "$LMCACHE_CUDA_INDEX"
         python3 -c \
             "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
             >/dev/null

From 07e114d287a4180a16f210acba887755375bfd78 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Thu, 13 Aug 2026 11:45:57 -0700
Subject: [PATCH 05/18] kimik3 b300 lmcache: chunk size 3072 to match the 1536
 attention block

---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh       | 24 +++++++++++--------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 9a9c4ce89..823405965 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -186,15 +186,19 @@ case "${KV_OFFLOAD_BACKEND:-}" in
             >/dev/null
 
         # One MP server for the node, per the Kimi-K3 recipe
-        # (docs.lmcache.ai/recipes/kimi_k3.html). --chunk-size 768 is that
-        # recipe's CUDA-path value; the connector requires the chunk to be a
-        # multiple of every engine KV group's tokens_per_block, so if the
-        # server log reports a mismatch, raise it to the least common multiple
-        # of the block sizes vLLM prints at startup ("Setting attention block
-        # size to N" plus the KDA state group's own size) -- the ROCm sister
-        # arm needs 3072 for exactly this reason. K3's hybrid KDA/MLA layout
-        # registers more than one KV-cache group under MTP, which additionally
-        # requires one object group per sliding-window size:
+        # (docs.lmcache.ai/recipes/kimi_k3.html), but NOT that recipe's
+        # --chunk-size 768: the connector requires the chunk to be a multiple
+        # of every engine KV group's tokens_per_block, and this stack's blocks
+        # are far larger than 768. On this exact image and script, vLLM pins
+        # the attention group to 1536 (run 31404943911, interface.py:911,
+        # "Setting attention block size to 1536 tokens to ensure that
+        # attention page size is >= mamba page size") and pads the KDA/mamba
+        # page to match (interface.py:935). That alignment is generic vLLM
+        # code, not platform-specific, which is why the MI355X sister arm
+        # lands on the same 1536 plus a 3072-token KDA state group and also
+        # needs 3072. 768 is smaller than the attention block, so it is not a
+        # multiple of it and fails at connector init. The multi-group layout
+        # additionally requires one object group per sliding-window size:
         # --separate-object-groups.
         LMCACHE_PORT=6555
         LMCACHE_HTTP_PORT=8090
@@ -214,7 +218,7 @@ case "${KV_OFFLOAD_BACKEND:-}" in
             --http-port "$LMCACHE_HTTP_PORT"
             --l1-size-gb "$LMCACHE_L1_SIZE_GB"
             --l1-init-size-gb 10
-            --chunk-size 768
+            --chunk-size 3072
             --separate-object-groups
             --enable-extra-logging
             --extra-logging-interval 30

From eebc5525f4a0e4db1a3b9b8303ac5a03420c935a Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Thu, 13 Aug 2026 18:08:55 -0700
Subject: [PATCH 06/18] perf-changelog: point the LMCache B300 entry at #2597

---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 84513f45b..f7ce5a0ee 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -5926,4 +5926,4 @@
   description:
     - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/10/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config."
     - "Run one LMCache MP server per node with --separate-object-groups for the hybrid KDA/MLA multi-group layout, keeping the L1 in process memory (--shm-name \"\") so the DRAM budget is not capped by /dev/shm."
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2593
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2597

From 2eddc8973121d4c92abaa8fa05caa617845e01f0 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 13:16:25 -0700
Subject: [PATCH 07/18] kimik3 b300 lmcache: install the CUDA-13 wheel and fail
 on c_ops fallback

The b300-nv nodes report CUDA 13.0, but the arm installed LMCache from
the `-cu129` release assets, whose compiled `lmcache.c_ops` links
libcudart.so.12. That .so cannot be dlopen'd here, and LMCache does not
treat it as fatal: it logs

    lmcache.c_ops compiled extension not found;
    CudaDeviceOps stays on the torch baseline for all ops

and silently falls back to lmcache/v1/platform/torch_ops.py. That
fallback is broken for this stack's hybrid multi-KV-group KDA/MLA
layout -- 77-98% of stores failed with cudaErrorInvalidValue in
index_select/cudaMemcpy across every concurrency (c4 97.6%, c8 98.4%,
c10 89.1%, c16 77.5%), so the offload tier stayed empty and the sweep
measured a cache that had never stored anything.

Switch to the default release assets, whose c_ops links libcudart.so.13,
and add two guards so this degrades loudly instead of silently: import
lmcache.c_ops after install, and abort if the server log shows the
fallback warning.

The MI355X sister arm installs the `-rocm` assets, whose c_ops loads
correctly -- zero store failures there, which is why this was
CUDA-only.
---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh       | 48 ++++++++++++-------
 1 file changed, 30 insertions(+), 18 deletions(-)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 823405965..3c89ed3c8 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -159,30 +159,32 @@ case "${KV_OFFLOAD_BACKEND:-}" in
     lmcache)
         require_agentic_kv_offload_backend lmcache
 
-        # LMCache's own CUDA 12.9 build, not the generic PyPI wheel -- the
-        # release publishes a dedicated -cu129 asset set, the CUDA counterpart
-        # of the -rocm assets the MI355X sister arm installs. This is the
-        # upstream install line, which reads in uv form as:
+        # The LMCache wheel MUST match the image's CUDA major version. Its
+        # compiled `lmcache.c_ops` extension is linked against a specific
+        # libcudart, and LMCache does NOT fail when that .so cannot be
+        # dlopen'd -- it logs "lmcache.c_ops compiled extension not found;
+        # CudaDeviceOps stays on the torch baseline for all ops" and silently
+        # falls back to the pure-torch path in lmcache/v1/platform/torch_ops.py.
+        # That fallback is broken for this stack's hybrid multi-KV-group
+        # KDA/MLA layout: 77-98% of stores die with cudaErrorInvalidValue in
+        # index_select/cudaMemcpy, so the offload tier stays empty and the arm
+        # silently measures a cache that never stored anything.
         #
-        #   uv pip install lmcache==v$VERSION \
-        #     --extra-index-url https://download.pytorch.org/whl/cu129 \
-        #     --find-links .../expanded_assets/v$VERSION-cu129 \
-        #     --index-strategy unsafe-best-match
-        #
-        # --index-strategy has no pip counterpart because it does not need one:
-        # pip already resolves best-match across every configured index, which
-        # is exactly what unsafe-best-match restores in uv. torch is an
-        # unpinned LMCache requirement that the image already satisfies, so
-        # its tested build is left alone and the cu129 torch index is only
-        # consulted for CUDA wheels pip would otherwise miss.
+        # This node reports CUDA 13.0, so use the DEFAULT release assets, whose
+        # c_ops links libcudart.so.13 (cuda_13.0). Do NOT use the `-cu129`
+        # asset set: its c_ops links libcudart.so.12 (cuda_12.9) and cannot
+        # load here. The `-rocm` assets are the MI355X sister arm's equivalent.
+        # If the image's CUDA major version ever changes, this URL must change
+        # with it -- verify with `nvidia-smi | grep "CUDA Version"`.
         LMCACHE_VERSION="0.5.4rc2"
-        LMCACHE_CUDA_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-cu129"
+        LMCACHE_CUDA_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}"
         agentic_pip_install --quiet --no-cache-dir \
             "lmcache==${LMCACHE_VERSION}" \
-            --extra-index-url https://download.pytorch.org/whl/cu129 \
             --find-links "$LMCACHE_CUDA_INDEX"
+        # Import c_ops explicitly so an ABI mismatch aborts the job instead of
+        # degrading to the broken torch fallback.
         python3 -c \
-            "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
+            "import cupy; import lmcache.c_ops; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
             >/dev/null
 
         # One MP server for the node, per the Kimi-K3 recipe
@@ -238,6 +240,16 @@ case "${KV_OFFLOAD_BACKEND:-}" in
             --sleep-interval 1 \
             --timeout 600
 
+        # Second guard, on the server process itself: the import check above
+        # only proves c_ops loads in this shell's python3. Abort rather than
+        # benchmark a silently-degraded offload tier (see the install note).
+        if grep -q "c_ops compiled extension not found" "$LMCACHE_LOG"; then
+            echo "Error: LMCache fell back to the torch baseline (c_ops did not load)." >&2
+            echo "       The wheel's CUDA major version does not match this image." >&2
+            grep -m1 "c_ops compiled extension not found" "$LMCACHE_LOG" >&2
+            exit 1
+        fi
+
         # 100k-330k-token agentic prefixes make single retrieves large; use the
         # same MQ timeout headroom as the MI355X arm.
         OFFLOAD_ARGS=(

From 25e305fca26ded3d9eabe820f69927e799f4c9b5 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 14:01:55 -0700
Subject: [PATCH 08/18] kimik3 b300 lmcache: force-reinstall lmcache; the pin
 was a silent no-op

The image already ships lmcache 0.5.4rc2, so `pip install lmcache==0.5.4rc2`
found the requirement satisfied and installed nothing -- the --find-links
URL was never consulted. Two sweeps proved it: one pinned to the -cu129
assets and one to the default assets both produced the identical build,
`LMCache v0.5.4rc2 (gf82f6fd3)`, in ~3 seconds. Every run so far has used
the image's copy, whose c_ops does not load here, so the arm silently ran
on the broken torch fallback.

Add --force-reinstall (with --no-deps, to leave the image's torch/CUDA
stack alone) so the pin actually takes effect, and drop --quiet plus log
`pip show lmcache`, lmcache.__file__, the resolved c_ops artifact and the
container's libcudart -- a silent install is what hid this for two sweeps.

Also drop the `import lmcache.c_ops` guard: it is ineffective. It
succeeded while device_ops.ensure_native()'s identical import raised
ImportError in the same process, so lmcache.c_ops resolves to a shim that
imports cleanly and fails later. The server-log guard is the one that
works -- it caught this in ~60s instead of ~90 GPU-minutes per point --
so keep it, and correct its message, which still asserted the earlier
(wrong) CUDA-mismatch explanation.
---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh       | 23 +++++++++++++++----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 3c89ed3c8..c8ce925d4 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -176,15 +176,25 @@ case "${KV_OFFLOAD_BACKEND:-}" in
         # load here. The `-rocm` assets are the MI355X sister arm's equivalent.
         # If the image's CUDA major version ever changes, this URL must change
         # with it -- verify with `nvidia-smi | grep "CUDA Version"`.
+        # --force-reinstall is REQUIRED, not cosmetic: this image already ships
+        # lmcache 0.5.4rc2, so a plain `pip install lmcache==0.5.4rc2` finds the
+        # requirement already satisfied and installs nothing -- the --find-links
+        # URL is never consulted. Two earlier sweeps proved it: one pinned to
+        # the -cu129 assets and one to the default assets both produced the
+        # identical build, `LMCache v0.5.4rc2 (gf82f6fd3)`, in ~3 seconds. We
+        # were always running the image's copy, whose c_ops does not load here.
         LMCACHE_VERSION="0.5.4rc2"
         LMCACHE_CUDA_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}"
-        agentic_pip_install --quiet --no-cache-dir \
+        agentic_pip_install --no-cache-dir --force-reinstall --no-deps \
             "lmcache==${LMCACHE_VERSION}" \
             --find-links "$LMCACHE_CUDA_INDEX"
-        # Import c_ops explicitly so an ABI mismatch aborts the job instead of
-        # degrading to the broken torch fallback.
+        # Record what actually landed. Silent installs are how the no-op above
+        # went unnoticed for two full sweeps; do not add --quiet back.
+        python3 -m pip show lmcache 2>/dev/null | grep -E "^(Version|Location):"
+        python3 -c "import lmcache, glob, os; print('lmcache:', lmcache.__file__); print('c_ops so:', glob.glob(os.path.join(os.path.dirname(lmcache.__file__), 'c_ops*')))"
+        ls -1 /usr/local/cuda*/lib64/libcudart.so* /usr/lib/x86_64-linux-gnu/libcudart.so* 2>/dev/null || true
         python3 -c \
-            "import cupy; import lmcache.c_ops; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
+            "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
             >/dev/null
 
         # One MP server for the node, per the Kimi-K3 recipe
@@ -245,7 +255,10 @@ case "${KV_OFFLOAD_BACKEND:-}" in
         # benchmark a silently-degraded offload tier (see the install note).
         if grep -q "c_ops compiled extension not found" "$LMCACHE_LOG"; then
             echo "Error: LMCache fell back to the torch baseline (c_ops did not load)." >&2
-            echo "       The wheel's CUDA major version does not match this image." >&2
+            echo "       The installed lmcache's compiled extension is unusable here." >&2
+            echo "       Check the pip show / c_ops / libcudart lines logged above:" >&2
+            echo "       either the install did not take, or the build's CUDA ABI" >&2
+            echo "       does not match this image." >&2
             grep -m1 "c_ops compiled extension not found" "$LMCACHE_LOG" >&2
             exit 1
         fi

From 42a9f4ab647d46e34f85a1c6aac3845a3076099d Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 14:18:56 -0700
Subject: [PATCH 09/18] kimik3 b300 lmcache: surface the real c_ops dlopen
 error

--force-reinstall took (image shipped 0.5.3; 0.5.4rc2 now installed) and
both the c_ops .so and libcudart.so.13 are present on the node, yet the
import still fails. device_ops.ensure_native() catches it in a bare
`except ImportError` and logs only a generic message, so print the
exception ourselves and run ldd on the .so -- that names the unresolved
shared object directly instead of leaving it to inference.
---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh        | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index c8ce925d4..2d22d6b02 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -193,6 +193,24 @@ case "${KV_OFFLOAD_BACKEND:-}" in
         python3 -m pip show lmcache 2>/dev/null | grep -E "^(Version|Location):"
         python3 -c "import lmcache, glob, os; print('lmcache:', lmcache.__file__); print('c_ops so:', glob.glob(os.path.join(os.path.dirname(lmcache.__file__), 'c_ops*')))"
         ls -1 /usr/local/cuda*/lib64/libcudart.so* /usr/lib/x86_64-linux-gnu/libcudart.so* 2>/dev/null || true
+        # device_ops.ensure_native() swallows this in `except ImportError`, so
+        # surface the real dlopen error ourselves -- without it the failure is
+        # indistinguishable from a missing file.
+        python3 - <<'PYEOF' || true
+import traceback
+try:
+    import lmcache.c_ops
+    print("c_ops import: OK")
+except BaseException as exc:
+    print("c_ops import FAILED:", type(exc).__name__, exc)
+    traceback.print_exc()
+PYEOF
+        # ldd names the missing/unresolved shared object directly.
+        LMCACHE_SO=$(python3 -c "import lmcache, glob, os; print((glob.glob(os.path.join(os.path.dirname(lmcache.__file__), 'c_ops*.so')) or [''])[0])")
+        if [ -n "$LMCACHE_SO" ]; then
+            echo "ldd $LMCACHE_SO"
+            ldd "$LMCACHE_SO" 2>&1 | grep -E "not found|libcudart|libtorch|libc10" || true
+        fi
         python3 -c \
             "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
             >/dev/null

From cb45449db2c006040b65cf9ee52ad7a3a74868f8 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 14:25:17 -0700
Subject: [PATCH 10/18] kimik3 b300 lmcache: put torch's lib dir on
 LD_LIBRARY_PATH for c_ops

The diagnostic run pinned it down. c_ops.so links libtorch/libc10 but they
are not on the default loader path -- ldd reports libc10.so, libtorch.so,
libtorch_cpu.so, libtorch_python.so, libc10_cuda.so and libtorch_cuda.so
as "not found", while libcudart resolves fine. So this was never a CUDA
version mismatch.

Normally the unresolved torch libs are harmless, because `import torch`
loads them RTLD_GLOBAL before any extension needs them. But
CudaDeviceOps.ensure_native() sets `self._native_bound = True` BEFORE
attempting `import lmcache.c_ops`, so a single failure early in process
startup -- before torch is loaded -- permanently disables native ops and
pins the server to the torch fallback. The same import succeeds moments
later, which is why the standalone check printed "c_ops import: OK" in
the very run whose server logged "compiled extension not found".

Export torch's lib dir so the extension resolves regardless of import
order.
---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh           | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 2d22d6b02..618b7bc55 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -205,6 +205,21 @@ except BaseException as exc:
     print("c_ops import FAILED:", type(exc).__name__, exc)
     traceback.print_exc()
 PYEOF
+        # c_ops.so links libtorch/libc10 but they are not on the default
+        # loader path -- `ldd` reports libc10.so, libtorch.so, libtorch_cpu.so,
+        # libtorch_python.so, libc10_cuda.so and libtorch_cuda.so as "not
+        # found" (libcudart resolves fine). Normally that is harmless because
+        # `import torch` loads them RTLD_GLOBAL first, but
+        # CudaDeviceOps.ensure_native() sets `self._native_bound = True` BEFORE
+        # its `import lmcache.c_ops`, so one early failure -- before torch is
+        # in the process -- permanently disables native ops and silently pins
+        # the whole server to the broken torch fallback. Putting torch's lib
+        # dir on LD_LIBRARY_PATH makes the extension loadable regardless of
+        # import order.
+        TORCH_LIB_DIR=$(python3 -c "import os, torch; print(os.path.join(os.path.dirname(torch.__file__), 'lib'))")
+        export LD_LIBRARY_PATH="${TORCH_LIB_DIR}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+        echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
+
         # ldd names the missing/unresolved shared object directly.
         LMCACHE_SO=$(python3 -c "import lmcache, glob, os; print((glob.glob(os.path.join(os.path.dirname(lmcache.__file__), 'c_ops*.so')) or [''])[0])")
         if [ -n "$LMCACHE_SO" ]; then

From 51d2951a4dc9b7e0d1f96046458e078c59aa31c0 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 14:55:12 -0700
Subject: [PATCH 11/18] kimik3 b300 lmcache: install plain from PyPI; probe
 c_ops import order

The GitHub release expanded_assets URLs were never doing anything: pip
resolved the identical PyPI wheel
(lmcache-0.5.4rc2-cp312-cp312-manylinux_2_27_x86_64...whl, 15,550,150
bytes) in every run, whether --find-links pointed at the -cu129 assets or
the default ones. Drop the indirection and just pin the version.

--force-reinstall stays: the image ships 0.5.3.

Also flip the diagnostic to import torch BEFORE lmcache. ensure_native()
latches _native_bound=True on its first attempt and swallows the error, so
the open question is whether c_ops fails because torch is not yet loaded
(import order) or because of a symbol/ABI mismatch. Preloading torch
distinguishes the two in one run.
---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh       | 47 ++++++++++---------
 1 file changed, 26 insertions(+), 21 deletions(-)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 618b7bc55..c794a568c 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -159,23 +159,24 @@ case "${KV_OFFLOAD_BACKEND:-}" in
     lmcache)
         require_agentic_kv_offload_backend lmcache
 
-        # The LMCache wheel MUST match the image's CUDA major version. Its
-        # compiled `lmcache.c_ops` extension is linked against a specific
-        # libcudart, and LMCache does NOT fail when that .so cannot be
-        # dlopen'd -- it logs "lmcache.c_ops compiled extension not found;
-        # CudaDeviceOps stays on the torch baseline for all ops" and silently
-        # falls back to the pure-torch path in lmcache/v1/platform/torch_ops.py.
-        # That fallback is broken for this stack's hybrid multi-KV-group
-        # KDA/MLA layout: 77-98% of stores die with cudaErrorInvalidValue in
-        # index_select/cudaMemcpy, so the offload tier stays empty and the arm
-        # silently measures a cache that never stored anything.
+        # Plain PyPI. The GitHub release `expanded_assets` URLs were never
+        # doing anything here -- pip resolved the identical PyPI wheel
+        # (lmcache-0.5.4rc2-cp312-cp312-manylinux_2_27_x86_64...whl, 15.6 MB)
+        # in every run, whether --find-links pointed at the -cu129 assets or
+        # the default ones.
         #
-        # This node reports CUDA 13.0, so use the DEFAULT release assets, whose
-        # c_ops links libcudart.so.13 (cuda_13.0). Do NOT use the `-cu129`
-        # asset set: its c_ops links libcudart.so.12 (cuda_12.9) and cannot
-        # load here. The `-rocm` assets are the MI355X sister arm's equivalent.
-        # If the image's CUDA major version ever changes, this URL must change
-        # with it -- verify with `nvidia-smi | grep "CUDA Version"`.
+        # --force-reinstall IS required: this image ships lmcache 0.5.3, so a
+        # plain install of 0.5.4rc2 upgrades it, but without the flag an
+        # already-matching version would be left in place silently.
+        #
+        # Why any of this matters: LMCache does not fail when its compiled
+        # `lmcache.c_ops` extension cannot be loaded. It logs "compiled
+        # extension not found; CudaDeviceOps stays on the torch baseline" and
+        # silently falls back to lmcache/v1/platform/torch_ops.py, which is
+        # broken for this stack's hybrid multi-KV-group KDA/MLA layout: 77-98%
+        # of stores die with cudaErrorInvalidValue, so the offload tier stays
+        # empty and the arm measures a cache that never stored anything.
+        # The guard after server start is what makes that loud.
         # --force-reinstall is REQUIRED, not cosmetic: this image already ships
         # lmcache 0.5.4rc2, so a plain `pip install lmcache==0.5.4rc2` finds the
         # requirement already satisfied and installs nothing -- the --find-links
@@ -184,10 +185,8 @@ case "${KV_OFFLOAD_BACKEND:-}" in
         # identical build, `LMCache v0.5.4rc2 (gf82f6fd3)`, in ~3 seconds. We
         # were always running the image's copy, whose c_ops does not load here.
         LMCACHE_VERSION="0.5.4rc2"
-        LMCACHE_CUDA_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}"
         agentic_pip_install --no-cache-dir --force-reinstall --no-deps \
-            "lmcache==${LMCACHE_VERSION}" \
-            --find-links "$LMCACHE_CUDA_INDEX"
+            "lmcache==${LMCACHE_VERSION}"
         # Record what actually landed. Silent installs are how the no-op above
         # went unnoticed for two full sweeps; do not add --quiet back.
         python3 -m pip show lmcache 2>/dev/null | grep -E "^(Version|Location):"
@@ -197,12 +196,18 @@ case "${KV_OFFLOAD_BACKEND:-}" in
         # surface the real dlopen error ourselves -- without it the failure is
         # indistinguishable from a missing file.
         python3 - <<'PYEOF' || true
+# ensure_native() swallows the real error and latches _native_bound=True on
+# the first attempt, so the open question is whether the failure is import
+# ORDER (torch not yet loaded) or a symbol/ABI mismatch. Import torch first:
+# if the warning disappears, it is ordering; if it persists, it is the build.
 import traceback
+import torch  # noqa: F401  -- deliberately first
+print("torch:", torch.__version__)
 try:
     import lmcache.c_ops
-    print("c_ops import: OK")
+    print("c_ops import (torch preloaded): OK")
 except BaseException as exc:
-    print("c_ops import FAILED:", type(exc).__name__, exc)
+    print("c_ops import (torch preloaded) FAILED:", type(exc).__name__, exc)
     traceback.print_exc()
 PYEOF
         # c_ops.so links libtorch/libc10 but they are not on the default

From f4410710a032c87202b1a20275a9f6ad7a23396c Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 15:02:35 -0700
Subject: [PATCH 12/18] kimik3 b300 lmcache: launch the MP server with torch
 preloaded

The probe run settled what c_ops actually needs. With torch imported first
the extension loads cleanly -- 'c_ops import (torch preloaded): OK' -- in
the very same job whose lmcache server still logged 'compiled extension
not found'. So it is not a CUDA version mismatch, not a missing libcudart,
not unresolved torch libs, and not the wheel source: all four were ruled
out by earlier runs. torch here is 2.13.0+cu130 and the PyPI wheel matches.

What is left is ordering. CudaDeviceOps.ensure_native() sets
_native_bound = True BEFORE its 'import lmcache.c_ops', so a single early
failure -- before torch is in the process -- permanently pins the server
to the torch fallback, which is broken for this stack's hybrid
multi-KV-group KDA/MLA layout (77-98% of stores fail).

Launch the server through python with torch imported first rather than the
bare console script. This is the one part of the sequence we control.
---
 .../single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh  | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 3e0b5cb08..6efac02b6 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -260,8 +260,18 @@ PYEOF
         # size of the node's /dev/shm mount.
         LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB"
 
+        # Launch via python with torch imported FIRST, instead of the bare
+        # `lmcache` console script. CudaDeviceOps.ensure_native() sets
+        # `self._native_bound = True` BEFORE attempting `import lmcache.c_ops`,
+        # so if that first attempt happens while torch is not yet loaded the
+        # extension is written off permanently and the server silently runs the
+        # broken torch fallback for its whole life. The same import succeeds
+        # once torch is up -- this run proved it ("c_ops import (torch
+        # preloaded): OK") in the very process whose server logged "compiled
+        # extension not found". Preloading torch removes the ordering hazard.
         LMCACHE_CMD=(
-            lmcache server
+            python3 -c "import sys, torch; from lmcache.cli.main import main; sys.argv[0]='lmcache'; main()"
+            server
             --host 127.0.0.1
             --port "$LMCACHE_PORT"
             --http-host 127.0.0.1

From 273ab1303cd99ddd7c79bdcf128f391cd18747f2 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 15:10:15 -0700
Subject: [PATCH 13/18] kimik3 b300 lmcache: instrument c_ops to capture the
 swallowed error

Preloading torch in the server process did NOT help -- the server launched
as `python3 -c 'import sys, torch; ...'` still logged 'compiled extension
not found'. So import order is ruled out, alongside CUDA version, wheel
source, missing torch libs and ABI (the extension imports fine on demand).

Replace the guesswork with instrumentation that captures both halves:
wrap builtins.__import__ to record the exception ensure_native() swallows
in its bare `except ImportError`, and hook the logger to dump the caller
stack at the moment the warning is emitted. Also report whether torch is
already in sys.modules, whether _native_bound is latched, and whether a
direct import succeeds at the same point.

This names WHAT fails and WHO triggered it in a single ~60s job instead of
another hypothesis.
---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh       | 78 ++++++++++++++++---
 1 file changed, 67 insertions(+), 11 deletions(-)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 6efac02b6..572e39672 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -196,19 +196,75 @@ case "${KV_OFFLOAD_BACKEND:-}" in
         # surface the real dlopen error ourselves -- without it the failure is
         # indistinguishable from a missing file.
         python3 - <<'PYEOF' || true
-# ensure_native() swallows the real error and latches _native_bound=True on
-# the first attempt, so the open question is whether the failure is import
-# ORDER (torch not yet loaded) or a symbol/ABI mismatch. Import torch first:
-# if the warning disappears, it is ordering; if it persists, it is the build.
+# Definitive c_ops instrumentation.
+#
+# ensure_native() swallows the real exception in `except ImportError` and
+# latches _native_bound=True on the first attempt, so neither the error nor the
+# caller is ever visible. Wrap __import__ to capture the exception, and hook the
+# logger to capture the stack at the moment the warning is emitted -- together
+# these name both WHAT fails and WHO triggered it.
+import builtins
+import logging
+import sys
 import traceback
-import torch  # noqa: F401  -- deliberately first
-print("torch:", torch.__version__)
+
+_real_import = builtins.__import__
+_failures = []
+
+
+def _tracing_import(name, globals=None, locals=None, fromlist=(), level=0):
+    try:
+        return _real_import(name, globals, locals, fromlist, level)
+    except BaseException as exc:  # noqa: BLE001 - diagnostic
+        if "c_ops" in name:
+            _failures.append((name, exc))
+            print(f"[probe] IMPORT FAILED: {name}: {type(exc).__name__}: {exc}", flush=True)
+            traceback.print_exc()
+        raise
+
+
+class _WarnHook(logging.Handler):
+    def emit(self, record):
+        try:
+            msg = record.getMessage()
+        except Exception:  # noqa: BLE001
+            return
+        if "c_ops compiled extension not found" in msg:
+            print("[probe] ensure_native() gave up -- caller stack:", flush=True)
+            traceback.print_stack()
+
+
+builtins.__import__ = _tracing_import
+logging.getLogger().addHandler(_WarnHook())
+logging.getLogger().setLevel(logging.DEBUG)
+
+print("[probe] torch loaded before lmcache?", "torch" in sys.modules, flush=True)
+
+# Import exactly what the server imports, in the server's order.
+import lmcache.integration.vllm.lmcache_mp_connector  # noqa: E402,F401
+
+print("[probe] torch in sys.modules now:", "torch" in sys.modules, flush=True)
+print("[probe] c_ops in sys.modules:", "lmcache.c_ops" in sys.modules, flush=True)
+print(f"[probe] captured c_ops import failures: {len(_failures)}", flush=True)
+for name, exc in _failures:
+    print(f"[probe]   {name}: {type(exc).__name__}: {exc}", flush=True)
+
+# Now show whether the singleton is latched off despite the extension being loadable.
 try:
-    import lmcache.c_ops
-    print("c_ops import (torch preloaded): OK")
-except BaseException as exc:
-    print("c_ops import (torch preloaded) FAILED:", type(exc).__name__, exc)
-    traceback.print_exc()
+    from lmcache.v1.platform.cuda.device_ops import CudaDeviceOps
+
+    ops = CudaDeviceOps()
+    print("[probe] CudaDeviceOps._native_bound =", getattr(ops, "_native_bound", "?"), flush=True)
+except BaseException as exc:  # noqa: BLE001
+    print("[probe] could not inspect CudaDeviceOps:", type(exc).__name__, exc, flush=True)
+
+# And whether a direct import works at this point.
+try:
+    import lmcache.c_ops  # noqa: F401
+
+    print("[probe] direct import lmcache.c_ops AFTER: OK", flush=True)
+except BaseException as exc:  # noqa: BLE001
+    print("[probe] direct import lmcache.c_ops AFTER FAILED:", type(exc).__name__, exc, flush=True)
 PYEOF
         # c_ops.so links libtorch/libc10 but they are not on the default
         # loader path -- `ldd` reports libc10.so, libtorch.so, libtorch_cpu.so,

From 7a31be3786a0e4b7c2087d42118493e4ed15ce27 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 15:17:22 -0700
Subject: [PATCH 14/18] kimik3 b300 lmcache: LD_PRELOAD torch libs so c_ops
 resolves c10 symbols

Root cause, captured by instrumenting the exception ensure_native()
swallows:

  ImportError: .../lmcache/c_ops.cpython-312-x86_64-linux-gnu.so:
  undefined symbol: _ZN3c104impl3cow23materialize_cow_storageERNS_11StorageImplE
    ( c10::impl::cow::materialize_cow_storage(c10::StorageImpl&) )

It is not an ABI mismatch: the probe shows the identical import succeeding
moments later once torch is in the process ('direct import lmcache.c_ops
AFTER: OK'), so the symbol does exist in this image's torch 2.13.0+cu130.
c_ops.so simply needs libc10/libtorch loaded RTLD_GLOBAL first.

What makes a transient miss permanent is that
CudaDeviceOps.ensure_native() sets _native_bound = True BEFORE attempting
the import and swallows the error, so the process is pinned to the
pure-torch fallback -- which is broken for this stack's multi-KV-group
KDA/MLA layout and fails 77-98% of stores.

Preloading torch in the parent was already tried and is not sufficient:
the server forks CPU/GPU workers that import lmcache fresh. LD_PRELOAD
covers the whole process tree. Scoped via `env` to the lmcache command so
the vLLM server's environment is untouched.
---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh       | 38 +++++++++++++------
 1 file changed, 27 insertions(+), 11 deletions(-)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 572e39672..3b2e1a4df 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -316,18 +316,34 @@ PYEOF
         # size of the node's /dev/shm mount.
         LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB"
 
-        # Launch via python with torch imported FIRST, instead of the bare
-        # `lmcache` console script. CudaDeviceOps.ensure_native() sets
-        # `self._native_bound = True` BEFORE attempting `import lmcache.c_ops`,
-        # so if that first attempt happens while torch is not yet loaded the
-        # extension is written off permanently and the server silently runs the
-        # broken torch fallback for its whole life. The same import succeeds
-        # once torch is up -- this run proved it ("c_ops import (torch
-        # preloaded): OK") in the very process whose server logged "compiled
-        # extension not found". Preloading torch removes the ordering hazard.
+        # c_ops.so resolves c10/torch symbols only once libtorch is loaded
+        # RTLD_GLOBAL. Captured directly:
+        #
+        #   ImportError: .../lmcache/c_ops.cpython-312-x86_64-linux-gnu.so:
+        #   undefined symbol: _ZN3c104impl3cow23materialize_cow_storageERNS_11StorageImplE
+        #     ( c10::impl::cow::materialize_cow_storage(c10::StorageImpl&) )
+        #
+        # It is NOT an ABI mismatch -- the same import succeeds moments later,
+        # once torch is in the process, so the symbol does exist in this
+        # image's torch. But CudaDeviceOps.ensure_native() latches
+        # _native_bound = True on its FIRST attempt and swallows the error, so
+        # one early miss pins the process to the torch fallback for good, and
+        # that fallback is broken for this stack's multi-KV-group KDA/MLA
+        # layout (77-98% of stores fail).
+        #
+        # Preloading torch in the parent is not enough: the server forks CPU
+        # and GPU workers that import lmcache fresh. LD_PRELOAD applies to the
+        # whole process tree, so the symbols are resolvable everywhere. Scoped
+        # to this command via `env` rather than exported, to leave the vLLM
+        # server's environment untouched.
+        LMCACHE_LD_PRELOAD="${TORCH_LIB_DIR}/libc10.so:${TORCH_LIB_DIR}/libtorch_cpu.so:${TORCH_LIB_DIR}/libtorch.so"
+        if [ -f "${TORCH_LIB_DIR}/libc10_cuda.so" ]; then
+            LMCACHE_LD_PRELOAD="${LMCACHE_LD_PRELOAD}:${TORCH_LIB_DIR}/libc10_cuda.so:${TORCH_LIB_DIR}/libtorch_cuda.so"
+        fi
+        echo "LMCACHE_LD_PRELOAD=$LMCACHE_LD_PRELOAD"
         LMCACHE_CMD=(
-            python3 -c "import sys, torch; from lmcache.cli.main import main; sys.argv[0]='lmcache'; main()"
-            server
+            env "LD_PRELOAD=$LMCACHE_LD_PRELOAD"
+            lmcache server
             --host 127.0.0.1
             --port "$LMCACHE_PORT"
             --http-host 127.0.0.1

From 9fdd6de8c604bf2455ebbc564c4d8f7ef525d2de Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 15:24:30 -0700
Subject: [PATCH 15/18] kimik3 b300 lmcache: locate which torch lib defines
 materialize_cow_storage

LD_PRELOAD of libc10/libtorch_cpu/libtorch was accepted (no ld.so errors)
but the server still fell back, so none of those three provides
c10::impl::cow::materialize_cow_storage. Scan every torch library with nm
for the definition instead of guessing which one to preload, and confirm
c_ops lists it as undefined.
---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh            | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 3b2e1a4df..21f22f428 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -287,6 +287,20 @@ PYEOF
             echo "ldd $LMCACHE_SO"
             ldd "$LMCACHE_SO" 2>&1 | grep -E "not found|libcudart|libtorch|libc10" || true
         fi
+        # Which torch library actually EXPORTS the symbol c_ops needs?
+        # LD_PRELOAD of libc10/libtorch_cpu/libtorch did not resolve it, so
+        # locate the definition rather than guessing which .so to preload.
+        echo "=== searching torch libs for materialize_cow_storage ==="
+        for _so in "$TORCH_LIB_DIR"/libc10.so "$TORCH_LIB_DIR"/libc10_cuda.so \
+                   "$TORCH_LIB_DIR"/libtorch.so "$TORCH_LIB_DIR"/libtorch_cpu.so \
+                   "$TORCH_LIB_DIR"/libtorch_cuda.so "$TORCH_LIB_DIR"/libtorch_python.so; do
+            [ -f "$_so" ] || continue
+            _n=$(nm -D --defined-only "$_so" 2>/dev/null | grep -c "materialize_cow_storage" || true)
+            _u=$(nm -D --undefined-only "$_so" 2>/dev/null | grep -c "materialize_cow_storage" || true)
+            echo "  $(basename "$_so"): defined=$_n undefined=$_u"
+        done
+        echo "  (c_ops needs it) $(nm -D --undefined-only "$LMCACHE_SO" 2>/dev/null | grep -c materialize_cow_storage || true)"
+
         python3 -c \
             "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
             >/dev/null

From 0e29d1798054c3e916472b311dd8e69276c063a4 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 15:32:51 -0700
Subject: [PATCH 16/18] kimik3 b300 lmcache: check the image's own lmcache
 before overwriting it

nm settles it: no torch library in this image defines
c10::impl::cow::materialize_cow_storage --

  libc10.so / libc10_cuda.so / libtorch.so / libtorch_cpu.so /
  libtorch_cuda.so / libtorch_python.so  ->  defined=0
  c_ops.so                               ->  needs it (1)

so the 0.5.4rc2 wheel is a genuine ABI mismatch against this image's torch
2.13.0+cu130. No ordering trick, LD_PRELOAD or LD_LIBRARY_PATH can supply
a symbol that does not exist. (My earlier 'not an ABI mismatch' read was
wrong: the retry that printed 'OK' was a sys.modules cache hit, not a
fresh symbol resolution.)

The image ships lmcache 0.5.3, presumably built against its own torch. Log
its version, whether its c_ops needs the same symbol, and whether it
imports -- before --force-reinstall replaces it. If 0.5.3 loads, pinning
0.5.4rc2 on this image is simply not viable and the config should say so.
---
 .../agentic/kimik3_fp4_b300_vllm_mtp.sh           | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 21f22f428..6a283a600 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -184,6 +184,21 @@ case "${KV_OFFLOAD_BACKEND:-}" in
         # the -cu129 assets and one to the default assets both produced the
         # identical build, `LMCache v0.5.4rc2 (gf82f6fd3)`, in ~3 seconds. We
         # were always running the image's copy, whose c_ops does not load here.
+        # Before overwriting it: does the image's OWN lmcache work? The
+        # 0.5.4rc2 wheel's c_ops needs c10::impl::cow::materialize_cow_storage,
+        # which nm shows NO torch library in this image defines -- a real ABI
+        # mismatch, not an ordering problem. The image ships 0.5.3, presumably
+        # built against this torch, so establish whether it loads before we
+        # replace it.
+        echo "=== image's pre-existing lmcache ==="
+        python3 -m pip show lmcache 2>/dev/null | grep -E "^Version:" || echo "  (not installed)"
+        _IMG_SO=$(python3 -c "import lmcache, glob, os; print((glob.glob(os.path.join(os.path.dirname(lmcache.__file__), 'c_ops*.so')) or [''])[0])" 2>/dev/null || true)
+        if [ -n "$_IMG_SO" ]; then
+            echo "  c_ops: $_IMG_SO"
+            echo "  needs materialize_cow_storage: $(nm -D --undefined-only "$_IMG_SO" 2>/dev/null | grep -c materialize_cow_storage || true)"
+            python3 -c "import lmcache.c_ops; print('  IMAGE c_ops import: OK')" 2>&1 | tail -2 || echo "  IMAGE c_ops import: FAILED"
+        fi
+
         LMCACHE_VERSION="0.5.4rc2"
         agentic_pip_install --no-cache-dir --force-reinstall --no-deps \
             "lmcache==${LMCACHE_VERSION}"

From 32f721d23bbacb099a526319d721efa406ea4870 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 15:55:38 -0700
Subject: [PATCH 17/18] trigger sweep: probe image-bundled lmcache c_ops


From 3044c26bce79f9f8de71be96d3032bd889d8a7c0 Mon Sep 17 00:00:00 2001
From: Samuel Shen <slshen@tensormesh.ai>
Date: Fri, 14 Aug 2026 16:09:31 -0700
Subject: [PATCH 18/18] kimik3 b300 lmcache: characterise the torch/c10 COW ABI
 gap

The wheel embeds no torch version, so which torch it was built against has
to be inferred from symbols. c_ops requires two c10 COW entry points:

  c10::impl::cow::is_cow_data_ptr(c10::DataPtr const&)
  c10::impl::cow::materialize_cow_storage(c10::StorageImpl&)

Dump torch.__version__/torch.version.cuda plus every c10::impl::cow symbol
libc10.so actually exports, alongside the ones c_ops needs. That shows
whether this torch dropped COW entirely or changed the signatures, which
is the difference between 'lmcache needs a rebuild' and 'lmcache needs an
older torch'.
---
 .../single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh  | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 6a283a600..9a987363b 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -305,6 +305,18 @@ PYEOF
         # Which torch library actually EXPORTS the symbol c_ops needs?
         # LD_PRELOAD of libc10/libtorch_cpu/libtorch did not resolve it, so
         # locate the definition rather than guessing which .so to preload.
+        # Which torch is this, and what does its c10 actually export under
+        # c10::impl::cow? The wheel embeds no torch version, so the mismatch
+        # has to be characterised from the symbols. c_ops needs BOTH
+        # c10::impl::cow::is_cow_data_ptr and ::materialize_cow_storage.
+        python3 -c "import torch; print('  torch:', torch.__version__, '| built with CUDA', torch.version.cuda)"
+        echo "  --- all c10::impl::cow symbols exported by libc10.so ---"
+        nm -D --defined-only "$TORCH_LIB_DIR/libc10.so" 2>/dev/null | grep -oE "_ZN3c104impl3cow[A-Za-z0-9_]*" | sort -u | head -20 || true
+        echo "  --- (count) ---"
+        nm -D --defined-only "$TORCH_LIB_DIR/libc10.so" 2>/dev/null | grep -c "3cow" || true
+        echo "  --- what c_ops requires ---"
+        nm -D --undefined-only "$LMCACHE_SO" 2>/dev/null | grep -oE "_ZN3c104impl3cow[A-Za-z0-9_]*" | sort -u || true
+
         echo "=== searching torch libs for materialize_cow_storage ==="
         for _so in "$TORCH_LIB_DIR"/libc10.so "$TORCH_LIB_DIR"/libc10_cuda.so \
                    "$TORCH_LIB_DIR"/libtorch.so "$TORCH_LIB_DIR"/libtorch_cpu.so \