kimik3-fp4-b300-vllm-agentic-dspark: add LMCache DRAM KV-offload arm

benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh

@@ -120,6 +120,20 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
 SERVER_LOG="$RESULT_DIR/server.log"
 mkdir -p "$RESULT_DIR"
 
+# The LMCache arm starts a second long-lived process that must not outlive this
+# job (its L1 holds the whole host-DRAM budget). The vLLM server's lifecycle is
+# left exactly as it was -- the job wrapper still owns it.
+LMCACHE_PID=""
+
+cleanup_lmcache_server() {
+    local exit_code=$?
+    trap - EXIT
+    set +e
+    stop_background_process_tree "$LMCACHE_PID" "LMCache server"
+    exit "$exit_code"
+}
+trap cleanup_lmcache_server EXIT
+
 # ---- KV offloading ----------------------------------------------------------
 # The generated TOTAL_CPU_DRAM_GB budget is the aggregate host-DRAM pool for the
 # node; SimpleCPUOffloadConnector is sized per rank. At dram-utilization 0.63 on
@@ -142,8 +156,283 @@ case "${KV_OFFLOAD_BACKEND:-}" in
             "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":${CPU_BYTES_PER_RANK},\"lazy_offload\":false}}"
         )
         ;;
+    lmcache)
+        require_agentic_kv_offload_backend lmcache
+
+        # Plain PyPI. The GitHub release `expanded_assets` URLs were never
+        # doing anything here -- pip resolved the identical PyPI wheel
+        # (lmcache-0.5.4rc2-cp312-cp312-manylinux_2_27_x86_64...whl, 15.6 MB)
+        # in every run, whether --find-links pointed at the -cu129 assets or
+        # the default ones.
+        #
+        # --force-reinstall IS required: this image ships lmcache 0.5.3, so a
+        # plain install of 0.5.4rc2 upgrades it, but without the flag an
+        # already-matching version would be left in place silently.
+        #
+        # Why any of this matters: LMCache does not fail when its compiled
+        # `lmcache.c_ops` extension cannot be loaded. It logs "compiled
+        # extension not found; CudaDeviceOps stays on the torch baseline" and
+        # silently falls back to lmcache/v1/platform/torch_ops.py, which is
+        # broken for this stack's hybrid multi-KV-group KDA/MLA layout: 77-98%
+        # of stores die with cudaErrorInvalidValue, so the offload tier stays
+        # empty and the arm measures a cache that never stored anything.
+        # The guard after server start is what makes that loud.
+        # --force-reinstall is REQUIRED, not cosmetic: this image already ships
+        # lmcache 0.5.4rc2, so a plain `pip install lmcache==0.5.4rc2` finds the
+        # requirement already satisfied and installs nothing -- the --find-links
+        # URL is never consulted. Two earlier sweeps proved it: one pinned to
+        # the -cu129 assets and one to the default assets both produced the
+        # identical build, `LMCache v0.5.4rc2 (gf82f6fd3)`, in ~3 seconds. We
+        # were always running the image's copy, whose c_ops does not load here.
+        # Before overwriting it: does the image's OWN lmcache work? The
+        # 0.5.4rc2 wheel's c_ops needs c10::impl::cow::materialize_cow_storage,
+        # which nm shows NO torch library in this image defines -- a real ABI
+        # mismatch, not an ordering problem. The image ships 0.5.3, presumably
+        # built against this torch, so establish whether it loads before we
+        # replace it.
+        echo "=== image's pre-existing lmcache ==="
+        python3 -m pip show lmcache 2>/dev/null | grep -E "^Version:" || echo "  (not installed)"
+        _IMG_SO=$(python3 -c "import lmcache, glob, os; print((glob.glob(os.path.join(os.path.dirname(lmcache.__file__), 'c_ops*.so')) or [''])[0])" 2>/dev/null || true)
+        if [ -n "$_IMG_SO" ]; then
+            echo "  c_ops: $_IMG_SO"
+            echo "  needs materialize_cow_storage: $(nm -D --undefined-only "$_IMG_SO" 2>/dev/null | grep -c materialize_cow_storage || true)"
+            python3 -c "import lmcache.c_ops; print('  IMAGE c_ops import: OK')" 2>&1 | tail -2 || echo "  IMAGE c_ops import: FAILED"
+        fi
+
+        LMCACHE_VERSION="0.5.4rc2"
+        agentic_pip_install --no-cache-dir --force-reinstall --no-deps \
+            "lmcache==${LMCACHE_VERSION}"
+        # Record what actually landed. Silent installs are how the no-op above
+        # went unnoticed for two full sweeps; do not add --quiet back.
+        python3 -m pip show lmcache 2>/dev/null | grep -E "^(Version|Location):"
+        python3 -c "import lmcache, glob, os; print('lmcache:', lmcache.__file__); print('c_ops so:', glob.glob(os.path.join(os.path.dirname(lmcache.__file__), 'c_ops*')))"
+        ls -1 /usr/local/cuda*/lib64/libcudart.so* /usr/lib/x86_64-linux-gnu/libcudart.so* 2>/dev/null || true
+        # device_ops.ensure_native() swallows this in `except ImportError`, so
+        # surface the real dlopen error ourselves -- without it the failure is
+        # indistinguishable from a missing file.
+        python3 - <<'PYEOF' || true
+# Definitive c_ops instrumentation.
+#
+# ensure_native() swallows the real exception in `except ImportError` and
+# latches _native_bound=True on the first attempt, so neither the error nor the
+# caller is ever visible. Wrap __import__ to capture the exception, and hook the
+# logger to capture the stack at the moment the warning is emitted -- together
+# these name both WHAT fails and WHO triggered it.
+import builtins
+import logging
+import sys
+import traceback
+
+_real_import = builtins.__import__
+_failures = []
+
+
+def _tracing_import(name, globals=None, locals=None, fromlist=(), level=0):
+    try:
+        return _real_import(name, globals, locals, fromlist, level)
+    except BaseException as exc:  # noqa: BLE001 - diagnostic
+        if "c_ops" in name:
+            _failures.append((name, exc))
+            print(f"[probe] IMPORT FAILED: {name}: {type(exc).__name__}: {exc}", flush=True)
+            traceback.print_exc()
+        raise
+
+
+class _WarnHook(logging.Handler):
+    def emit(self, record):
+        try:
+            msg = record.getMessage()
+        except Exception:  # noqa: BLE001
+            return
+        if "c_ops compiled extension not found" in msg:
+            print("[probe] ensure_native() gave up -- caller stack:", flush=True)
+            traceback.print_stack()
+
+
+builtins.__import__ = _tracing_import
+logging.getLogger().addHandler(_WarnHook())
+logging.getLogger().setLevel(logging.DEBUG)
+
+print("[probe] torch loaded before lmcache?", "torch" in sys.modules, flush=True)
+
+# Import exactly what the server imports, in the server's order.
+import lmcache.integration.vllm.lmcache_mp_connector  # noqa: E402,F401
+
+print("[probe] torch in sys.modules now:", "torch" in sys.modules, flush=True)
+print("[probe] c_ops in sys.modules:", "lmcache.c_ops" in sys.modules, flush=True)
+print(f"[probe] captured c_ops import failures: {len(_failures)}", flush=True)
+for name, exc in _failures:
+    print(f"[probe]   {name}: {type(exc).__name__}: {exc}", flush=True)
+
+# Now show whether the singleton is latched off despite the extension being loadable.
+try:
+    from lmcache.v1.platform.cuda.device_ops import CudaDeviceOps
+
+    ops = CudaDeviceOps()
+    print("[probe] CudaDeviceOps._native_bound =", getattr(ops, "_native_bound", "?"), flush=True)
+except BaseException as exc:  # noqa: BLE001
+    print("[probe] could not inspect CudaDeviceOps:", type(exc).__name__, exc, flush=True)
+
+# And whether a direct import works at this point.
+try:
+    import lmcache.c_ops  # noqa: F401
+
+    print("[probe] direct import lmcache.c_ops AFTER: OK", flush=True)
+except BaseException as exc:  # noqa: BLE001
+    print("[probe] direct import lmcache.c_ops AFTER FAILED:", type(exc).__name__, exc, flush=True)
+PYEOF
+        # c_ops.so links libtorch/libc10 but they are not on the default
+        # loader path -- `ldd` reports libc10.so, libtorch.so, libtorch_cpu.so,
+        # libtorch_python.so, libc10_cuda.so and libtorch_cuda.so as "not
+        # found" (libcudart resolves fine). Normally that is harmless because
+        # `import torch` loads them RTLD_GLOBAL first, but
+        # CudaDeviceOps.ensure_native() sets `self._native_bound = True` BEFORE
+        # its `import lmcache.c_ops`, so one early failure -- before torch is
+        # in the process -- permanently disables native ops and silently pins
+        # the whole server to the broken torch fallback. Putting torch's lib
+        # dir on LD_LIBRARY_PATH makes the extension loadable regardless of
+        # import order.
+        TORCH_LIB_DIR=$(python3 -c "import os, torch; print(os.path.join(os.path.dirname(torch.__file__), 'lib'))")
+        export LD_LIBRARY_PATH="${TORCH_LIB_DIR}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+        echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
+
+        # ldd names the missing/unresolved shared object directly.
+        LMCACHE_SO=$(python3 -c "import lmcache, glob, os; print((glob.glob(os.path.join(os.path.dirname(lmcache.__file__), 'c_ops*.so')) or [''])[0])")
+        if [ -n "$LMCACHE_SO" ]; then
+            echo "ldd $LMCACHE_SO"
+            ldd "$LMCACHE_SO" 2>&1 | grep -E "not found|libcudart|libtorch|libc10" || true
+        fi
+        # Which torch library actually EXPORTS the symbol c_ops needs?
+        # LD_PRELOAD of libc10/libtorch_cpu/libtorch did not resolve it, so
+        # locate the definition rather than guessing which .so to preload.
+        # Which torch is this, and what does its c10 actually export under
+        # c10::impl::cow? The wheel embeds no torch version, so the mismatch
+        # has to be characterised from the symbols. c_ops needs BOTH
+        # c10::impl::cow::is_cow_data_ptr and ::materialize_cow_storage.
+        python3 -c "import torch; print('  torch:', torch.__version__, '| built with CUDA', torch.version.cuda)"
+        echo "  --- all c10::impl::cow symbols exported by libc10.so ---"
+        nm -D --defined-only "$TORCH_LIB_DIR/libc10.so" 2>/dev/null | grep -oE "_ZN3c104impl3cow[A-Za-z0-9_]*" | sort -u | head -20 || true
+        echo "  --- (count) ---"
+        nm -D --defined-only "$TORCH_LIB_DIR/libc10.so" 2>/dev/null | grep -c "3cow" || true
+        echo "  --- what c_ops requires ---"
+        nm -D --undefined-only "$LMCACHE_SO" 2>/dev/null | grep -oE "_ZN3c104impl3cow[A-Za-z0-9_]*" | sort -u || true
+
+        echo "=== searching torch libs for materialize_cow_storage ==="
+        for _so in "$TORCH_LIB_DIR"/libc10.so "$TORCH_LIB_DIR"/libc10_cuda.so \
+                   "$TORCH_LIB_DIR"/libtorch.so "$TORCH_LIB_DIR"/libtorch_cpu.so \
+                   "$TORCH_LIB_DIR"/libtorch_cuda.so "$TORCH_LIB_DIR"/libtorch_python.so; do
+            [ -f "$_so" ] || continue
+            _n=$(nm -D --defined-only "$_so" 2>/dev/null | grep -c "materialize_cow_storage" || true)
+            _u=$(nm -D --undefined-only "$_so" 2>/dev/null | grep -c "materialize_cow_storage" || true)
+            echo "  $(basename "$_so"): defined=$_n undefined=$_u"
+        done
+        echo "  (c_ops needs it) $(nm -D --undefined-only "$LMCACHE_SO" 2>/dev/null | grep -c materialize_cow_storage || true)"
+
+        python3 -c \
+            "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
+            >/dev/null
+
+        # One MP server for the node, per the Kimi-K3 recipe
+        # (docs.lmcache.ai/recipes/kimi_k3.html), but NOT that recipe's
+        # --chunk-size 768: the connector requires the chunk to be a multiple
+        # of every engine KV group's tokens_per_block, and this stack's blocks
+        # are far larger than 768. On this exact image and script, vLLM pins
+        # the attention group to 1536 (run 31404943911, interface.py:911,
+        # "Setting attention block size to 1536 tokens to ensure that
+        # attention page size is >= mamba page size") and pads the KDA/mamba
+        # page to match (interface.py:935). That alignment is generic vLLM
+        # code, not platform-specific, which is why the MI355X sister arm
+        # lands on the same 1536 plus a 3072-token KDA state group and also
+        # needs 3072. 768 is smaller than the attention block, so it is not a
+        # multiple of it and fails at connector init. The multi-group layout
+        # additionally requires one object group per sliding-window size:
+        # --separate-object-groups.
+        LMCACHE_PORT=6555
+        LMCACHE_HTTP_PORT=8090
+        LMCACHE_LOG="$RESULT_DIR/lmcache_server.log"
+
+        # Consume the generated aggregate budget verbatim, per
+        # benchmarks/single_node/agentic/README.md. --shm-name "" keeps L1 in
+        # ordinary process memory so the budget is not silently capped by the
+        # size of the node's /dev/shm mount.
+        LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB"
+
+        # c_ops.so resolves c10/torch symbols only once libtorch is loaded
+        # RTLD_GLOBAL. Captured directly:
+        #
+        #   ImportError: .../lmcache/c_ops.cpython-312-x86_64-linux-gnu.so:
+        #   undefined symbol: _ZN3c104impl3cow23materialize_cow_storageERNS_11StorageImplE
+        #     ( c10::impl::cow::materialize_cow_storage(c10::StorageImpl&) )
+        #
+        # It is NOT an ABI mismatch -- the same import succeeds moments later,
+        # once torch is in the process, so the symbol does exist in this
+        # image's torch. But CudaDeviceOps.ensure_native() latches
+        # _native_bound = True on its FIRST attempt and swallows the error, so
+        # one early miss pins the process to the torch fallback for good, and
+        # that fallback is broken for this stack's multi-KV-group KDA/MLA
+        # layout (77-98% of stores fail).
+        #
+        # Preloading torch in the parent is not enough: the server forks CPU
+        # and GPU workers that import lmcache fresh. LD_PRELOAD applies to the
+        # whole process tree, so the symbols are resolvable everywhere. Scoped
+        # to this command via `env` rather than exported, to leave the vLLM
+        # server's environment untouched.
+        LMCACHE_LD_PRELOAD="${TORCH_LIB_DIR}/libc10.so:${TORCH_LIB_DIR}/libtorch_cpu.so:${TORCH_LIB_DIR}/libtorch.so"
+        if [ -f "${TORCH_LIB_DIR}/libc10_cuda.so" ]; then
+            LMCACHE_LD_PRELOAD="${LMCACHE_LD_PRELOAD}:${TORCH_LIB_DIR}/libc10_cuda.so:${TORCH_LIB_DIR}/libtorch_cuda.so"
+        fi
+        echo "LMCACHE_LD_PRELOAD=$LMCACHE_LD_PRELOAD"
+        LMCACHE_CMD=(
+            env "LD_PRELOAD=$LMCACHE_LD_PRELOAD"
+            lmcache server
+            --host 127.0.0.1
+            --port "$LMCACHE_PORT"
+            --http-host 127.0.0.1
+            --http-port "$LMCACHE_HTTP_PORT"
+            --l1-size-gb "$LMCACHE_L1_SIZE_GB"
+            --l1-init-size-gb 10
+            --chunk-size 3072
+            --separate-object-groups
+            --enable-extra-logging
+            --extra-logging-interval 30
+            --max-cpu-workers 8
+            --max-gpu-workers 1
+            --eviction-policy LRU
+            --supported-transfer-mode lmcache_driven
+            --shm-name ""
+        )
+        append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}"
+        "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 &
+        LMCACHE_PID=$!
+        wait_for_ready \
+            --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \
+            --log "$LMCACHE_LOG" \
+            --pid "$LMCACHE_PID" \
+            --sleep-interval 1 \
+            --timeout 600
+
+        # Second guard, on the server process itself: the import check above
+        # only proves c_ops loads in this shell's python3. Abort rather than
+        # benchmark a silently-degraded offload tier (see the install note).
+        if grep -q "c_ops compiled extension not found" "$LMCACHE_LOG"; then
+            echo "Error: LMCache fell back to the torch baseline (c_ops did not load)." >&2
+            echo "       The installed lmcache's compiled extension is unusable here." >&2
+            echo "       Check the pip show / c_ops / libcudart lines logged above:" >&2
+            echo "       either the install did not take, or the build's CUDA ABI" >&2
+            echo "       does not match this image." >&2
+            grep -m1 "c_ops compiled extension not found" "$LMCACHE_LOG" >&2
+            exit 1
+        fi
+
+        # 100k-330k-token agentic prefixes make single retrieves large; use the
+        # same MQ timeout headroom as the MI355X arm.
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}"
+        )
+        ;;
     *)
-        echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or vllm-simple)" >&2
+        echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty, vllm-simple, or lmcache)" >&2
         exit 1
         ;;
 esac

configs/nvidia-master.yaml

@@ -1537,6 +1537,33 @@ kimik3-fp4-b300-vllm-agentic-dspark:
       # TP8 SimpleCPUOffload (host DRAM)
       - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16] }
 
+# LMCache MP-server DRAM offload on top of the same DSpark MTP serving stack as
+# kimik3-fp4-b300-vllm-agentic-dspark (same image, script, and topology). A
+# dedicated key so LMCache points can be selected and swept without re-running
+# the resident and vllm-simple arms of the base key.
+kimik3-fp4-b300-vllm-agentic-dspark-lmcache:
+  image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21
+  model: moonshotai/Kimi-K3
+  model-prefix: kimik3
+  runner: cluster:b300-nv
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    # Agentic-coding only, and 0.63 matches the base key so the LMCache L1 gets
+    # exactly the host-DRAM budget the SimpleCPUOffload arm gets (~1,889 GB
+    # aggregate at TP8). The LMCache server runs with --shm-name "" so that L1
+    # lives in ordinary process memory and is not capped by /dev/shm.
+    agentic-coding:
+    - dram-utilization: 0.63
+      search-space:
+      # 4/8/16 land on the base key's vllm-simple ladder at its top three
+      # offload points, so LMCache is directly comparable there; 10 fills the
+      # 8->16 gap, where the MI355X sister arm's ladder is densest. TP8-only
+      # for the same reason as the base key: a ~1.5 TB MXFP4 checkpoint does
+      # not fit below 8 GPUs.
+      - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.4rc2" }, conc-list: [4, 8, 10, 16] }
+
 dsr1-fp8-b200-trt:
   image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14
   model: deepseek-ai/DeepSeek-R1-0528

perf-changelog.yaml

@@ -5981,3 +5981,12 @@
     - "Inject the committed synthetic MTP acceptance length only for GB300 AgentX throughput; keep eval-only jobs on real target verification."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2571
 
+
+- config-keys:
+    - kimik3-fp4-b300-vllm-agentic-dspark-lmcache
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/10/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config."
+    - "Run one LMCache MP server per node with --separate-object-groups for the hybrid KDA/MLA multi-group layout, keeping the L1 in process memory (--shm-name \"\") so the DRAM budget is not capped by /dev/shm."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2597