kimik3-fp4-b300-vllm-agentic-dspark: add LMCache DRAM KV-offload arm

benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh

@@ -120,6 +120,20 @@
 SERVER_LOG="$RESULT_DIR/server.log"
 mkdir -p "$RESULT_DIR"
 
+# The LMCache arm starts a second long-lived process that must not outlive this
+# job (its L1 holds the whole host-DRAM budget). The vLLM server's lifecycle is
+# left exactly as it was -- the job wrapper still owns it.
+LMCACHE_PID=""
+
+cleanup_lmcache_server() {
+    local exit_code=$?
+    trap - EXIT
+    set +e
+    stop_background_process_tree "$LMCACHE_PID" "LMCache server"
+    exit "$exit_code"
+}
+trap cleanup_lmcache_server EXIT
+
 # ---- KV offloading ----------------------------------------------------------
 # The generated TOTAL_CPU_DRAM_GB budget is the aggregate host-DRAM pool for the
 # node; SimpleCPUOffloadConnector is sized per rank. At dram-utilization 0.63 on
@@ -142,8 +156,97 @@
             "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":${CPU_BYTES_PER_RANK},\"lazy_offload\":false}}"
         )
         ;;
+    lmcache)
+        require_agentic_kv_offload_backend lmcache
+
+        # LMCache's own CUDA 12.9 build, not the generic PyPI wheel -- the
+        # release publishes a dedicated -cu129 asset set, the CUDA counterpart
+        # of the -rocm assets the MI355X sister arm installs. This is the
+        # upstream install line, which reads in uv form as:
+        #
+        #   uv pip install lmcache==v$VERSION \
+        #     --extra-index-url https://download.pytorch.org/whl/cu129 \
+        #     --find-links .../expanded_assets/v$VERSION-cu129 \
+        #     --index-strategy unsafe-best-match
+        #
+        # --index-strategy has no pip counterpart because it does not need one:
+        # pip already resolves best-match across every configured index, which
+        # is exactly what unsafe-best-match restores in uv. torch is an
+        # unpinned LMCache requirement that the image already satisfies, so
+        # its tested build is left alone and the cu129 torch index is only
+        # consulted for CUDA wheels pip would otherwise miss.
+        LMCACHE_VERSION="0.5.4rc2"
+        LMCACHE_CUDA_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-cu129"
+        agentic_pip_install --quiet --no-cache-dir \
+            "lmcache==${LMCACHE_VERSION}" \
+            --extra-index-url https://download.pytorch.org/whl/cu129 \
+            --find-links "$LMCACHE_CUDA_INDEX"
+        python3 -c \
+            "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
+            >/dev/null
+
+        # One MP server for the node, per the Kimi-K3 recipe
+        # (docs.lmcache.ai/recipes/kimi_k3.html), but NOT that recipe's
+        # --chunk-size 768: the connector requires the chunk to be a multiple
+        # of every engine KV group's tokens_per_block, and this stack's blocks
+        # are far larger than 768. On this exact image and script, vLLM pins
+        # the attention group to 1536 (run 31404943911, interface.py:911,
+        # "Setting attention block size to 1536 tokens to ensure that
+        # attention page size is >= mamba page size") and pads the KDA/mamba
+        # page to match (interface.py:935). That alignment is generic vLLM
+        # code, not platform-specific, which is why the MI355X sister arm
+        # lands on the same 1536 plus a 3072-token KDA state group and also
+        # needs 3072. 768 is smaller than the attention block, so it is not a
+        # multiple of it and fails at connector init. The multi-group layout
+        # additionally requires one object group per sliding-window size:
+        # --separate-object-groups.
+        LMCACHE_PORT=6555
+        LMCACHE_HTTP_PORT=8090
+        LMCACHE_LOG="$RESULT_DIR/lmcache_server.log"
+
+        # Consume the generated aggregate budget verbatim, per
+        # benchmarks/single_node/agentic/README.md. --shm-name "" keeps L1 in
+        # ordinary process memory so the budget is not silently capped by the
+        # size of the node's /dev/shm mount.
+        LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB"
+
+        LMCACHE_CMD=(
+            lmcache server
+            --host 127.0.0.1
+            --port "$LMCACHE_PORT"
+            --http-host 127.0.0.1
+            --http-port "$LMCACHE_HTTP_PORT"
+            --l1-size-gb "$LMCACHE_L1_SIZE_GB"
+            --l1-init-size-gb 10
+            --chunk-size 3072
+            --separate-object-groups
+            --enable-extra-logging
+            --extra-logging-interval 30
+            --max-cpu-workers 8
+            --max-gpu-workers 1
+            --eviction-policy LRU
+            --supported-transfer-mode lmcache_driven
+            --shm-name ""
+        )
+        append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}"
+        "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 &
+        LMCACHE_PID=$!
+        wait_for_ready \
+            --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \
+            --log "$LMCACHE_LOG" \
+            --pid "$LMCACHE_PID" \
+            --sleep-interval 1 \
+            --timeout 600
+
+        # 100k-330k-token agentic prefixes make single retrieves large; use the
+        # same MQ timeout headroom as the MI355X arm.
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}"
+        )
+        ;;
     *)
-        echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or vllm-simple)" >&2
+        echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty, vllm-simple, or lmcache)" >&2
         exit 1
         ;;
 esac

configs/nvidia-master.yaml

@@ -1546,6 +1546,33 @@ kimik3-fp4-b300-vllm-agentic-dspark:
       # TP8 SimpleCPUOffload (host DRAM)
       - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16] }
 
+# LMCache MP-server DRAM offload on top of the same DSpark MTP serving stack as
+# kimik3-fp4-b300-vllm-agentic-dspark (same image, script, and topology). A
+# dedicated key so LMCache points can be selected and swept without re-running
+# the resident and vllm-simple arms of the base key.
+kimik3-fp4-b300-vllm-agentic-dspark-lmcache:
+  image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21
+  model: moonshotai/Kimi-K3
+  model-prefix: kimik3
+  runner: cluster:b300-nv
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    # Agentic-coding only, and 0.63 matches the base key so the LMCache L1 gets
+    # exactly the host-DRAM budget the SimpleCPUOffload arm gets (~1,889 GB
+    # aggregate at TP8). The LMCache server runs with --shm-name "" so that L1
+    # lives in ordinary process memory and is not capped by /dev/shm.
+    agentic-coding:
+    - dram-utilization: 0.63
+      search-space:
+      # 4/8/16 land on the base key's vllm-simple ladder at its top three
+      # offload points, so LMCache is directly comparable there; 10 fills the
+      # 8->16 gap, where the MI355X sister arm's ladder is densest. TP8-only
+      # for the same reason as the base key: a ~1.5 TB MXFP4 checkpoint does
+      # not fit below 8 GPUs.
+      - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.4rc2" }, conc-list: [4, 8, 10, 16] }
+
 dsr1-fp8-b200-trt:
   image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14
   model: deepseek-ai/DeepSeek-R1-0528

perf-changelog.yaml

@@ -5918,3 +5918,12 @@
     - "Add a TEP2 arm (tp 2, ep 2) to the qwen3.5-fp4-b200-sglang-mtp 8k/1k sweep at concurrency 16, 32, and 64"
     - "Rides on the NVFP4-V2 checkpoint switch from #2205"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2550
+
+- config-keys:
+    - kimik3-fp4-b300-vllm-agentic-dspark-lmcache
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/10/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config."
+    - "Run one LMCache MP server per node with --separate-object-groups for the hybrid KDA/MLA multi-group layout, keeping the L1 in process memory (--shm-name \"\") so the DRAM budget is not capped by /dev/shm."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2593