diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh index 1d9bff132..823405965 100755 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh @@ -120,6 +120,20 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" +# The LMCache arm starts a second long-lived process that must not outlive this +# job (its L1 holds the whole host-DRAM budget). The vLLM server's lifecycle is +# left exactly as it was -- the job wrapper still owns it. +LMCACHE_PID="" + +cleanup_lmcache_server() { + local exit_code=$? + trap - EXIT + set +e + stop_background_process_tree "$LMCACHE_PID" "LMCache server" + exit "$exit_code" +} +trap cleanup_lmcache_server EXIT + # ---- KV offloading ---------------------------------------------------------- # The generated TOTAL_CPU_DRAM_GB budget is the aggregate host-DRAM pool for the # node; SimpleCPUOffloadConnector is sized per rank. At dram-utilization 0.63 on @@ -142,8 +156,97 @@ case "${KV_OFFLOAD_BACKEND:-}" in "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":${CPU_BYTES_PER_RANK},\"lazy_offload\":false}}" ) ;; + lmcache) + require_agentic_kv_offload_backend lmcache + + # LMCache's own CUDA 12.9 build, not the generic PyPI wheel -- the + # release publishes a dedicated -cu129 asset set, the CUDA counterpart + # of the -rocm assets the MI355X sister arm installs. This is the + # upstream install line, which reads in uv form as: + # + # uv pip install lmcache==v$VERSION \ + # --extra-index-url https://download.pytorch.org/whl/cu129 \ + # --find-links .../expanded_assets/v$VERSION-cu129 \ + # --index-strategy unsafe-best-match + # + # --index-strategy has no pip counterpart because it does not need one: + # pip already resolves best-match across every configured index, which + # is exactly what unsafe-best-match restores in uv. torch is an + # unpinned LMCache requirement that the image already satisfies, so + # its tested build is left alone and the cu129 torch index is only + # consulted for CUDA wheels pip would otherwise miss. + LMCACHE_VERSION="0.5.4rc2" + LMCACHE_CUDA_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-cu129" + agentic_pip_install --quiet --no-cache-dir \ + "lmcache==${LMCACHE_VERSION}" \ + --extra-index-url https://download.pytorch.org/whl/cu129 \ + --find-links "$LMCACHE_CUDA_INDEX" + python3 -c \ + "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ + >/dev/null + + # One MP server for the node, per the Kimi-K3 recipe + # (docs.lmcache.ai/recipes/kimi_k3.html), but NOT that recipe's + # --chunk-size 768: the connector requires the chunk to be a multiple + # of every engine KV group's tokens_per_block, and this stack's blocks + # are far larger than 768. On this exact image and script, vLLM pins + # the attention group to 1536 (run 31404943911, interface.py:911, + # "Setting attention block size to 1536 tokens to ensure that + # attention page size is >= mamba page size") and pads the KDA/mamba + # page to match (interface.py:935). That alignment is generic vLLM + # code, not platform-specific, which is why the MI355X sister arm + # lands on the same 1536 plus a 3072-token KDA state group and also + # needs 3072. 768 is smaller than the attention block, so it is not a + # multiple of it and fails at connector init. The multi-group layout + # additionally requires one object group per sliding-window size: + # --separate-object-groups. + LMCACHE_PORT=6555 + LMCACHE_HTTP_PORT=8090 + LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" + + # Consume the generated aggregate budget verbatim, per + # benchmarks/single_node/agentic/README.md. --shm-name "" keeps L1 in + # ordinary process memory so the budget is not silently capped by the + # size of the node's /dev/shm mount. + LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB" + + LMCACHE_CMD=( + lmcache server + --host 127.0.0.1 + --port "$LMCACHE_PORT" + --http-host 127.0.0.1 + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$LMCACHE_L1_SIZE_GB" + --l1-init-size-gb 10 + --chunk-size 3072 + --separate-object-groups + --enable-extra-logging + --extra-logging-interval 30 + --max-cpu-workers 8 + --max-gpu-workers 1 + --eviction-policy LRU + --supported-transfer-mode lmcache_driven + --shm-name "" + ) + append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + wait_for_ready \ + --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \ + --log "$LMCACHE_LOG" \ + --pid "$LMCACHE_PID" \ + --sleep-interval 1 \ + --timeout 600 + + # 100k-330k-token agentic prefixes make single retrieves large; use the + # same MQ timeout headroom as the MI355X arm. + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}" + ) + ;; *) - echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or vllm-simple)" >&2 + echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty, vllm-simple, or lmcache)" >&2 exit 1 ;; esac diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b95f59618..32ab2dfec 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1546,6 +1546,33 @@ kimik3-fp4-b300-vllm-agentic-dspark: # TP8 SimpleCPUOffload (host DRAM) - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16] } +# LMCache MP-server DRAM offload on top of the same DSpark MTP serving stack as +# kimik3-fp4-b300-vllm-agentic-dspark (same image, script, and topology). A +# dedicated key so LMCache points can be selected and swept without re-running +# the resident and vllm-simple arms of the base key. +kimik3-fp4-b300-vllm-agentic-dspark-lmcache: + image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b300-nv + precision: fp4 + framework: vllm + multinode: false + scenarios: + # Agentic-coding only, and 0.63 matches the base key so the LMCache L1 gets + # exactly the host-DRAM budget the SimpleCPUOffload arm gets (~1,889 GB + # aggregate at TP8). The LMCache server runs with --shm-name "" so that L1 + # lives in ordinary process memory and is not capped by /dev/shm. + agentic-coding: + - dram-utilization: 0.63 + search-space: + # 4/8/16 land on the base key's vllm-simple ladder at its top three + # offload points, so LMCache is directly comparable there; 10 fills the + # 8->16 gap, where the MI355X sister arm's ladder is densest. TP8-only + # for the same reason as the base key: a ~1.5 TB MXFP4 checkpoint does + # not fit below 8 GPUs. + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.4rc2" }, conc-list: [4, 8, 10, 16] } + dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 model: deepseek-ai/DeepSeek-R1-0528 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6082f2b86..84513f45b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5918,3 +5918,12 @@ - "Add a TEP2 arm (tp 2, ep 2) to the qwen3.5-fp4-b200-sglang-mtp 8k/1k sweep at concurrency 16, 32, and 64" - "Rides on the NVFP4-V2 checkpoint switch from #2205" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2550 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark-lmcache + scenario-type: + - agentic-coding + description: + - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/10/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config." + - "Run one LMCache MP server per node with --separate-object-groups for the hybrid KDA/MLA multi-group layout, keeping the L1 in process memory (--shm-name \"\") so the DRAM budget is not capped by /dev/shm." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2593