diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh index a9256a48a..9a987363b 100755 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh @@ -120,6 +120,20 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" +# The LMCache arm starts a second long-lived process that must not outlive this +# job (its L1 holds the whole host-DRAM budget). The vLLM server's lifecycle is +# left exactly as it was -- the job wrapper still owns it. +LMCACHE_PID="" + +cleanup_lmcache_server() { + local exit_code=$? + trap - EXIT + set +e + stop_background_process_tree "$LMCACHE_PID" "LMCache server" + exit "$exit_code" +} +trap cleanup_lmcache_server EXIT + # ---- KV offloading ---------------------------------------------------------- # The generated TOTAL_CPU_DRAM_GB budget is the aggregate host-DRAM pool for the # node; SimpleCPUOffloadConnector is sized per rank. At dram-utilization 0.63 on @@ -142,8 +156,283 @@ case "${KV_OFFLOAD_BACKEND:-}" in "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":${CPU_BYTES_PER_RANK},\"lazy_offload\":false}}" ) ;; + lmcache) + require_agentic_kv_offload_backend lmcache + + # Plain PyPI. The GitHub release `expanded_assets` URLs were never + # doing anything here -- pip resolved the identical PyPI wheel + # (lmcache-0.5.4rc2-cp312-cp312-manylinux_2_27_x86_64...whl, 15.6 MB) + # in every run, whether --find-links pointed at the -cu129 assets or + # the default ones. + # + # --force-reinstall IS required: this image ships lmcache 0.5.3, so a + # plain install of 0.5.4rc2 upgrades it, but without the flag an + # already-matching version would be left in place silently. + # + # Why any of this matters: LMCache does not fail when its compiled + # `lmcache.c_ops` extension cannot be loaded. It logs "compiled + # extension not found; CudaDeviceOps stays on the torch baseline" and + # silently falls back to lmcache/v1/platform/torch_ops.py, which is + # broken for this stack's hybrid multi-KV-group KDA/MLA layout: 77-98% + # of stores die with cudaErrorInvalidValue, so the offload tier stays + # empty and the arm measures a cache that never stored anything. + # The guard after server start is what makes that loud. + # --force-reinstall is REQUIRED, not cosmetic: this image already ships + # lmcache 0.5.4rc2, so a plain `pip install lmcache==0.5.4rc2` finds the + # requirement already satisfied and installs nothing -- the --find-links + # URL is never consulted. Two earlier sweeps proved it: one pinned to + # the -cu129 assets and one to the default assets both produced the + # identical build, `LMCache v0.5.4rc2 (gf82f6fd3)`, in ~3 seconds. We + # were always running the image's copy, whose c_ops does not load here. + # Before overwriting it: does the image's OWN lmcache work? The + # 0.5.4rc2 wheel's c_ops needs c10::impl::cow::materialize_cow_storage, + # which nm shows NO torch library in this image defines -- a real ABI + # mismatch, not an ordering problem. The image ships 0.5.3, presumably + # built against this torch, so establish whether it loads before we + # replace it. + echo "=== image's pre-existing lmcache ===" + python3 -m pip show lmcache 2>/dev/null | grep -E "^Version:" || echo " (not installed)" + _IMG_SO=$(python3 -c "import lmcache, glob, os; print((glob.glob(os.path.join(os.path.dirname(lmcache.__file__), 'c_ops*.so')) or [''])[0])" 2>/dev/null || true) + if [ -n "$_IMG_SO" ]; then + echo " c_ops: $_IMG_SO" + echo " needs materialize_cow_storage: $(nm -D --undefined-only "$_IMG_SO" 2>/dev/null | grep -c materialize_cow_storage || true)" + python3 -c "import lmcache.c_ops; print(' IMAGE c_ops import: OK')" 2>&1 | tail -2 || echo " IMAGE c_ops import: FAILED" + fi + + LMCACHE_VERSION="0.5.4rc2" + agentic_pip_install --no-cache-dir --force-reinstall --no-deps \ + "lmcache==${LMCACHE_VERSION}" + # Record what actually landed. Silent installs are how the no-op above + # went unnoticed for two full sweeps; do not add --quiet back. + python3 -m pip show lmcache 2>/dev/null | grep -E "^(Version|Location):" + python3 -c "import lmcache, glob, os; print('lmcache:', lmcache.__file__); print('c_ops so:', glob.glob(os.path.join(os.path.dirname(lmcache.__file__), 'c_ops*')))" + ls -1 /usr/local/cuda*/lib64/libcudart.so* /usr/lib/x86_64-linux-gnu/libcudart.so* 2>/dev/null || true + # device_ops.ensure_native() swallows this in `except ImportError`, so + # surface the real dlopen error ourselves -- without it the failure is + # indistinguishable from a missing file. + python3 - <<'PYEOF' || true +# Definitive c_ops instrumentation. +# +# ensure_native() swallows the real exception in `except ImportError` and +# latches _native_bound=True on the first attempt, so neither the error nor the +# caller is ever visible. Wrap __import__ to capture the exception, and hook the +# logger to capture the stack at the moment the warning is emitted -- together +# these name both WHAT fails and WHO triggered it. +import builtins +import logging +import sys +import traceback + +_real_import = builtins.__import__ +_failures = [] + + +def _tracing_import(name, globals=None, locals=None, fromlist=(), level=0): + try: + return _real_import(name, globals, locals, fromlist, level) + except BaseException as exc: # noqa: BLE001 - diagnostic + if "c_ops" in name: + _failures.append((name, exc)) + print(f"[probe] IMPORT FAILED: {name}: {type(exc).__name__}: {exc}", flush=True) + traceback.print_exc() + raise + + +class _WarnHook(logging.Handler): + def emit(self, record): + try: + msg = record.getMessage() + except Exception: # noqa: BLE001 + return + if "c_ops compiled extension not found" in msg: + print("[probe] ensure_native() gave up -- caller stack:", flush=True) + traceback.print_stack() + + +builtins.__import__ = _tracing_import +logging.getLogger().addHandler(_WarnHook()) +logging.getLogger().setLevel(logging.DEBUG) + +print("[probe] torch loaded before lmcache?", "torch" in sys.modules, flush=True) + +# Import exactly what the server imports, in the server's order. +import lmcache.integration.vllm.lmcache_mp_connector # noqa: E402,F401 + +print("[probe] torch in sys.modules now:", "torch" in sys.modules, flush=True) +print("[probe] c_ops in sys.modules:", "lmcache.c_ops" in sys.modules, flush=True) +print(f"[probe] captured c_ops import failures: {len(_failures)}", flush=True) +for name, exc in _failures: + print(f"[probe] {name}: {type(exc).__name__}: {exc}", flush=True) + +# Now show whether the singleton is latched off despite the extension being loadable. +try: + from lmcache.v1.platform.cuda.device_ops import CudaDeviceOps + + ops = CudaDeviceOps() + print("[probe] CudaDeviceOps._native_bound =", getattr(ops, "_native_bound", "?"), flush=True) +except BaseException as exc: # noqa: BLE001 + print("[probe] could not inspect CudaDeviceOps:", type(exc).__name__, exc, flush=True) + +# And whether a direct import works at this point. +try: + import lmcache.c_ops # noqa: F401 + + print("[probe] direct import lmcache.c_ops AFTER: OK", flush=True) +except BaseException as exc: # noqa: BLE001 + print("[probe] direct import lmcache.c_ops AFTER FAILED:", type(exc).__name__, exc, flush=True) +PYEOF + # c_ops.so links libtorch/libc10 but they are not on the default + # loader path -- `ldd` reports libc10.so, libtorch.so, libtorch_cpu.so, + # libtorch_python.so, libc10_cuda.so and libtorch_cuda.so as "not + # found" (libcudart resolves fine). Normally that is harmless because + # `import torch` loads them RTLD_GLOBAL first, but + # CudaDeviceOps.ensure_native() sets `self._native_bound = True` BEFORE + # its `import lmcache.c_ops`, so one early failure -- before torch is + # in the process -- permanently disables native ops and silently pins + # the whole server to the broken torch fallback. Putting torch's lib + # dir on LD_LIBRARY_PATH makes the extension loadable regardless of + # import order. + TORCH_LIB_DIR=$(python3 -c "import os, torch; print(os.path.join(os.path.dirname(torch.__file__), 'lib'))") + export LD_LIBRARY_PATH="${TORCH_LIB_DIR}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" + + # ldd names the missing/unresolved shared object directly. + LMCACHE_SO=$(python3 -c "import lmcache, glob, os; print((glob.glob(os.path.join(os.path.dirname(lmcache.__file__), 'c_ops*.so')) or [''])[0])") + if [ -n "$LMCACHE_SO" ]; then + echo "ldd $LMCACHE_SO" + ldd "$LMCACHE_SO" 2>&1 | grep -E "not found|libcudart|libtorch|libc10" || true + fi + # Which torch library actually EXPORTS the symbol c_ops needs? + # LD_PRELOAD of libc10/libtorch_cpu/libtorch did not resolve it, so + # locate the definition rather than guessing which .so to preload. + # Which torch is this, and what does its c10 actually export under + # c10::impl::cow? The wheel embeds no torch version, so the mismatch + # has to be characterised from the symbols. c_ops needs BOTH + # c10::impl::cow::is_cow_data_ptr and ::materialize_cow_storage. + python3 -c "import torch; print(' torch:', torch.__version__, '| built with CUDA', torch.version.cuda)" + echo " --- all c10::impl::cow symbols exported by libc10.so ---" + nm -D --defined-only "$TORCH_LIB_DIR/libc10.so" 2>/dev/null | grep -oE "_ZN3c104impl3cow[A-Za-z0-9_]*" | sort -u | head -20 || true + echo " --- (count) ---" + nm -D --defined-only "$TORCH_LIB_DIR/libc10.so" 2>/dev/null | grep -c "3cow" || true + echo " --- what c_ops requires ---" + nm -D --undefined-only "$LMCACHE_SO" 2>/dev/null | grep -oE "_ZN3c104impl3cow[A-Za-z0-9_]*" | sort -u || true + + echo "=== searching torch libs for materialize_cow_storage ===" + for _so in "$TORCH_LIB_DIR"/libc10.so "$TORCH_LIB_DIR"/libc10_cuda.so \ + "$TORCH_LIB_DIR"/libtorch.so "$TORCH_LIB_DIR"/libtorch_cpu.so \ + "$TORCH_LIB_DIR"/libtorch_cuda.so "$TORCH_LIB_DIR"/libtorch_python.so; do + [ -f "$_so" ] || continue + _n=$(nm -D --defined-only "$_so" 2>/dev/null | grep -c "materialize_cow_storage" || true) + _u=$(nm -D --undefined-only "$_so" 2>/dev/null | grep -c "materialize_cow_storage" || true) + echo " $(basename "$_so"): defined=$_n undefined=$_u" + done + echo " (c_ops needs it) $(nm -D --undefined-only "$LMCACHE_SO" 2>/dev/null | grep -c materialize_cow_storage || true)" + + python3 -c \ + "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ + >/dev/null + + # One MP server for the node, per the Kimi-K3 recipe + # (docs.lmcache.ai/recipes/kimi_k3.html), but NOT that recipe's + # --chunk-size 768: the connector requires the chunk to be a multiple + # of every engine KV group's tokens_per_block, and this stack's blocks + # are far larger than 768. On this exact image and script, vLLM pins + # the attention group to 1536 (run 31404943911, interface.py:911, + # "Setting attention block size to 1536 tokens to ensure that + # attention page size is >= mamba page size") and pads the KDA/mamba + # page to match (interface.py:935). That alignment is generic vLLM + # code, not platform-specific, which is why the MI355X sister arm + # lands on the same 1536 plus a 3072-token KDA state group and also + # needs 3072. 768 is smaller than the attention block, so it is not a + # multiple of it and fails at connector init. The multi-group layout + # additionally requires one object group per sliding-window size: + # --separate-object-groups. + LMCACHE_PORT=6555 + LMCACHE_HTTP_PORT=8090 + LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" + + # Consume the generated aggregate budget verbatim, per + # benchmarks/single_node/agentic/README.md. --shm-name "" keeps L1 in + # ordinary process memory so the budget is not silently capped by the + # size of the node's /dev/shm mount. + LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB" + + # c_ops.so resolves c10/torch symbols only once libtorch is loaded + # RTLD_GLOBAL. Captured directly: + # + # ImportError: .../lmcache/c_ops.cpython-312-x86_64-linux-gnu.so: + # undefined symbol: _ZN3c104impl3cow23materialize_cow_storageERNS_11StorageImplE + # ( c10::impl::cow::materialize_cow_storage(c10::StorageImpl&) ) + # + # It is NOT an ABI mismatch -- the same import succeeds moments later, + # once torch is in the process, so the symbol does exist in this + # image's torch. But CudaDeviceOps.ensure_native() latches + # _native_bound = True on its FIRST attempt and swallows the error, so + # one early miss pins the process to the torch fallback for good, and + # that fallback is broken for this stack's multi-KV-group KDA/MLA + # layout (77-98% of stores fail). + # + # Preloading torch in the parent is not enough: the server forks CPU + # and GPU workers that import lmcache fresh. LD_PRELOAD applies to the + # whole process tree, so the symbols are resolvable everywhere. Scoped + # to this command via `env` rather than exported, to leave the vLLM + # server's environment untouched. + LMCACHE_LD_PRELOAD="${TORCH_LIB_DIR}/libc10.so:${TORCH_LIB_DIR}/libtorch_cpu.so:${TORCH_LIB_DIR}/libtorch.so" + if [ -f "${TORCH_LIB_DIR}/libc10_cuda.so" ]; then + LMCACHE_LD_PRELOAD="${LMCACHE_LD_PRELOAD}:${TORCH_LIB_DIR}/libc10_cuda.so:${TORCH_LIB_DIR}/libtorch_cuda.so" + fi + echo "LMCACHE_LD_PRELOAD=$LMCACHE_LD_PRELOAD" + LMCACHE_CMD=( + env "LD_PRELOAD=$LMCACHE_LD_PRELOAD" + lmcache server + --host 127.0.0.1 + --port "$LMCACHE_PORT" + --http-host 127.0.0.1 + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$LMCACHE_L1_SIZE_GB" + --l1-init-size-gb 10 + --chunk-size 3072 + --separate-object-groups + --enable-extra-logging + --extra-logging-interval 30 + --max-cpu-workers 8 + --max-gpu-workers 1 + --eviction-policy LRU + --supported-transfer-mode lmcache_driven + --shm-name "" + ) + append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + wait_for_ready \ + --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \ + --log "$LMCACHE_LOG" \ + --pid "$LMCACHE_PID" \ + --sleep-interval 1 \ + --timeout 600 + + # Second guard, on the server process itself: the import check above + # only proves c_ops loads in this shell's python3. Abort rather than + # benchmark a silently-degraded offload tier (see the install note). + if grep -q "c_ops compiled extension not found" "$LMCACHE_LOG"; then + echo "Error: LMCache fell back to the torch baseline (c_ops did not load)." >&2 + echo " The installed lmcache's compiled extension is unusable here." >&2 + echo " Check the pip show / c_ops / libcudart lines logged above:" >&2 + echo " either the install did not take, or the build's CUDA ABI" >&2 + echo " does not match this image." >&2 + grep -m1 "c_ops compiled extension not found" "$LMCACHE_LOG" >&2 + exit 1 + fi + + # 100k-330k-token agentic prefixes make single retrieves large; use the + # same MQ timeout headroom as the MI355X arm. + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}" + ) + ;; *) - echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or vllm-simple)" >&2 + echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty, vllm-simple, or lmcache)" >&2 exit 1 ;; esac diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b469673b4..dff0f93e2 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1537,6 +1537,33 @@ kimik3-fp4-b300-vllm-agentic-dspark: # TP8 SimpleCPUOffload (host DRAM) - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16] } +# LMCache MP-server DRAM offload on top of the same DSpark MTP serving stack as +# kimik3-fp4-b300-vllm-agentic-dspark (same image, script, and topology). A +# dedicated key so LMCache points can be selected and swept without re-running +# the resident and vllm-simple arms of the base key. +kimik3-fp4-b300-vllm-agentic-dspark-lmcache: + image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b300-nv + precision: fp4 + framework: vllm + multinode: false + scenarios: + # Agentic-coding only, and 0.63 matches the base key so the LMCache L1 gets + # exactly the host-DRAM budget the SimpleCPUOffload arm gets (~1,889 GB + # aggregate at TP8). The LMCache server runs with --shm-name "" so that L1 + # lives in ordinary process memory and is not capped by /dev/shm. + agentic-coding: + - dram-utilization: 0.63 + search-space: + # 4/8/16 land on the base key's vllm-simple ladder at its top three + # offload points, so LMCache is directly comparable there; 10 fills the + # 8->16 gap, where the MI355X sister arm's ladder is densest. TP8-only + # for the same reason as the base key: a ~1.5 TB MXFP4 checkpoint does + # not fit below 8 GPUs. + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.4rc2" }, conc-list: [4, 8, 10, 16] } + dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 model: deepseek-ai/DeepSeek-R1-0528 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 067b8a79f..a63b0534f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5981,3 +5981,12 @@ - "Inject the committed synthetic MTP acceptance length only for GB300 AgentX throughput; keep eval-only jobs on real target verification." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2571 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark-lmcache + scenario-type: + - agentic-coding + description: + - "Add a dedicated LMCache 0.5.4rc2 DRAM KV-offload key at TP8 conc 4/8/10/16 on top of the unchanged kimik3-fp4-b300-vllm-agentic-dspark DSpark MTP stack, with the version pinned in the master config." + - "Run one LMCache MP server per node with --separate-object-groups for the hybrid KDA/MLA multi-group layout, keeping the L1 in process memory (--shm-name \"\") so the DRAM budget is not capped by /dev/shm." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2597