NVIDIA · chuangz0 · Aug 20, 2026 · Aug 13, 2026 · Aug 13, 2026 · Aug 14, 2026
@@ -56,7 +56,10 @@
 from tensorrt_llm._torch.distributed import Distributed
 from tensorrt_llm._torch.pyexecutor.hang_detector import HangDetector
 from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2
-from tensorrt_llm._torch.pyexecutor.kv_cache_transceiver import create_kv_cache_transceiver
+from tensorrt_llm._torch.pyexecutor.kv_cache_transceiver import (
+    create_kv_cache_transceiver,
+    maybe_enable_fabric_memory_for_python_transceiver,
+)
 from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest, LlmRequestState, LlmRequestType
 from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager
 from tensorrt_llm.llmapi.llm_args import BlockReuseConfig, CacheTransceiverConfig
@@ -830,6 +833,28 @@ def open_sock():
     zmq_sock = open_sock()
 
     cases = build_cases(cfg)
+    # The C++ fabric-memory env getter is cached on its first KV pool
+    # allocation. Enable the default before any matrix case builds a pool, even
+    # when a C++ transceiver case appears before Python+V1 in the matrix.
+    python_v1_case = next(
+        (case for case in cases if case["runtime"] == "PYTHON" and case["cache_manager"] == "V1"),
+        None,
+    )
+    if python_v1_case is not None:
+        maybe_enable_fabric_memory_for_python_transceiver(
+            CacheTransceiverConfig(
+                backend=python_v1_case["backend"],
+                transceiver_runtime="PYTHON",
+            ),
+            KVCacheManager,
+        )
+        print(
+            f"[{role} rank={rank}] PYTHON+V1 case in matrix: "
+            "TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY="
+            f"{os.environ.get('TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY')} "
+            "applies to every case in this run, including C++ transceiver ones",
+            flush=True,
+        )
     req_lens = cfg["test_matrix"]["request_lengths"]
     warmup = cfg["test_matrix"]["warmup_requests"]
     num_req = cfg["test_matrix"]["num_requests_per_length"]

@@ -2373,8 +2373,10 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                         fi
                     done
 
-                    # Kill tail -f process
-                    kill \$tailPid
+                    # Stop and reap the log follower. It may have already exited
+                    # when the remote log stream closes; that is not a test failure.
+                    kill \$tailPid 2>/dev/null || true
+                    wait \$tailPid 2>/dev/null || true
 
                     # Wait briefly to ensure accounting is consistent
                     sleep 10

@@ -44,15 +44,19 @@
         "rocep198s0:1,rocep199s0:1,rocep205s0:1,rocep206s0:1"
         " UCX_IB_GID_INDEX=auto UCX_IB_TRAFFIC_CLASS=52 UCX_IB_SL=0",
     ),
-    # oci-aga: avoid transports that fail on this VF fabric, disable DEVX to
-    # avoid UAR allocation failures, and pin the GPU-connected rail VFs.
+    # oci-aga: use TCP over IPv4 alongside the local CUDA/shared-memory
+    # transports.
     (
         "oci-aga*",
         "*",
-        "export UCX_TLS=^tcp,rc_gda,gga UCX_IB_MLX5_DEVX=n "
-        "UCX_NET_DEVICES="
-        "rdma_vf_rail0:1,rdma_vf_rail1:1,rdma_vf_rail2:1,rdma_vf_rail3:1 "
-        "UCX_IB_TRAFFIC_CLASS=96 TRTLLM_NIXL_NUM_THREADS=1",
+        "export UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp UCX_TCP_AF_PRIO=inet",
+    ),
+    # oci-hsg: UCX picks wrong RDMA devices; pin the usable mlx5 ports and
+    # keep eth0 as the TCP fallback device.
+    (
+        "oci-hsg*",
+        "*",
+        "export UCX_NET_DEVICES=mlx5_0:1,mlx5_1:1,mlx5_3:1,mlx5_4:1,eth0",
     ),
     # nsc-svg: UCX picks wrong RDMA devices; pin the usable mlx5 ports.
     (
@@ -61,9 +65,15 @@
         "export UCX_NET_DEVICES="
         "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1",
     ),
-    # aws-cmh: UCX transport auto-selection hangs on this fabric; pin the
-    # working transport set explicitly.
-    ("aws-cmh*", "*", "export UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp"),
+    # aws-cmh: UCX transport/device auto-selection hangs on this fabric; pin
+    # the working transport set and Ethernet/RDMA devices explicitly.
+    (
+        "aws-cmh*",
+        "*",
+        "export UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp "
+        "UCX_NET_DEVICES=eth0,mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,"
+        "mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1",
+    ),
     # aws-dfw: gdr_copy is broken on this cluster; exclude it.
     ("aws-dfw*", "*", "export UCX_TLS=^gdr_copy"),
     # Default: base unset only.

@@ -1,3 +1,6 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 # Cache-transceiver precheck gate for the disaggregated perf-sanity launch
 # script. submit.py splices this file into the generated launch script ahead
 # of slurm_launch_draft.sh, which calls run_cache_transceiver_precheck after
@@ -21,7 +24,7 @@ ct_xml_escape() {
 # First few root-cause-shaped lines of a step log — the tail alone can miss
 # the real error when it happened early and retry spam follows.
 ct_first_errors() {
-    grep -m 5 -nE "Traceback \(most recent call last\)|MPI_ABORT|MPIR_Err|srun: error|Segmentation fault|CUDA error|RuntimeError|AssertionError|INIT_ERROR|TRANSFER_ERROR" \
+    grep -m 5 -nE "Traceback \(most recent call last\)|MPI_ABORT|MPIR_Err|srun: error|Segmentation fault|CUDA error|RuntimeError|AssertionError|INIT_ERROR|TRANSFER_ERROR|WATCHDOG_KILL|EXTERNAL_TIMEOUT|EXTERNAL_KILL|PROCESS_EXIT" \
         "$1" 2>/dev/null || true
 }
 
@@ -40,6 +43,29 @@ ct_step_excerpt() {
     tail -n 60 "$stepLog" 2>/dev/null || true
 }
 
+# The driver normally owns verdict files. If the external total-runtime
+# backstop or srun kills it first, synthesize a precise verdict here so junit
+# never degrades the failure to NO_STATUS.
+ct_record_step_exit() {
+    local name=$1 rc=$2
+    local statusFile="$precheckDir/status/$name.status"
+    # Preserve a driver's specific failure, but never let a stale/premature
+    # PASS mask the non-zero process exit observed by the gate.
+    if [ -f "$statusFile" ] && ! grep -q '^PASS' "$statusFile"; then
+        return 0
+    fi
+    if [ "$rc" -eq 124 ]; then
+        printf 'EXTERNAL_TIMEOUT %s: step exceeded the %ss total-runtime backstop\n' \
+            "$name" "${ctPrecheckTimeout:-900}" > "$statusFile"
+    elif [ "$rc" -eq 137 ]; then
+        printf 'EXTERNAL_KILL %s: step received SIGKILL (possibly timeout -k escalation) before the driver wrote a verdict\n' \
+            "$name" > "$statusFile"
+    else
+        printf 'PROCESS_EXIT %s: srun exited with code %s before the driver wrote a verdict\n' \
+            "$name" "$rc" > "$statusFile"
+    fi
+}
+
 # Console summary for a failed precheck: per-instance verdicts, first error
 # lines + tail of each failing step log, and UCX red-flag lines.
 # Uses: precheckDir, precheckNames.
@@ -110,7 +136,7 @@ run_cache_transceiver_precheck() {
     fi
     echo "Starting cache transceiver precheck..."
     precheckDir="$testOutputDir/cache_transceiver_precheck"
-    mkdir -p "$precheckDir/logs"
+    mkdir -p "$precheckDir/logs" "$precheckDir/status"
     # A reused work dir (Slurm requeue reruns this batch script with the same
     # directories) may hold a previous run's rendezvous/status/csv/abort files:
     # stale addr files would point gen leaders at dead ports, stale status files
@@ -120,17 +146,21 @@ run_cache_transceiver_precheck() {
     # (the Python transceiver's perf_<uuid>_<rank>.csv are per-run and appended)
     # would make parse_python_bandwidth_gbps median over two runs' samples. The
     # driver also job-id-stamps addr files as a second line of defense.
-    rm -f "$precheckDir"/rendezvous/*.addr "$precheckDir"/status/*.status \
-        "$precheckDir"/status/*.json "$precheckDir"/precheck.abort 2>/dev/null || true
+    rm -f "$precheckDir"/rendezvous/*.addr "$precheckDir"/progress/*.json \
+        "$precheckDir"/status/*.status "$precheckDir"/status/*.json \
+        "$precheckDir"/precheck.abort 2>/dev/null || true
     rm -rf "$precheckDir"/csv 2>/dev/null || true
     precheckPids=()
     precheckNames=()
     # ct_launch_step <role> <idx> <nodes> <gpusPerNode> <nodeList> <pytestCmd>
     ct_launch_step() {
         local role=$1 i=$2 nodes=$3 gpusPerNode=$4 nodeList=$5 pytestCmd=$6
-        export DISAGG_SERVING_TYPE="${role^^}_PRECHECK_$i"
-        export pytestCommand="$pytestCmd --server-idx $i"
-        timeout -k 60 "${ctPrecheckTimeout:-900}" \
+        # Scope the launch identity and command to this precheck process. The
+        # parent shell subsequently launches the real perf workers, so these
+        # must not replace its DISAGG_SERVING_TYPE/pytestCommand values.
+        DISAGG_SERVING_TYPE="${role^^}_PRECHECK_$i" \
+            pytestCommand="$pytestCmd --server-idx $i" \
+            timeout -k 60 "${ctPrecheckTimeout:-900}" \
             srun "${srunArgs[@]}" --mpi=pmix --kill-on-bad-exit=1 \
             -N "$nodes" \
             -w "$nodeList" \
@@ -158,6 +188,7 @@ run_cache_transceiver_precheck() {
         else
             rc=$?
             echo "Precheck step ${precheckNames[$k]} FAILED (exit $rc; 124 = external timeout)"
+            ct_record_step_exit "${precheckNames[$k]}" "$rc"
             precheckFailed=1
         fi
     done

@@ -137,12 +137,53 @@ def _test_nodeid(test_line):
         The pytest node ID from the entry.
     """
     return re.split(
-        r"\s+(?:XFAIL|SKIP|UNSTABLE|TIMEOUT)(?:\s|$)",
+        r"\s+(?:XFAIL|SKIP|UNSTABLE|TIMEOUT)(?=[\s(]|$)",
         test_line,
         maxsplit=1,
     )[0]
 
 
+def _test_marker(test_line):
+    """Return a test-list execution marker, or ``None`` when absent."""
+    line = test_line.partition("#")[0].strip()
+    match = re.search(r"\s+(XFAIL|SKIP|UNSTABLE|TIMEOUT)(?=[\s(]|$)", line)
+    return match.group(1) if match else None
+
+
+def selected_test_is_skip_waived(selected_test_line, waives_file, test_prefix=None):
+    """Whether pytest will skip the selected case before executing its body.
+
+    The CI pipeline merges remote waives into the repository waives file
+    before invoking this launcher. Mirror the exact-nodeid SKIP decision here
+    so a skipped test does not run an otherwise unrelated precheck first.
+    """
+    if _test_marker(selected_test_line) == "SKIP":
+        return True
+
+    selected_nodeid = _test_nodeid(selected_test_line).strip()
+    try:
+        waive_lines = _read_test_list_lines(waives_file)
+    except (FileNotFoundError, ValueError):
+        return False
+
+    for line in waive_lines:
+        if _test_marker(line) != "SKIP":
+            continue
+        waived_nodeid = _test_nodeid(line).strip()
+        if waived_nodeid.startswith("full:"):
+            scope, separator, waived_nodeid = waived_nodeid[5:].partition("/")
+            if not separator or not test_prefix:
+                continue
+            # Match the platform-prefix handling in test_list_parser. SM
+            # waives require runtime GPU discovery and remain pytest-owned.
+            platform_prefix = test_prefix.split("-", 1)[0]
+            if scope.startswith("sm") or platform_prefix not in scope:
+                continue
+        if waived_nodeid == selected_nodeid:
+            return True
+    return False
+
+
 def _load_pytest_split_durations(tokens, llm_src):
     """Load pytest-split duration data using the launcher's path fallback.
 
@@ -759,6 +800,14 @@ def main():
         script_prefix_lines,
         args.split_group,
     )
+    pytest_tokens = _pytest_command_tokens(script_prefix_lines)
+    selected_test_skipped = selected_test_is_skip_waived(
+        selected_test_line,
+        os.path.join(args.llm_src, "tests", "integration", "test_lists", "waives.txt"),
+        test_prefix=_pytest_option(pytest_tokens, "--test-prefix"),
+    )
+    if selected_test_skipped:
+        print("Selected test is SKIP-waived; cache-transceiver precheck will not run")
     config_yaml, server_name, benchmark_mode, runtime_mode = parse_test_case_name(
         args.llm_src,
         selected_test_line,
@@ -903,9 +952,22 @@ def main():
         # (single owner, shared with the local flow).
         pcfg = _import_precheck_config(args.llm_src)
         precheck_enabled = pcfg.precheck_enabled(config)
-        llm_models_root = (
-            _resolve_llm_models_root(script_prefix_lines) if precheck_enabled else None
-        )
+        precheck_will_run = precheck_enabled and not selected_test_skipped
+        # The model root is only consumed by the precheck (auto KV-cache-manager
+        # resolution needs the model config). Fail fast only when the precheck
+        # will actually run; otherwise degrade to a warning so stages whose
+        # pytestCommand does not carry LLM_MODELS_ROOT inline keep submitting.
+        llm_models_root = None
+        if precheck_enabled:
+            try:
+                llm_models_root = _resolve_llm_models_root(script_prefix_lines)
+            except ValueError as e:
+                if precheck_will_run:
+                    raise
+                print(
+                    f"WARNING: {e}; "
+                    "cache-transceiver precheck is skipped for this config so continuing"
+                )
         script_prefix_lines.extend(
             pcfg.precheck_prefix_lines(
                 config,
@@ -918,6 +980,7 @@ def main():
                 ),
                 stage_name=args.stage_name,
                 llm_models_root=llm_models_root,
+                skip_precheck=selected_test_skipped,
             )
         )
         srun_args_lines.extend(

@@ -57,7 +57,9 @@
 from .dwdp import DwdpManager
 from .guided_decoder import GuidedDecoder
 from .kv_cache_manager_v2 import KVCacheManagerV2
-from .kv_cache_transceiver import AttentionTypeCpp, create_kv_cache_transceiver
+from .kv_cache_transceiver import (
+    AttentionTypeCpp, create_kv_cache_transceiver,
+    maybe_enable_fabric_memory_for_python_transceiver)
 from .llm_request import ExecutorResponse, LlmRequestState
 from .mamba_cache_manager import (BaseMambaCacheManager,
                                   CppMambaHybridCacheManager,
@@ -589,28 +591,8 @@ def __init__(
         self._maybe_enable_fabric_memory_for_python_transceiver()
 
     def _maybe_enable_fabric_memory_for_python_transceiver(self) -> None:
-        """Default TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=1 for the Python
-        transceiver on the C++ V1 KV cache manager.
-
-        The Python transceiver (KvCacheTransceiverV2) transfers KV blocks
-        directly out of the C++ pool, so the pool should be allocated with
-        fabric memory to enable MNNVL transfers. This must run before any
-        pool allocation because the C++ env getter caches the value on first
-        read. Explicit user settings are respected, and platforms without
-        fabric memory support fall back to standard allocation in C++.
-        """
-        if (self._cache_transceiver_config is None
-                or self._cache_transceiver_config.backend is None or
-                self._cache_transceiver_config.transceiver_runtime != "PYTHON"):
-            return
-        if not issubclass(self._kv_cache_manager_cls, KVCacheManager):
-            return
-        if os.environ.get("TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY") is None:
-            os.environ["TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY"] = "1"
-            logger.info(
-                "Python cache transceiver with C++ KV cache manager detected; "
-                "defaulting TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=1 (set it "
-                "to 0 explicitly to disable)")
+        maybe_enable_fabric_memory_for_python_transceiver(
+            self._cache_transceiver_config, self._kv_cache_manager_cls)
 
     def _get_model_kv_cache_manager_cls(
         self,

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from abc import ABC, abstractmethod
-from os import getenv
+from os import environ, getenv
 from typing import Any, Dict, List, Optional
 
 import tensorrt_llm
@@ -30,10 +30,40 @@
 _DISABLE_KV_CACHE_TRANSFER_OVERLAP_ENV = "TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP"
 _DISAGG_LAYERWISE_ENV = "TRTLLM_DISAGG_LAYERWISE"
 _TRY_ZCOPY_FOR_KV_CACHE_TRANSFER_ENV = "TRTLLM_TRY_ZCOPY_FOR_KVCACHE_TRANSFER"
+_KVCACHE_POOL_USE_FABRIC_MEMORY_ENV = "TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY"
 _SUPPORTED_INFLIGHT_CANCEL_NIXL_BACKEND = "UCX"
 _disagg_inflight_cancel_enabled_cache: Optional[bool] = None
 
 
+def maybe_enable_fabric_memory_for_python_transceiver(
+        cache_transceiver_config: Optional[CacheTransceiverConfig],
+        kv_cache_manager_cls: type) -> None:
+    """Default the C++ V1 KV pool to fabric memory for the Python transceiver.
+
+    This must run before any KV pool allocation because the C++ environment
+    getter caches the value on first read. Explicit user settings are always
+    respected.
+
+    Args:
+        cache_transceiver_config: Configuration used to select the cache
+            transceiver runtime and backend.
+        kv_cache_manager_cls: KV-cache manager class to check for C++ V1 pool
+            allocation.
+    """
+    if (cache_transceiver_config is None
+            or cache_transceiver_config.backend is None
+            or cache_transceiver_config.transceiver_runtime != "PYTHON"):
+        return
+    if not issubclass(kv_cache_manager_cls, KVCacheManager):
+        return
+    if getenv(_KVCACHE_POOL_USE_FABRIC_MEMORY_ENV) is None:
+        environ[_KVCACHE_POOL_USE_FABRIC_MEMORY_ENV] = "1"
+        logger.info(
+            "Python cache transceiver with C++ KV cache manager detected; "
+            f"defaulting {_KVCACHE_POOL_USE_FABRIC_MEMORY_ENV}=1 (set it "
+            "to 0 explicitly to disable)")
+
+
 def is_disagg_inflight_cancel_enabled() -> bool:
     """Return whether disaggregated in-flight KV transfer cancellation is enabled."""
     global _disagg_inflight_cancel_enabled_cache