diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 638e404bddf2..3c77f693f0d6 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -366,6 +366,14 @@ def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String
def resultsFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/results*.xml"
downloadResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, resultsFilePath, "${stageName}/"), returnStatus: true, numRetries: 3) == 0
+ // Also download the merged rerun report XML when run_tests.py emits it.
+ // The collectTestResults "Rerun Report" stage globs '*/rerun_results.xml' across
+ // every stage tarball, and the glob above (results*.xml) does NOT match
+ // 'rerun_results.xml'. Without this extra scp, sbatch-based stages drop out of
+ // the cross-stage rerun report.
+ def rerunResultsPath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/rerun_results.xml"
+ Utils.exec(pipeline, script: scpFromRemoteCmd(remote, rerunResultsPath, "${stageName}/"), returnStatus: true, numRetries: 1)
+
// Download perf test results
def perfResultsBasePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}"
def folderListOutput = Utils.exec(
@@ -1576,7 +1584,10 @@ def getPytestBaseCommandLine(
testCmdLine += ["--run-ray"]
}
def unittestMarkExpr = (stageName.startsWith("CPU-")) ? "cpu_only" : "not cpu_only"
- testCmdLine += ["--unittest-markexpr='${unittestMarkExpr}'"]
+ testCmdLine += ["--unittest-markexpr=\"${unittestMarkExpr}\""]
+ if (ENABLE_UPLOAD_TEST_RESULTS) {
+ testCmdLine += ["-o console_output_style=progress-even-when-capture-no"]
+ }
if (extraArgs) {
testCmdLine += extraArgs
}
@@ -1822,25 +1833,24 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
)
// Generate Pytest command
+ // NOTE: the launcher (trtllm-llmapi-launch) is NOT included in
+ // pytestCommand. For multi-node, slurm_run.sh wraps run_tests.py
+ // itself with the launcher so that MPI worker nodes stay alive
+ // across all pytest invocations (regular, isolated, rerun).
String pytestUtil = ""
if (nodeCount > 1) {
pytestUtil = "$llmSrcNode/tensorrt_llm/llmapi/trtllm-llmapi-launch"
}
def uploadPath = "${env.JOB_NAME}/${env.BUILD_NUMBER}"
- def clusterDurationsArgsNode = []
+ def clusterDurationsPathNode = ""
if (useClusterDurations) {
def clusterKey = partition.clusterName.replaceAll('[^a-zA-Z0-9]', '_')
- def clusterDurationsPathNode = "${llmSrcNode}/tests/integration/defs/.test_durations_${clusterKey}"
- clusterDurationsArgsNode = ["--durations-path ${clusterDurationsPathNode}"]
+ clusterDurationsPathNode = "${llmSrcNode}/tests/integration/defs/.test_durations_${clusterKey}"
}
- def extraArgs = [
- "--test-list=$testListPathNode",
- "--splitting-algorithm least_duration",
- "--splits $splits",
- "--group $splitId",
- *clusterDurationsArgsNode,
- ]
+ // test-list/splits/group/durations are now handled by run_tests.py;
+ // only pass S3 upload args as part of the pytest base command.
+ def extraArgs = []
if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) {
extraArgs += [
"--capture=fd",
@@ -1855,9 +1865,10 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
perfMode,
jobWorkspace,
"$jobWorkspace/.coveragerc",
- pytestUtil,
- extraArgs,
+ "", // pytestUtil excluded — launcher wraps run_tests.py instead
+ extraArgs, // test-list/splits/group handled by run_tests.py
).join(" ")
+ def failSignaturesList = trtllm_utils.getFailSignaturesList().join(",")
// Generate Job Launch Script
def container = LLM_DOCKER_IMAGE
@@ -2011,6 +2022,13 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
"export ${varName}=\"${escapedValue}\""
}.join('\n')
+ // Escape pytestCommand for embedding inside bash double-quoted export.
+ // --unittest-markexpr="not cpu_only" contains inner double quotes that
+ // would prematurely close the outer "..." and cause an export error.
+ def safePytestCmd = pytestCommand
+ .replace('\\', '\\\\')
+ .replace('"', '\\"')
+
def scriptLaunchPrefix = """#!/bin/bash
#SBATCH ${exemptionComment}
#SBATCH --output=${slurmJobLogPath}
@@ -2032,8 +2050,14 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
export stageName=$stageName
export perfMode=$perfMode
export resourcePathNode=$resourcePathNode
- export pytestCommand="$pytestCommand"
+ export pytestCommand="$safePytestCmd"
export coverageConfigFile="$coverageConfigFile"
+ export testListPathNode="$testListPathNode"
+ export testSplits="$splits"
+ export testGroup="$splitId"
+ export failSignaturesList="$failSignaturesList"
+ export pytestUtil="$pytestUtil"
+ export testDurationsPath="$clusterDurationsPathNode"
export HF_TOKEN=$HF_TOKEN
if [ -f "${s3SecretKeyPathNode}" ]; then
set +x
@@ -4513,23 +4537,17 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
stage ("[${stageName}] Run Pytest")
{
- def noRegularTests = false
- def noIsolateTests = false
- def rerunFailed = false
-
// When useClusterDurations is set, use a per-cluster durations file keyed on
// partition.clusterName (e.g. "oci-hsg", "dlcluster"). This lets each cluster
// build its own timing baseline so sharding is not skewed by timings collected
// on different hardware. Falls back to the shared .test_durations when unset.
- def clusterDurationsArgs = []
- def clusterDurationsPath = ""
String clusterNameForDurations = null
+ def clusterDurationsPath = ""
if (useClusterDurations) {
def partition = SlurmConfig.resolvePlatform(platform)
def clusterKey = partition.clusterName.replaceAll('[^a-zA-Z0-9]', '_')
clusterNameForDurations = clusterKey
clusterDurationsPath = "${llmSrc}/tests/integration/defs/.test_durations_${clusterKey}"
- clusterDurationsArgs = ["--durations-path ${clusterDurationsPath}"]
}
def testDBList = renderTestDB(pipeline, testList, llmSrc, stageName, null, clusterNameForDurations)
@@ -4542,9 +4560,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
reusePassedTestResults(llmSrc, stageName, "${llmSrc}/tests/integration/test_lists/waives.txt", postTag)
}
- // Process shard test list and create separate files for regular and isolate tests
- def preprocessedLists = processShardTestList(llmSrc, testDBList, splitId, splits, perfMode, clusterDurationsPath)
-
// Test Coverage
def TRTLLM_WHL_PATH = sh(returnStdout: true, script: "pip3 show tensorrt_llm | grep Location | cut -d ' ' -f 2").replaceAll("\\s","")
sh "echo ${TRTLLM_WHL_PATH}"
@@ -4573,10 +4588,9 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
def containerPortNum = GlobalState.PORT_SECTION_SIZE
def uploadPath = UPLOAD_PATH.replaceFirst("sw-tensorrt-generic/llm-artifacts/LLM/", "")
- // Some clusters do not allow dmesg -C so we add || true
- // Temporarily disable to reduce the log size
- // sh 'if [ "$(id -u)" -eq 0 ]; then dmesg -C || true; fi'
- def extraArgs = [*clusterDurationsArgs]
+ // clusterDurationsArgs removed — --durations-path is now passed directly to run_tests.py.
+ // Only S3 upload args remain in extraArgs for the pytest base command.
+ def extraArgs = []
if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) {
extraArgs += [
"--capture=fd",
@@ -4592,16 +4606,11 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
"${WORKSPACE}/${stageName}",
coverageConfigFile,
"", // pytestUtil
- extraArgs, // extraArgs
+ extraArgs, // S3 upload args; test-list/durations handled by run_tests.py
containerPortStart,
containerPortNum
)
- // Only add --test-list if there are regular tests to run
- if (preprocessedLists.regularCount > 0) {
- pytestCommand += ["--test-list=${preprocessedLists.regular}"]
- }
-
def containerPIP_LLM_LIB_PATH = sh(script: "pip3 show tensorrt_llm | grep \"Location\" | awk -F\":\" '{ gsub(/ /, \"\", \$2); print \$2\"/tensorrt_llm/libs\"}'", returnStdout: true).replaceAll("\\s","")
def containerLD_LIBRARY_PATH = sh(script: "echo \${LD_LIBRARY_PATH}", returnStdout: true).replaceAll("\\s","")
if (!containerLD_LIBRARY_PATH.contains("${containerPIP_LLM_LIB_PATH}:")) {
@@ -4616,67 +4625,39 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
string(credentialsId: 'llm_evaltool_repo_url', variable: 'EVALTOOL_REPO_URL')
]) {
sh "env | sort"
- try {
- try {
- if (preprocessedLists.regularCount > 0) {
- sh """
- rm -rf ${stageName}/ && \
- cd ${llmSrc}/tests/integration/defs && \
- ${pytestCommand.join(" ")}
- """
- } else {
- echo "No regular tests to run for stage ${stageName}"
- noRegularTests = true
- sh "mkdir -p ${stageName}"
- // Create an empty results.xml file for consistency
- sh """
- echo '' > ${stageName}/results.xml
- echo '' >> ${stageName}/results.xml
- echo '' >> ${stageName}/results.xml
- echo '' >> ${stageName}/results.xml
- echo '' >> ${stageName}/results.xml
- """
- }
- } catch (InterruptedException e) {
- throw e
- } catch (Exception e) {
- def isRerunFailed = rerunFailedTests(stageName, llmSrc, pytestCommand, "results.xml", "regular")
- if (isRerunFailed) {
- catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
- error "Regular tests failed after rerun attempt"
- }
- rerunFailed = true
- } else if (generateTimeoutTestResultXml(pipeline, stageName)) {
- // Rerun passed but the first run had a timeout: mark this
- // stage FAILURE so "[${stageName}] Run Pytest" turns red,
- // not just the enclosing parent stage.
- catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
- error "Some tests terminated unexpectedly, please check the test report."
- }
- }
- }
- // Run the isolated tests if exists
- if (preprocessedLists.isolateCount > 0) {
- stage ("[${stageName}] Run Pytest (Isolated)") {
- echo "There are ${preprocessedLists.isolateCount} isolated tests to run"
- rerunFailed = runIsolatedTests(preprocessedLists, pytestCommand, llmSrc, stageName) || rerunFailed
- }
- } else {
- echo "No isolated tests to run for stage ${stageName}"
- noIsolateTests = true
- }
+ // Build fail signatures list for rerun eligibility
+ def failSignaturesList = trtllm_utils.getFailSignaturesList().join(",")
- if (noRegularTests && noIsolateTests) {
- error "No tests were executed for stage ${stageName}, please check the test list and test-db rendering result."
- }
- } finally {
- if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) {
- sh """
- python3 ${llmSrc}/tests/test_common/s3_output.py \
- --drain-spool "${WORKSPACE}/${stageName}" || true
- """
- }
+ // Use unified run_tests.py for render + regular + isolated + rerun + merge.
+ // catchError lets execution continue to the report-upload and timeout-XML steps
+ // below even when run_tests.py exits non-zero (test failures), preserving the
+ // same behaviour as main where those steps ran inside a catch block before the
+ // terminal error.
+ catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+ sh """
+ rm -rf ${stageName}/ && \
+ cd ${llmSrc}/tests/integration/defs && \
+ python3 ${llmSrc}/jenkins/scripts/run_tests.py \
+ --render \
+ --test-db-list ${testDBList} \
+ --splits ${splits} \
+ --group ${splitId} \
+ ${perfMode ? '--perf-mode' : ''} \
+ --pytest-base-cmd '${pytestCommand.join(" ")}' \
+ --stage-name ${stageName} \
+ --output-dir ${WORKSPACE}/${stageName} \
+ --working-dir ${llmSrc}/tests/integration/defs \
+ --fail-signatures '${failSignaturesList}' \
+ --max-rerun-tests 5 \
+ ${clusterDurationsPath ? "--durations-path ${clusterDurationsPath}" : ''}
+ """
+ }
+ if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) {
+ sh """
+ python3 ${llmSrc}/tests/test_common/s3_output.py \
+ --drain-spool "${WORKSPACE}/${stageName}" || true
+ """
}
}
@@ -4690,15 +4671,13 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
}
}
- // Generate comprehensive rerun report if any reruns occurred
- stage ("Generate Report") {
- timeout(time: 15, unit: 'MINUTES'){
- generateRerunReport(stageName, llmSrc)
- }
- }
-
- if (rerunFailed) {
- error "Some tests still failed after rerun attempts, please check the test report."
+ // Upload rerun report if generated by run_tests.py
+ if (fileExists("${WORKSPACE}/${stageName}/rerun_results.html")) {
+ trtllm_utils.uploadArtifacts(
+ "${WORKSPACE}/${stageName}/rerun_results.html",
+ "${UPLOAD_PATH}/rerun_reports/${stageName}_rerun_results.html"
+ )
+ echo "Test rerun report: https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/rerun_reports/${stageName}_rerun_results.html"
}
if (fileExists("${stageName}/results-timeout.xml") || generateTimeoutTestResultXml(pipeline, stageName)) {
diff --git a/jenkins/scripts/run_tests.py b/jenkins/scripts/run_tests.py
new file mode 100644
index 000000000000..ff0334a69277
--- /dev/null
+++ b/jenkins/scripts/run_tests.py
@@ -0,0 +1,985 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unified test runner for TensorRT-LLM CI.
+
+Handles regular tests, isolated tests, rerun logic, and result merging
+in a single script that can be invoked from both Groovy (K8s pod) and
+SLURM (slurm_run.sh) execution paths.
+"""
+
+import argparse
+import collections
+import os
+import re
+import shlex
+import subprocess
+import sys
+import time
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+# Allow importing test_rerun from the same directory
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import test_rerun
+
+# Force line-buffered stdout so banners and phase messages appear in real-time
+# interleaved correctly with subprocess (pytest) output, rather than being
+# block-buffered and flushed all at once when the script exits.
+sys.stdout.reconfigure(line_buffering=True)
+
+BANNER_WIDTH = 80
+COLLECT_TIMEOUT_SECONDS = 600
+
+
+def print_banner(title, char="="):
+ """Print a prominent phase banner."""
+ border = char * BANNER_WIDTH
+ print(f"\n{border}")
+ print(f" {title}")
+ print(f"{border}\n")
+
+
+def print_phase_summary(phase, passed, failed, duration_s):
+ """Print a summary line after a phase completes."""
+ status = "PASSED" if failed == 0 else "FAILED"
+ print(f"\n>> [{phase}] {status}: {passed} passed, {failed} failed ({duration_s:.1f}s)")
+
+
+def format_duration(seconds):
+ """Format seconds into a human-readable string."""
+ m, s = divmod(int(seconds), 60)
+ h, m = divmod(m, 60)
+ if h > 0:
+ return f"{h}h {m}m {s}s"
+ if m > 0:
+ return f"{m}m {s}s"
+ return f"{s}s"
+
+
+# ---------------------------------------------------------------------------
+# render_test_list: replaces Groovy processShardTestList()
+# ---------------------------------------------------------------------------
+def render_test_list(test_db_list, working_dir, splits, group, perf_mode, durations_path=None):
+ """Parse test-db list, split into shards, separate regular and isolated tests.
+
+ Args:
+ test_db_list: Path to the test-db rendered test list file.
+ working_dir: Working directory for pytest (tests/integration/defs).
+ splits: Total number of shards.
+ group: This shard's group number (1-based).
+ perf_mode: If True, skip pytest collection and use all tests as regular.
+ durations_path: Optional path to per-cluster .test_durations file for
+ least_duration splitting. Falls back to the default .test_durations
+ when None.
+
+ Returns:
+ Tuple of (regular_list_path, isolate_list_path, regular_count, isolate_count).
+ """
+ original_lines = Path(test_db_list).read_text().splitlines()
+
+ cleaned_lines = []
+ isolation_tests = set()
+
+ for line in original_lines:
+ stripped = line.strip()
+ if not stripped:
+ continue
+ if "ISOLATION" in stripped:
+ cleaned = stripped
+ if "ISOLATION," in cleaned:
+ cleaned = cleaned.replace("ISOLATION,", "").strip()
+ elif ",ISOLATION" in cleaned:
+ cleaned = cleaned.replace(",ISOLATION", "").strip()
+ else:
+ cleaned = cleaned.replace(" ISOLATION", "").strip()
+ isolation_tests.add(cleaned)
+ cleaned_lines.append(cleaned)
+ else:
+ cleaned_lines.append(stripped)
+
+ # Write cleaned test list (without ISOLATION markers)
+ test_db_path = Path(test_db_list)
+ cleaned_file = str(test_db_path.with_stem(test_db_path.stem + "_cleaned"))
+ Path(cleaned_file).write_text("\n".join(cleaned_lines) + "\n" if cleaned_lines else "")
+ print(f"Created cleaned testDBList: {cleaned_file} with {len(cleaned_lines)} lines")
+ print(f"Original testDBList contains {len(isolation_tests)} tests with ISOLATION markers")
+
+ shard_tests = []
+
+ if perf_mode:
+ print("Performance mode enabled - skipping pytest collection, using all tests as regular")
+ else:
+ # Use pytest --collect-only to determine which tests belong to this shard
+ # Clear MPI/SLURM env vars to prevent MPI_Init during collection
+ # (same prefixes as slurm_run.sh and trtllm-llmapi-launch)
+ mpi_prefixes = ("PMI", "PMIX", "MPI", "OMPI", "SLURM", "UCX")
+ env_vars = {
+ k: v for k, v in os.environ.items() if not any(k.startswith(p) for p in mpi_prefixes)
+ }
+ collect_output_dir = os.path.join(os.path.dirname(test_db_list), "collect_output")
+ os.makedirs(collect_output_dir, exist_ok=True)
+ durations_arg = f" --durations-path={durations_path}" if durations_path else ""
+ collect_cmd = (
+ f"pytest --collect-only --splitting-algorithm least_duration "
+ f"--test-list={cleaned_file} --quiet "
+ f"--splits {splits} --group {group} "
+ f"--output-dir={collect_output_dir}"
+ f"{durations_arg}"
+ )
+ print(f"Running: {collect_cmd}")
+ try:
+ result = subprocess.run(
+ collect_cmd,
+ shell=True,
+ capture_output=True,
+ text=True,
+ cwd=working_dir,
+ env=env_vars,
+ timeout=COLLECT_TIMEOUT_SECONDS,
+ )
+ except subprocess.TimeoutExpired as e:
+ partial_stdout = (e.stdout or "").strip() if isinstance(e.stdout, str) else ""
+ partial_stderr = (e.stderr or "").strip() if isinstance(e.stderr, str) else ""
+ print(
+ f"Error: pytest --collect-only timed out after {COLLECT_TIMEOUT_SECONDS}s "
+ f"(possible import-time deadlock)"
+ )
+ if partial_stdout:
+ print(f"partial stdout:\n{partial_stdout}")
+ if partial_stderr:
+ print(f"partial stderr:\n{partial_stderr}")
+ sys.exit(1)
+
+ output = result.stdout.strip()
+ print("<<>>")
+ print(output)
+ print("<<>>")
+
+ if result.stderr:
+ print(f"pytest --collect-only stderr:\n{result.stderr}")
+
+ if result.returncode != 0:
+ print(f"Error: pytest --collect-only failed with exit code {result.returncode}")
+ print(f"stderr: {result.stderr}")
+ sys.exit(1)
+
+ # Parse output: collect test IDs after "Running N items in this shard" line
+ found_running_line = False
+ for line in output.split("\n"):
+ if re.search(r"Running \d+ items in this shard", line) or re.search(
+ r"\[pytest-split\] Running group", line
+ ):
+ found_running_line = True
+ continue
+ if found_running_line and "======================" in line:
+ found_running_line = False
+ continue
+ if found_running_line and "::" in line:
+ shard_tests.append(line.strip())
+
+ print(f"Filtering complete. shard_tests size: {len(shard_tests)}")
+
+ # Split into regular and isolate
+ regular_tests = []
+ isolate_tests = []
+
+ if perf_mode:
+ regular_tests = [t for t in cleaned_lines if t.strip()]
+ else:
+ for test in shard_tests:
+ trimmed = test.strip()
+ if not trimmed:
+ continue
+ # Handle test_unittests.py::test_unittests_v2[xxxx] pattern
+ if trimmed.startswith("test_unittests.py::test_unittests_v2[") and trimmed.endswith(
+ "]"
+ ):
+ start = trimmed.index("[") + 1
+ end = trimmed.rindex("]")
+ trimmed = trimmed[start:end]
+
+ # Check if this test is in the isolation set
+ isolation_match = next((t for t in isolation_tests if trimmed in t), None)
+ if isolation_match:
+ isolate_tests.append(isolation_match)
+ else:
+ cleaned_match = next((t for t in cleaned_lines if trimmed in t), None)
+ if cleaned_match:
+ regular_tests.append(cleaned_match)
+
+ # Write regular and isolate list files
+ regular_file = str(test_db_path.with_stem(test_db_path.stem + "_regular"))
+ isolate_file = str(test_db_path.with_stem(test_db_path.stem + "_isolate"))
+
+ Path(regular_file).write_text("\n".join(regular_tests) + "\n" if regular_tests else "")
+ Path(isolate_file).write_text("\n".join(isolate_tests) + "\n" if isolate_tests else "")
+
+ print(f"Created {regular_file} with {len(regular_tests)} regular tests")
+ print(f"Created {isolate_file} with {len(isolate_tests)} isolate tests")
+
+ return regular_file, isolate_file, len(regular_tests), len(isolate_tests)
+
+
+# ---------------------------------------------------------------------------
+# Core pytest execution
+# ---------------------------------------------------------------------------
+def run_pytest(pytest_cmd, working_dir, tail_lines=0):
+ """Execute a pytest command and return the exit code and duration.
+
+ Args:
+ pytest_cmd: The full pytest command string.
+ working_dir: Working directory for pytest execution.
+ tail_lines: When > 0, stream output line-by-line via Popen, keep the
+ last N lines in a buffer, and return them as the third tuple
+ element. Use for isolation tests so collection errors that
+ disappear between the session header and the summary line are
+ captured and can be re-printed in the [FAST FAILURE] banner.
+
+ Returns:
+ Tuple of (exit_code, duration_seconds, captured_tail: list[str]).
+ captured_tail is an empty list when tail_lines == 0.
+ """
+ print(f"Running pytest: {pytest_cmd}")
+ sys.stdout.flush()
+ t0 = time.monotonic()
+
+ if tail_lines > 0:
+ buf = collections.deque(maxlen=tail_lines)
+ # PYTHONUNBUFFERED=1 prevents pytest from block-buffering stdout when
+ # it detects a pipe (non-tty), which would otherwise delay or lose
+ # collection-error output before the process exits.
+ env = {**os.environ, "PYTHONUNBUFFERED": "1"}
+ proc = subprocess.Popen(
+ pytest_cmd,
+ shell=True,
+ cwd=working_dir,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ bufsize=1,
+ env=env,
+ )
+ for line in proc.stdout:
+ sys.stdout.write(line)
+ sys.stdout.flush()
+ buf.append(line.rstrip("\n"))
+ proc.wait()
+ rc = proc.returncode
+ elapsed = time.monotonic() - t0
+ print(f"Pytest finished with exit code {rc} ({format_duration(elapsed)})")
+ return rc, elapsed, list(buf)
+
+ result = subprocess.run(pytest_cmd, shell=True, cwd=working_dir, stderr=subprocess.STDOUT)
+ elapsed = time.monotonic() - t0
+ print(f"Pytest finished with exit code {result.returncode} ({format_duration(elapsed)})")
+ return result.returncode, elapsed, []
+
+
+# Isolation tests that finish in under this many seconds almost certainly
+# failed before the test body ran (server not ready, import error, etc.).
+_FAST_FAILURE_THRESHOLD_SECS = 120
+
+
+def print_xml_failure_summary(xml_path, tail_lines=None):
+ """Parse a pytest result XML and print failure/error messages.
+
+ Called after a fast isolation-test failure to surface the root cause
+ without requiring manual inspection of the SLURM job log.
+
+ Args:
+ xml_path: Path to the pytest JUnit XML result file.
+ tail_lines: Optional list of captured pytest output lines. Printed
+ when no XML is found (collection error) so the error is visible
+ in the [FAST FAILURE] banner even without a result file.
+ """
+ if not os.path.exists(xml_path):
+ print(" [No result XML — pytest exited before generating results]")
+ if tail_lines:
+ print(" [Last pytest output (collection error likely above):]")
+ for line in tail_lines:
+ print(f" {line}")
+ return
+ try:
+ root = ET.parse(xml_path).getroot()
+ except (OSError, ET.ParseError) as exc:
+ print(f" [Could not parse result XML: {exc}]")
+ return
+
+ cases = root.findall(".//testcase")
+ if not cases:
+ print(" [Result XML contains no test cases — 0 tests were collected]")
+ return
+
+ printed = 0
+ for tc in cases:
+ for tag in ("failure", "error"):
+ node = tc.find(tag)
+ if node is None:
+ continue
+ name = f"{tc.get('classname', '')}.{tc.get('name', '')}"
+ msg = (node.get("message") or "").strip()
+ detail = (node.text or "").strip()
+ print(f" [{tag.upper()}] {name}")
+ if msg:
+ print(f" message : {msg[:600]}")
+ if detail:
+ lines = detail.splitlines()
+ excerpt = "\n ".join(lines[-30:]) # last 30 lines
+ print(f" detail :\n {excerpt}")
+ printed += 1
+ if printed == 0:
+ print(" [No failure/error nodes found in result XML]")
+
+
+def rebuild_pytest_command(base_cmd, drop_patterns, append_args):
+ """Drop args matching any prefix in drop_patterns, then append append_args.
+
+ For tokens of the form ``--flag=value`` the single token is dropped; for
+ ``--flag value`` the following value token is also dropped (when it does
+ not look like another flag).
+
+ Shell pipeline prefixes (e.g. ``unset VAR && ``) are preserved verbatim
+ so that shlex.join does not quote ``&&`` into a literal argument.
+
+ Args:
+ base_cmd: The original pytest command string.
+ drop_patterns: Argument prefixes to strip from base_cmd.
+ append_args: Tokens to append after stripping (already individually
+ shell-safe; quoting is handled by ``shlex.join``).
+
+ Returns:
+ Rebuilt pytest command string.
+ """
+ # Preserve shell prefix (everything before the last " && ") verbatim so
+ # shlex.join never quotes && into '&&' and breaks the pipeline.
+ shell_prefix = ""
+ cmd_part = base_cmd
+ if " && " in base_cmd:
+ and_parts = base_cmd.split(" && ")
+ shell_prefix = " && ".join(and_parts[:-1]) + " && "
+ cmd_part = and_parts[-1]
+
+ parts = []
+ tokens = shlex.split(cmd_part)
+ skip_next = False
+ for i, token in enumerate(tokens):
+ if skip_next:
+ skip_next = False
+ continue
+ if any(token.startswith(p) for p in drop_patterns):
+ if "=" not in token and i + 1 < len(tokens) and not tokens[i + 1].startswith("-"):
+ skip_next = True
+ continue
+ parts.append(token)
+ parts.extend(append_args)
+ return shell_prefix + shlex.join(parts)
+
+
+def build_rerun_command(base_cmd, test_list, xml_path, csv_path, reruns):
+ """Build a rerun pytest command by stripping split/cov args and replacing test-list/output args.
+
+ Args:
+ base_cmd: The original pytest command string.
+ test_list: Path to the rerun test list file.
+ xml_path: Path for the rerun results XML.
+ csv_path: Path for the rerun results CSV.
+ reruns: Number of reruns (passed to --reruns flag).
+
+ Returns:
+ Modified pytest command string.
+ """
+ drop_patterns = [
+ "--splitting-algorithm",
+ "--splits",
+ "--group",
+ "--cov",
+ "--test-list",
+ "--csv",
+ "--periodic-junit-xmlpath",
+ ]
+ append_args = [
+ f"--test-list={test_list}",
+ f"--csv={csv_path}",
+ "--periodic-junit-xmlpath",
+ xml_path,
+ ]
+ if reruns > 0:
+ append_args.extend(["--reruns", str(reruns)])
+ return rebuild_pytest_command(base_cmd, drop_patterns, append_args)
+
+
+def build_isolated_command(base_cmd, test_list, xml_path, csv_path):
+ """Build an isolated test command by replacing test-list/output/prefix args.
+
+ Args:
+ base_cmd: The original pytest command string.
+ test_list: Path to the single-test list file.
+ xml_path: Path for the isolated results XML.
+ csv_path: Path for the isolated results CSV.
+
+ Returns:
+ Modified pytest command string.
+ """
+ drop_patterns = ["--test-list", "--test-prefix", "--csv", "--periodic-junit-xmlpath"]
+ append_args = [
+ f"--test-list={test_list}",
+ f"--csv={csv_path}",
+ "--periodic-junit-xmlpath",
+ xml_path,
+ "--cov-append",
+ ]
+ return rebuild_pytest_command(base_cmd, drop_patterns, append_args)
+
+
+# ---------------------------------------------------------------------------
+# Rerun logic: replaces Groovy rerunFailedTests()
+# ---------------------------------------------------------------------------
+def check_and_rerun(
+ result_xml,
+ base_cmd,
+ working_dir,
+ output_dir,
+ rerun_tag,
+ fail_signatures,
+ max_rerun_tests,
+ test_list_file,
+):
+ """Analyze test failures and rerun eligible tests.
+
+ Args:
+ result_xml: Path to the results XML to analyze.
+ base_cmd: The original pytest command string.
+ working_dir: Working directory for pytest execution.
+ output_dir: Base output directory for the stage.
+ rerun_tag: Tag for this rerun (e.g., "regular", "isolated_0").
+ fail_signatures: List of failure signature strings.
+ max_rerun_tests: Max number of failed tests that can trigger rerun.
+ test_list_file: Path to the test list file that drove this pytest run
+ (used to detect tests that never ran).
+
+ Returns:
+ Tuple of (is_rerun_failed: bool, rerun_xml_files: list of paths to rerun result XMLs).
+ """
+ if not os.path.exists(result_xml):
+ print(f"No {result_xml} file found, skip the rerun step")
+ return True, []
+
+ rerun_dir = os.path.join(output_dir, "rerun", rerun_tag)
+ os.makedirs(rerun_dir, exist_ok=True)
+
+ # Step 1: Generate rerun test lists
+ print(f"Generating rerun test lists for {rerun_tag}...")
+ unfinished_test_file = os.path.join(output_dir, "unfinished_test.txt")
+ test_rerun.generate_rerun_tests_list(
+ rerun_dir,
+ result_xml,
+ fail_signatures,
+ test_list_file,
+ unfinished_test_file,
+ )
+
+ # Step 2: Count total failed tests and enforce the cap
+ valid_count = 0
+ for times in [1, 2]:
+ rerun_file = os.path.join(rerun_dir, f"rerun_{times}.txt")
+ if os.path.exists(rerun_file):
+ lines = [line for line in Path(rerun_file).read_text().splitlines() if line.strip()]
+ count = len(lines)
+ print(f"Found {count} {rerun_tag} tests to rerun {times} time(s)")
+ valid_count += count
+
+ if valid_count > max_rerun_tests:
+ print(
+ f"Too many {rerun_tag} tests to rerun ({valid_count} > max_rerun_tests={max_rerun_tests}). "
+ "Skipping rerun."
+ )
+ return True, []
+
+ # Step 3: Execute reruns
+ is_rerun_failed = False
+ rerun_xml_files = []
+
+ for times in [1, 2]:
+ rerun_list = os.path.join(rerun_dir, f"rerun_{times}.txt")
+ if not os.path.exists(rerun_list):
+ print(f"No failed {rerun_tag} tests need to be rerun {times} time(s)")
+ continue
+
+ print(f"Rerun test list ({times}):")
+ print(Path(rerun_list).read_text())
+
+ xml_file = os.path.join(rerun_dir, f"rerun_results_{times}.xml")
+ csv_file = os.path.join(rerun_dir, f"rerun_report_{times}.csv")
+
+ rerun_cmd = build_rerun_command(base_cmd, rerun_list, xml_file, csv_file, times - 1)
+ rc, _, _tail = run_pytest(rerun_cmd, working_dir)
+
+ if os.path.exists(xml_file):
+ rerun_xml_files.append(xml_file)
+
+ if rc != 0:
+ if not os.path.exists(xml_file):
+ print(f"The {rerun_tag} tests crashed during rerun attempt (no XML produced).")
+ raise RuntimeError(f"Rerun crashed for {rerun_tag}, no XML produced")
+ print(f"The {rerun_tag} tests still failed after rerun attempt.")
+ is_rerun_failed = True
+
+ return is_rerun_failed, rerun_xml_files
+
+
+# ---------------------------------------------------------------------------
+# Regular test execution
+# ---------------------------------------------------------------------------
+def run_regular_tests(
+ pytest_cmd, regular_list, working_dir, output_dir, stage_name, fail_signatures, max_rerun_tests
+):
+ """Run regular tests and rerun failures if applicable.
+
+ Args:
+ pytest_cmd: The full pytest command string (already includes --test-list).
+ regular_list: Path to the regular test list file.
+ working_dir: Working directory for pytest execution.
+ output_dir: Output directory for the stage.
+ stage_name: Name of the test stage.
+ fail_signatures: List of failure signature strings.
+ max_rerun_tests: Max number of failed tests that can trigger rerun.
+
+ Returns:
+ Tuple of (rerun_failed: bool, all_xml_files: list of result XML paths).
+ """
+ regular_count = len(
+ [line for line in Path(regular_list).read_text().splitlines() if line.strip()]
+ )
+ print_banner(f"REGULAR TESTS — {regular_count} test(s) for stage [{stage_name}]")
+
+ result_xml = os.path.join(output_dir, "results.xml")
+ all_xml_files = [result_xml]
+
+ rc, elapsed, _tail = run_pytest(pytest_cmd, working_dir)
+
+ if rc != 0:
+ print_phase_summary("REGULAR TESTS", 0, 1, elapsed)
+ print_banner(
+ f"RERUN — checking failed regular tests for stage [{stage_name}]",
+ char="-",
+ )
+ try:
+ is_rerun_failed, rerun_xmls = check_and_rerun(
+ result_xml,
+ pytest_cmd,
+ working_dir,
+ output_dir,
+ "regular",
+ fail_signatures,
+ max_rerun_tests,
+ regular_list,
+ )
+ all_xml_files.extend(rerun_xmls)
+ return is_rerun_failed, all_xml_files
+ except RuntimeError as e:
+ print(f"Regular tests rerun crashed: {e}")
+ return True, all_xml_files
+
+ print_phase_summary("REGULAR TESTS", regular_count, 0, elapsed)
+ return False, all_xml_files
+
+
+# ---------------------------------------------------------------------------
+# Isolated test execution: replaces Groovy runIsolatedTests()
+# ---------------------------------------------------------------------------
+def run_isolated_tests(
+ pytest_cmd, isolate_list, working_dir, output_dir, stage_name, fail_signatures, max_rerun_tests
+):
+ """Run isolated tests one by one, rerunning each on failure.
+
+ Args:
+ pytest_cmd: The base pytest command string.
+ isolate_list: Path to the isolate test list file.
+ working_dir: Working directory for pytest execution.
+ output_dir: Output directory for the stage.
+ stage_name: Name of the test stage.
+ fail_signatures: List of failure signature strings.
+ max_rerun_tests: Max number of failed tests that can trigger rerun.
+
+ Returns:
+ Tuple of (rerun_failed: bool, all_xml_files: list of result XML paths).
+ """
+ isolate_tests = [
+ line.strip() for line in Path(isolate_list).read_text().splitlines() if line.strip()
+ ]
+
+ print_banner(f"ISOLATION TESTS — {len(isolate_tests)} test(s) for stage [{stage_name}]")
+
+ rerun_failed = False
+ all_xml_files = []
+ iso_passed = 0
+ iso_failed = 0
+ t0 = time.monotonic()
+
+ for i, test_name in enumerate(isolate_tests):
+ print(f"\n--- [ISOLATION {i + 1}/{len(isolate_tests)}] {test_name} ---")
+
+ # Create a temporary file for this single test
+ single_test_file = os.path.join(output_dir, f"isolated_{i}.txt")
+ Path(single_test_file).write_text(test_name + "\n")
+
+ xml_path = os.path.join(output_dir, f"results_isolated_{i}.xml")
+ csv_path = os.path.join(output_dir, f"report_isolated_{i}.csv")
+
+ isolated_cmd = build_isolated_command(pytest_cmd, single_test_file, xml_path, csv_path)
+ rc, elapsed, pytest_tail = run_pytest(isolated_cmd, working_dir, tail_lines=200)
+ all_xml_files.append(xml_path)
+
+ if rc != 0:
+ if elapsed < _FAST_FAILURE_THRESHOLD_SECS:
+ print(
+ f"\n [FAST FAILURE] Isolation test {i + 1} exited in "
+ f"{format_duration(elapsed)} (< {_FAST_FAILURE_THRESHOLD_SECS}s). "
+ f"Likely failed before the test body ran (server not ready, "
+ f"collection error, etc.)."
+ )
+ print_xml_failure_summary(xml_path, tail_lines=pytest_tail)
+ print_banner(
+ f"RERUN — isolated test {i + 1}/{len(isolate_tests)}: {test_name}",
+ char="-",
+ )
+ try:
+ is_rerun_failed, rerun_xmls = check_and_rerun(
+ xml_path,
+ isolated_cmd,
+ working_dir,
+ output_dir,
+ f"isolated_{i}",
+ fail_signatures,
+ max_rerun_tests,
+ single_test_file,
+ )
+ all_xml_files.extend(rerun_xmls)
+ if is_rerun_failed:
+ print(f"Isolated test {i + 1} ({test_name}) failed after rerun attempt")
+ rerun_failed = True
+ iso_failed += 1
+ else:
+ iso_passed += 1
+ except RuntimeError as e:
+ print(f"Isolated test {i + 1} ({test_name}) rerun crashed: {e}")
+ rerun_failed = True
+ iso_failed += 1
+ else:
+ iso_passed += 1
+
+ # Clean up temporary file
+ try:
+ os.remove(single_test_file)
+ except OSError:
+ pass
+
+ elapsed = time.monotonic() - t0
+ print_phase_summary("ISOLATION TESTS", iso_passed, iso_failed, elapsed)
+
+ return rerun_failed, all_xml_files
+
+
+# ---------------------------------------------------------------------------
+# Result merging: replaces Groovy generateRerunReport()
+# ---------------------------------------------------------------------------
+def merge_results(output_dir, stage_name, all_xml_files):
+ """Merge all result XMLs and generate rerun report.
+
+ Args:
+ output_dir: Output directory for the stage.
+ stage_name: Name of the test stage.
+ all_xml_files: List of all result XML file paths.
+ """
+ # Fix testsuite names in all XMLs
+ for xml_file in all_xml_files:
+ if not os.path.exists(xml_file):
+ continue
+ try:
+ tree = ET.parse(xml_file)
+ except (OSError, ET.ParseError) as e:
+ print(f"Warning: Failed to parse {xml_file}: {e}")
+ continue
+ modified = False
+ for ts in tree.iter("testsuite"):
+ if ts.get("name") == "pytest":
+ ts.set("name", stage_name)
+ modified = True
+ if modified:
+ try:
+ tree.write(xml_file, encoding="utf-8", xml_declaration=True)
+ except OSError as e:
+ print(f"Warning: Failed to write {xml_file}: {e}")
+
+ # Separate original results and rerun results
+ rerun_result_files = []
+ for xml_file in all_xml_files:
+ if not os.path.exists(xml_file):
+ continue
+ # Rerun results live under rerun/ directory
+ if "/rerun/" in xml_file and "rerun_results_" in xml_file:
+ rerun_result_files.append(xml_file)
+
+ # Also add original results that have corresponding reruns to the rerun report
+ rerun_base_dir = os.path.join(output_dir, "rerun")
+ original_results_with_reruns = []
+
+ # Check regular reruns
+ regular_rerun_dir = os.path.join(rerun_base_dir, "regular")
+ if os.path.isdir(regular_rerun_dir):
+ has_regular_reruns = any(
+ f.startswith("rerun_results_") and f.endswith(".xml")
+ for f in os.listdir(regular_rerun_dir)
+ )
+ if has_regular_reruns:
+ results_xml = os.path.join(output_dir, "results.xml")
+ if os.path.exists(results_xml):
+ original_results_with_reruns.append(results_xml)
+
+ # Check isolated reruns
+ if os.path.isdir(rerun_base_dir):
+ for d in os.listdir(rerun_base_dir):
+ if d.startswith("isolated_") and os.path.isdir(os.path.join(rerun_base_dir, d)):
+ iso_dir = os.path.join(rerun_base_dir, d)
+ has_iso_reruns = any(
+ f.startswith("rerun_results_") and f.endswith(".xml")
+ for f in os.listdir(iso_dir)
+ )
+ if has_iso_reruns:
+ iso_num = d.replace("isolated_", "")
+ iso_result = os.path.join(output_dir, f"results_isolated_{iso_num}.xml")
+ if os.path.exists(iso_result):
+ original_results_with_reruns.append(iso_result)
+
+ # Generate rerun report if any reruns occurred
+ rerun_report_inputs = original_results_with_reruns + rerun_result_files
+ if rerun_report_inputs:
+ print(f"Generating rerun report with input files: {rerun_report_inputs}")
+ rerun_report_xml = os.path.join(output_dir, "rerun_results.xml")
+ test_rerun.generate_rerun_report(rerun_report_xml, rerun_report_inputs)
+
+ # Merge all XMLs into a single results.xml for junit
+ existing_xml_files = [f for f in all_xml_files if os.path.exists(f)]
+ if existing_xml_files:
+ merged_output = os.path.join(output_dir, "results.xml")
+ print(f"Merging {len(existing_xml_files)} XML files into {merged_output}")
+ test_rerun.merge_junit_xmls(merged_output, existing_xml_files, deduplicate=True)
+
+ # Remove isolation results since they are merged into results.xml
+ for f in all_xml_files:
+ if "results_isolated_" in f and os.path.exists(f):
+ try:
+ os.remove(f)
+ except OSError:
+ pass
+
+ print("Result merging completed")
+
+
+# ---------------------------------------------------------------------------
+# Create empty results XML
+# ---------------------------------------------------------------------------
+def create_empty_results_xml(output_dir, stage_name):
+ """Create an empty results.xml for stages with no tests.
+
+ Args:
+ output_dir: Output directory for the stage.
+ stage_name: Name of the test stage.
+ """
+ os.makedirs(output_dir, exist_ok=True)
+ content = (
+ '\n'
+ "\n"
+ f'\n'
+ "\n"
+ "\n"
+ )
+ Path(os.path.join(output_dir, "results.xml")).write_text(content)
+
+
+# ---------------------------------------------------------------------------
+# Main entry point
+# ---------------------------------------------------------------------------
+def parse_args():
+ parser = argparse.ArgumentParser(description="Unified TensorRT-LLM test runner")
+
+ # Render mode arguments
+ parser.add_argument(
+ "--render",
+ action="store_true",
+ help="Run render_test_list to split test-db list into regular/isolate shards",
+ )
+ parser.add_argument("--test-db-list", help="Path to the test-db rendered test list file")
+ parser.add_argument("--splits", type=int, default=1, help="Total number of shards")
+ parser.add_argument("--group", type=int, default=1, help="This shard group number (1-based)")
+ parser.add_argument(
+ "--perf-mode", action="store_true", help="Performance mode: skip collection, run all tests"
+ )
+
+ # Run mode arguments
+ parser.add_argument("--pytest-base-cmd", help="Base pytest command string")
+ parser.add_argument(
+ "--regular-test-list", help="Path to regular test list (if not using --render)"
+ )
+ parser.add_argument(
+ "--isolate-test-list", help="Path to isolate test list (if not using --render)"
+ )
+ parser.add_argument("--stage-name", required=True, help="Name of the test stage")
+ parser.add_argument("--output-dir", required=True, help="Output directory for the stage")
+ parser.add_argument("--working-dir", help="Working directory for pytest (defaults to cwd)")
+ parser.add_argument(
+ "--fail-signatures",
+ default="",
+ help="Comma-separated list of failure signatures for rerun eligibility",
+ )
+ parser.add_argument(
+ "--max-rerun-tests", type=int, default=5, help="Max failed tests to trigger rerun"
+ )
+ parser.add_argument(
+ "--durations-path",
+ default=None,
+ help="Path to per-cluster .test_durations file for least_duration splitting. "
+ "Overrides the default .test_durations when set.",
+ )
+
+ return parser.parse_args()
+
+
+def main():
+ args = parse_args()
+
+ working_dir = args.working_dir or os.getcwd()
+ output_dir = args.output_dir
+ stage_name = args.stage_name
+ fail_signatures = [s for s in args.fail_signatures.split(",") if s]
+ max_rerun_tests = args.max_rerun_tests
+
+ os.makedirs(output_dir, exist_ok=True)
+
+ # Step 1: Render test list if requested
+ regular_list = args.regular_test_list
+ isolate_list = args.isolate_test_list
+ regular_count = 0
+ isolate_count = 0
+
+ if args.render:
+ if not args.test_db_list:
+ print("Error: --test-db-list is required when using --render")
+ sys.exit(1)
+ regular_list, isolate_list, regular_count, isolate_count = render_test_list(
+ args.test_db_list,
+ working_dir,
+ args.splits,
+ args.group,
+ args.perf_mode,
+ args.durations_path,
+ )
+ else:
+ if regular_list and os.path.exists(regular_list):
+ regular_count = len(
+ [line for line in Path(regular_list).read_text().splitlines() if line.strip()]
+ )
+ if isolate_list and os.path.exists(isolate_list):
+ isolate_count = len(
+ [line for line in Path(isolate_list).read_text().splitlines() if line.strip()]
+ )
+
+ if not args.pytest_base_cmd:
+ # Render-only mode: just output the lists and exit
+ if args.render:
+ print(f"regular_list={regular_list}")
+ print(f"isolate_list={isolate_list}")
+ print(f"regular_count={regular_count}")
+ print(f"isolate_count={isolate_count}")
+ sys.exit(0)
+ else:
+ print("Error: --pytest-base-cmd is required for test execution")
+ sys.exit(1)
+
+ pytest_base_cmd = args.pytest_base_cmd
+ total_t0 = time.monotonic()
+
+ print_banner(
+ f"TEST EXECUTION START — stage [{stage_name}] "
+ f"({regular_count} regular, {isolate_count} isolation)"
+ )
+
+ # Step 2: Run regular tests
+ all_xml_files = []
+ rerun_failed = False
+
+ if regular_count > 0:
+ # Add --test-list to the base command for regular tests
+ regular_cmd = f"{pytest_base_cmd} --test-list={regular_list}"
+ failed, xml_files = run_regular_tests(
+ regular_cmd,
+ regular_list,
+ working_dir,
+ output_dir,
+ stage_name,
+ fail_signatures,
+ max_rerun_tests,
+ )
+ rerun_failed = rerun_failed or failed
+ all_xml_files.extend(xml_files)
+ else:
+ print(f"No regular tests to run for stage {stage_name}")
+ create_empty_results_xml(output_dir, stage_name)
+ all_xml_files.append(os.path.join(output_dir, "results.xml"))
+
+ # Step 3: Run isolated tests
+ if isolate_count > 0:
+ failed, xml_files = run_isolated_tests(
+ pytest_base_cmd,
+ isolate_list,
+ working_dir,
+ output_dir,
+ stage_name,
+ fail_signatures,
+ max_rerun_tests,
+ )
+ rerun_failed = rerun_failed or failed
+ all_xml_files.extend(xml_files)
+ else:
+ print(f"No isolated tests to run for stage {stage_name}")
+
+ # Step 4: Check that at least some tests were executed
+ if regular_count == 0 and isolate_count == 0:
+ print(f"Error: No tests were executed for stage {stage_name}")
+ sys.exit(1)
+
+ # Step 5: Merge all results
+ print_banner("RESULT MERGING", char="-")
+ merge_results(output_dir, stage_name, all_xml_files)
+
+ # Step 6: Final summary and exit
+ total_elapsed = time.monotonic() - total_t0
+ print_banner(f"TEST EXECUTION COMPLETE — stage [{stage_name}]")
+ print(f" Regular tests : {regular_count}")
+ print(f" Isolation tests : {isolate_count}")
+ print(f" Total time : {format_duration(total_elapsed)}")
+
+ if rerun_failed:
+ print(" Result : FAILED")
+ print("\nSome tests still failed after rerun attempts, please check the test report.")
+ sys.exit(1)
+
+ print(" Result : PASSED")
+ print("\nAll tests passed.")
+ sys.exit(0)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh
index 4aecf0d98864..68e436aaf4b9 100755
--- a/jenkins/scripts/slurm_run.sh
+++ b/jenkins/scripts/slurm_run.sh
@@ -1,4 +1,18 @@
#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
# Set up error handling
set -xEeuo pipefail
@@ -79,31 +93,45 @@ pytest_exit_code=0
perf_check_exit_code=0
perf_report_exit_code=0
-eval $pytestCommand
+# Use unified run_tests.py for test execution, rerun, and result merging
+runTestsScript="$llmSrcNode/jenkins/scripts/run_tests.py"
+
+# Build the run_tests.py arguments array for proper quoting
+runTestsArgs=(
+ python3 "$runTestsScript"
+ --render
+ --test-db-list "$testListPathNode"
+ --splits "${testSplits:-1}"
+ --group "${testGroup:-1}"
+ --pytest-base-cmd "$pytestCommand"
+ --stage-name "$stageName"
+ --output-dir "$jobWorkspace"
+ --working-dir "$llmSrcNode/tests/integration/defs"
+ --fail-signatures "${failSignaturesList:-}"
+ --max-rerun-tests 5
+)
+if [ -n "${testDurationsPath:-}" ]; then
+ runTestsArgs+=(--durations-path "$testDurationsPath")
+fi
+if [ "$perfMode" = "true" ]; then
+ runTestsArgs+=(--perf-mode)
+fi
+
+# For multi-node runs, wrap run_tests.py with the MPI launcher so that
+# worker nodes stay alive across all pytest invocations (regular, isolated,
+# rerun). The launcher ensures rank 0 executes run_tests.py while workers
+# run mgmn_worker_node for the entire duration.
+if [ -n "${pytestUtil:-}" ] && [ "${SLURM_JOB_NUM_NODES:-1}" -gt 1 ]; then
+ echo "Multi-node mode: wrapping run_tests.py with $pytestUtil"
+ "$pytestUtil" "${runTestsArgs[@]}"
+else
+ "${runTestsArgs[@]}"
+fi
pytest_exit_code=$?
-echo "Rank${SLURM_PROCID} Pytest finished execution with exit code $pytest_exit_code"
+echo "Rank${SLURM_PROCID} run_tests.py finished execution with exit code $pytest_exit_code"
python3 "$llmSrcNode/tests/test_common/s3_output.py" \
--drain-spool "$jobWorkspace" || true
-# DEBUG: Diagnose intermittent "unrecognized arguments" failure (Exit Code 4)
-# Remove this after the issue is resolved
-if [ $pytest_exit_code -eq 4 ]; then
- echo "DEBUG: Pytest failed with usage error (exit code 4)"
- echo "DEBUG: Directory state at $(pwd):"
- ls -l
- echo "DEBUG: Directory state at $llmSrcNode/tests/integration/defs:"
- ls -l $llmSrcNode/tests/integration/defs
-
- echo "DEBUG: conftest.py content:"
- md5sum $llmSrcNode/tests/integration/defs/conftest.py
-
- echo "DEBUG: pytest.ini content:"
- md5sum $llmSrcNode/tests/integration/defs/pytest.ini
-
- echo "DEBUG: Check importability of conftest.py"
- python3 -c "import sys; sys.path.insert(0, '.'); import conftest; print('DEBUG: conftest imported successfully')"
-fi
-
if [ $SLURM_PROCID -eq 0 ] && [ "$perfMode" = "true" ]; then
# Only PyTorch perf stages remain; the TensorRT perf baseline was removed.
basePerfFilename="base_perf_pytorch.csv"
diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml
index a2c11d454409..30375cdbb492 100644
--- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml
+++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml
@@ -15,3 +15,8 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2:
backend: pytorch
tests:
- perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90)
+ - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90), ISOLATION
+ - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90), ISOLATION
+ - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90), ISOLATION
+ - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90), ISOLATION
+ - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90), ISOLATION