From a23cad3c644a19cceacf7235603f9854e8b5ca02 Mon Sep 17 00:00:00 2001 From: EmmaQiaoCh Date: Wed, 12 Aug 2026 10:42:29 +0300 Subject: [PATCH 1/5] [None][infra] Unify test execution under run_tests.py for K8s and SLURM paths Introduce jenkins/scripts/run_tests.py as a unified test runner that handles render, regular tests, isolation tests, rerun, and XML merge in a single invocation for both K8s (Blossom) and SLURM (sbatch/agent) CI paths. Key changes: - jenkins/scripts/run_tests.py: new -- unified runner; Popen tail-capture for collection errors, fail-signatures rerun eligibility, XML merge - jenkins/scripts/slurm_run.sh: replace eval $pytestCommand with run_tests.py runTestsArgs array; MPI launcher wraps run_tests.py for multi-node - jenkins/L0_Test.groovy: K8s path calls run_tests.py; sbatch path passes test-list/splits/group/durations via env vars to slurm_run.sh; markExpr fix (double-quote syntax + 'and not disabled' for CPU stages); adopt main's !testFilter[(DETAILED_LOG)] guard for S3 upload args Signed-off-by: EmmaQiaoCh --- jenkins/L0_Test.groovy | 175 +++---- jenkins/scripts/run_tests.py | 979 +++++++++++++++++++++++++++++++++++ jenkins/scripts/slurm_run.sh | 56 +- 3 files changed, 1083 insertions(+), 127 deletions(-) create mode 100644 jenkins/scripts/run_tests.py diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 9ca07ae63870..b464194879da 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -349,6 +349,14 @@ def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String def resultsFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/results*.xml" downloadResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, resultsFilePath, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 + // Also download the merged rerun report XML when run_tests.py emits it. + // The collectTestResults "Rerun Report" stage globs '*/rerun_results.xml' across + // every stage tarball, and the glob above (results*.xml) does NOT match + // 'rerun_results.xml'. Without this extra scp, sbatch-based stages drop out of + // the cross-stage rerun report. + def rerunResultsPath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/rerun_results.xml" + Utils.exec(pipeline, script: scpFromRemoteCmd(remote, rerunResultsPath, "${stageName}/"), returnStatus: true, numRetries: 1) + // Download perf test results def perfResultsBasePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}" def folderListOutput = Utils.exec( @@ -1527,8 +1535,11 @@ def getPytestBaseCommandLine( if (stageName.contains("-Ray-")) { testCmdLine += ["--run-ray"] } - def unittestMarkExpr = (stageName.startsWith("CPU-")) ? "cpu_only" : "not cpu_only" - testCmdLine += ["--unittest-markexpr='${unittestMarkExpr}'"] + def unittestMarkExpr = (stageName.startsWith("CPU-")) ? "cpu_only and not disabled" : "not cpu_only" + testCmdLine += ["--unittest-markexpr=\"${unittestMarkExpr}\""] + if (ENABLE_UPLOAD_TEST_RESULTS) { + testCmdLine += ["-o console_output_style=progress-even-when-capture-no"] + } if (extraArgs) { testCmdLine += extraArgs } @@ -1774,25 +1785,24 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG ) // Generate Pytest command + // NOTE: the launcher (trtllm-llmapi-launch) is NOT included in + // pytestCommand. For multi-node, slurm_run.sh wraps run_tests.py + // itself with the launcher so that MPI worker nodes stay alive + // across all pytest invocations (regular, isolated, rerun). String pytestUtil = "" if (nodeCount > 1) { pytestUtil = "$llmSrcNode/tensorrt_llm/llmapi/trtllm-llmapi-launch" } def uploadPath = "${env.JOB_NAME}/${env.BUILD_NUMBER}" - def clusterDurationsArgsNode = [] + def clusterDurationsPathNode = "" if (useClusterDurations) { def clusterKey = partition.clusterName.replaceAll('[^a-zA-Z0-9]', '_') - def clusterDurationsPathNode = "${llmSrcNode}/tests/integration/defs/.test_durations_${clusterKey}" - clusterDurationsArgsNode = ["--durations-path ${clusterDurationsPathNode}"] + clusterDurationsPathNode = "${llmSrcNode}/tests/integration/defs/.test_durations_${clusterKey}" } - def extraArgs = [ - "--test-list=$testListPathNode", - "--splitting-algorithm least_duration", - "--splits $splits", - "--group $splitId", - *clusterDurationsArgsNode, - ] + // test-list/splits/group/durations are now handled by run_tests.py; + // only pass S3 upload args as part of the pytest base command. + def extraArgs = [] if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { extraArgs += [ "--capture=fd", @@ -1807,9 +1817,10 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG perfMode, jobWorkspace, "$jobWorkspace/.coveragerc", - pytestUtil, - extraArgs, + "", // pytestUtil excluded — launcher wraps run_tests.py instead + extraArgs, // test-list/splits/group handled by run_tests.py ).join(" ") + def failSignaturesList = trtllm_utils.getFailSignaturesList().join(",") // Generate Job Launch Script def container = LLM_DOCKER_IMAGE @@ -1963,6 +1974,13 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "export ${varName}=\"${escapedValue}\"" }.join('\n') + // Escape pytestCommand for embedding inside bash double-quoted export. + // --unittest-markexpr="not cpu_only" contains inner double quotes that + // would prematurely close the outer "..." and cause an export error. + def safePytestCmd = pytestCommand + .replace('\\', '\\\\') + .replace('"', '\\"') + def scriptLaunchPrefix = """#!/bin/bash #SBATCH ${exemptionComment} #SBATCH --output=${slurmJobLogPath} @@ -1984,8 +2002,14 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG export stageName=$stageName export perfMode=$perfMode export resourcePathNode=$resourcePathNode - export pytestCommand="$pytestCommand" + export pytestCommand="$safePytestCmd" export coverageConfigFile="$coverageConfigFile" + export testListPathNode="$testListPathNode" + export testSplits="$splits" + export testGroup="$splitId" + export failSignaturesList="$failSignaturesList" + export pytestUtil="$pytestUtil" + export testDurationsPath="$clusterDurationsPathNode" export HF_TOKEN=$HF_TOKEN if [ -f "${s3SecretKeyPathNode}" ]; then set +x @@ -4416,23 +4440,17 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO stage ("[${stageName}] Run Pytest") { - def noRegularTests = false - def noIsolateTests = false - def rerunFailed = false - // When useClusterDurations is set, use a per-cluster durations file keyed on // partition.clusterName (e.g. "oci-hsg", "dlcluster"). This lets each cluster // build its own timing baseline so sharding is not skewed by timings collected // on different hardware. Falls back to the shared .test_durations when unset. - def clusterDurationsArgs = [] - def clusterDurationsPath = "" String clusterNameForDurations = null + def clusterDurationsPath = "" if (useClusterDurations) { def partition = SlurmConfig.resolvePlatform(platform) def clusterKey = partition.clusterName.replaceAll('[^a-zA-Z0-9]', '_') clusterNameForDurations = clusterKey clusterDurationsPath = "${llmSrc}/tests/integration/defs/.test_durations_${clusterKey}" - clusterDurationsArgs = ["--durations-path ${clusterDurationsPath}"] } def testDBList = renderTestDB(pipeline, testList, llmSrc, stageName, null, clusterNameForDurations) @@ -4445,9 +4463,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO reusePassedTestResults(llmSrc, stageName, "${llmSrc}/tests/integration/test_lists/waives.txt", postTag) } - // Process shard test list and create separate files for regular and isolate tests - def preprocessedLists = processShardTestList(llmSrc, testDBList, splitId, splits, perfMode, clusterDurationsPath) - // Test Coverage def TRTLLM_WHL_PATH = sh(returnStdout: true, script: "pip3 show tensorrt_llm | grep Location | cut -d ' ' -f 2").replaceAll("\\s","") sh "echo ${TRTLLM_WHL_PATH}" @@ -4476,10 +4491,9 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def containerPortNum = GlobalState.PORT_SECTION_SIZE def uploadPath = UPLOAD_PATH.replaceFirst("sw-tensorrt-generic/llm-artifacts/LLM/", "") - // Some clusters do not allow dmesg -C so we add || true - // Temporarily disable to reduce the log size - // sh 'if [ "$(id -u)" -eq 0 ]; then dmesg -C || true; fi' - def extraArgs = [*clusterDurationsArgs] + // clusterDurationsArgs removed — --durations-path is now passed directly to run_tests.py. + // Only S3 upload args remain in extraArgs for the pytest base command. + def extraArgs = [] if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { extraArgs += [ "--capture=fd", @@ -4495,16 +4509,11 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO "${WORKSPACE}/${stageName}", coverageConfigFile, "", // pytestUtil - extraArgs, // extraArgs + extraArgs, // S3 upload args; test-list/durations handled by run_tests.py containerPortStart, containerPortNum ) - // Only add --test-list if there are regular tests to run - if (preprocessedLists.regularCount > 0) { - pytestCommand += ["--test-list=${preprocessedLists.regular}"] - } - def containerPIP_LLM_LIB_PATH = sh(script: "pip3 show tensorrt_llm | grep \"Location\" | awk -F\":\" '{ gsub(/ /, \"\", \$2); print \$2\"/tensorrt_llm/libs\"}'", returnStdout: true).replaceAll("\\s","") def containerLD_LIBRARY_PATH = sh(script: "echo \${LD_LIBRARY_PATH}", returnStdout: true).replaceAll("\\s","") if (!containerLD_LIBRARY_PATH.contains("${containerPIP_LLM_LIB_PATH}:")) { @@ -4519,68 +4528,28 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO string(credentialsId: 'llm_evaltool_repo_url', variable: 'EVALTOOL_REPO_URL') ]) { sh "env | sort" - try { - try { - if (preprocessedLists.regularCount > 0) { - sh """ - rm -rf ${stageName}/ && \ - cd ${llmSrc}/tests/integration/defs && \ - ${pytestCommand.join(" ")} - """ - } else { - echo "No regular tests to run for stage ${stageName}" - noRegularTests = true - sh "mkdir -p ${stageName}" - // Create an empty results.xml file for consistency - sh """ - echo '' > ${stageName}/results.xml - echo '' >> ${stageName}/results.xml - echo '' >> ${stageName}/results.xml - echo '' >> ${stageName}/results.xml - echo '' >> ${stageName}/results.xml - """ - } - } catch (InterruptedException e) { - throw e - } catch (Exception e) { - def isRerunFailed = rerunFailedTests(stageName, llmSrc, pytestCommand, "results.xml", "regular") - if (isRerunFailed) { - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - error "Regular tests failed after rerun attempt" - } - rerunFailed = true - } else if (generateTimeoutTestResultXml(pipeline, stageName)) { - // Rerun passed but the first run had a timeout: mark this - // stage FAILURE so "[${stageName}] Run Pytest" turns red, - // not just the enclosing parent stage. - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - error "Some tests terminated unexpectedly, please check the test report." - } - } - } - // Run the isolated tests if exists - if (preprocessedLists.isolateCount > 0) { - stage ("[${stageName}] Run Pytest (Isolated)") { - echo "There are ${preprocessedLists.isolateCount} isolated tests to run" - rerunFailed = runIsolatedTests(preprocessedLists, pytestCommand, llmSrc, stageName) || rerunFailed - } - } else { - echo "No isolated tests to run for stage ${stageName}" - noIsolateTests = true - } + // Build fail signatures list for rerun eligibility + def failSignaturesList = trtllm_utils.getFailSignaturesList().join(",") - if (noRegularTests && noIsolateTests) { - error "No tests were executed for stage ${stageName}, please check the test list and test-db rendering result." - } - } finally { - if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { - sh """ - python3 ${llmSrc}/tests/test_common/s3_output.py \ - --drain-spool "${WORKSPACE}/${stageName}" || true - """ - } - } + // Use unified run_tests.py for render + regular + isolated + rerun + merge + sh """ + rm -rf ${stageName}/ && \ + cd ${llmSrc}/tests/integration/defs && \ + python3 ${llmSrc}/jenkins/scripts/run_tests.py \ + --render \ + --test-db-list ${testDBList} \ + --splits ${splits} \ + --group ${splitId} \ + ${perfMode ? '--perf-mode' : ''} \ + --pytest-base-cmd '${pytestCommand.join(" ")}' \ + --stage-name ${stageName} \ + --output-dir ${WORKSPACE}/${stageName} \ + --working-dir ${llmSrc}/tests/integration/defs \ + --fail-signatures '${failSignaturesList}' \ + --max-rerun-tests 5 \ + ${clusterDurationsPath ? "--durations-path ${clusterDurationsPath}" : ''} + """ } // CBTS coverage liveness signal: log this stage's touch counts (no artifacts); never fails the stage (|| true). @@ -4593,15 +4562,13 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } } - // Generate comprehensive rerun report if any reruns occurred - stage ("Generate Report") { - timeout(time: 15, unit: 'MINUTES'){ - generateRerunReport(stageName, llmSrc) - } - } - - if (rerunFailed) { - error "Some tests still failed after rerun attempts, please check the test report." + // Upload rerun report if generated by run_tests.py + if (fileExists("${WORKSPACE}/${stageName}/rerun_results.html")) { + trtllm_utils.uploadArtifacts( + "${WORKSPACE}/${stageName}/rerun_results.html", + "${UPLOAD_PATH}/rerun_reports/${stageName}_rerun_results.html" + ) + echo "Test rerun report: https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/rerun_reports/${stageName}_rerun_results.html" } if (fileExists("${stageName}/results-timeout.xml") || generateTimeoutTestResultXml(pipeline, stageName)) { diff --git a/jenkins/scripts/run_tests.py b/jenkins/scripts/run_tests.py new file mode 100644 index 000000000000..2ec12eed97de --- /dev/null +++ b/jenkins/scripts/run_tests.py @@ -0,0 +1,979 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unified test runner for TensorRT-LLM CI. + +Handles regular tests, isolated tests, rerun logic, and result merging +in a single script that can be invoked from both Groovy (K8s pod) and +SLURM (slurm_run.sh) execution paths. +""" + +import argparse +import collections +import os +import re +import shlex +import subprocess +import sys +import time +import xml.etree.ElementTree as ET +from pathlib import Path + +# Allow importing test_rerun from the same directory +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import test_rerun + +# Force line-buffered stdout so banners and phase messages appear in real-time +# interleaved correctly with subprocess (pytest) output, rather than being +# block-buffered and flushed all at once when the script exits. +sys.stdout.reconfigure(line_buffering=True) + +BANNER_WIDTH = 80 +COLLECT_TIMEOUT_SECONDS = 600 + + +def print_banner(title, char="="): + """Print a prominent phase banner.""" + border = char * BANNER_WIDTH + print(f"\n{border}") + print(f" {title}") + print(f"{border}\n") + + +def print_phase_summary(phase, passed, failed, duration_s): + """Print a summary line after a phase completes.""" + status = "PASSED" if failed == 0 else "FAILED" + print(f"\n>> [{phase}] {status}: {passed} passed, {failed} failed ({duration_s:.1f}s)") + + +def format_duration(seconds): + """Format seconds into a human-readable string.""" + m, s = divmod(int(seconds), 60) + h, m = divmod(m, 60) + if h > 0: + return f"{h}h {m}m {s}s" + if m > 0: + return f"{m}m {s}s" + return f"{s}s" + + +# --------------------------------------------------------------------------- +# render_test_list: replaces Groovy processShardTestList() +# --------------------------------------------------------------------------- +def render_test_list(test_db_list, working_dir, splits, group, perf_mode, durations_path=None): + """Parse test-db list, split into shards, separate regular and isolated tests. + + Args: + test_db_list: Path to the test-db rendered test list file. + working_dir: Working directory for pytest (tests/integration/defs). + splits: Total number of shards. + group: This shard's group number (1-based). + perf_mode: If True, skip pytest collection and use all tests as regular. + durations_path: Optional path to per-cluster .test_durations file for + least_duration splitting. Falls back to the default .test_durations + when None. + + Returns: + Tuple of (regular_list_path, isolate_list_path, regular_count, isolate_count). + """ + original_lines = Path(test_db_list).read_text().splitlines() + + cleaned_lines = [] + isolation_tests = set() + + for line in original_lines: + stripped = line.strip() + if not stripped: + continue + if "ISOLATION" in stripped: + cleaned = stripped + if "ISOLATION," in cleaned: + cleaned = cleaned.replace("ISOLATION,", "").strip() + elif ",ISOLATION" in cleaned: + cleaned = cleaned.replace(",ISOLATION", "").strip() + else: + cleaned = cleaned.replace(" ISOLATION", "").strip() + isolation_tests.add(cleaned) + cleaned_lines.append(cleaned) + else: + cleaned_lines.append(stripped) + + # Write cleaned test list (without ISOLATION markers) + test_db_path = Path(test_db_list) + cleaned_file = str(test_db_path.with_stem(test_db_path.stem + "_cleaned")) + Path(cleaned_file).write_text("\n".join(cleaned_lines) + "\n" if cleaned_lines else "") + print(f"Created cleaned testDBList: {cleaned_file} with {len(cleaned_lines)} lines") + print(f"Original testDBList contains {len(isolation_tests)} tests with ISOLATION markers") + + shard_tests = [] + + if perf_mode: + print("Performance mode enabled - skipping pytest collection, using all tests as regular") + else: + # Use pytest --collect-only to determine which tests belong to this shard + # Clear MPI/SLURM env vars to prevent MPI_Init during collection + # (same prefixes as slurm_run.sh and trtllm-llmapi-launch) + mpi_prefixes = ("PMI", "PMIX", "MPI", "OMPI", "SLURM", "UCX") + env_vars = { + k: v for k, v in os.environ.items() if not any(k.startswith(p) for p in mpi_prefixes) + } + collect_output_dir = os.path.join(os.path.dirname(test_db_list), "collect_output") + os.makedirs(collect_output_dir, exist_ok=True) + durations_arg = f" --durations-path={durations_path}" if durations_path else "" + collect_cmd = ( + f"pytest --collect-only --splitting-algorithm least_duration " + f"--test-list={cleaned_file} --quiet " + f"--splits {splits} --group {group} " + f"--output-dir={collect_output_dir}" + f"{durations_arg}" + ) + print(f"Running: {collect_cmd}") + try: + result = subprocess.run( + collect_cmd, + shell=True, + capture_output=True, + text=True, + cwd=working_dir, + env=env_vars, + timeout=COLLECT_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as e: + partial_stdout = (e.stdout or "").strip() if isinstance(e.stdout, str) else "" + partial_stderr = (e.stderr or "").strip() if isinstance(e.stderr, str) else "" + print( + f"Error: pytest --collect-only timed out after {COLLECT_TIMEOUT_SECONDS}s " + f"(possible import-time deadlock)" + ) + if partial_stdout: + print(f"partial stdout:\n{partial_stdout}") + if partial_stderr: + print(f"partial stderr:\n{partial_stderr}") + sys.exit(1) + + output = result.stdout.strip() + print("<<>>") + print(output) + print("<<>>") + + if result.stderr: + print(f"pytest --collect-only stderr:\n{result.stderr}") + + if result.returncode != 0 and not output: + print(f"Error: pytest --collect-only failed with exit code {result.returncode}") + print(f"stderr: {result.stderr}") + sys.exit(1) + + # Parse output: collect test IDs after "Running N items in this shard" line + found_running_line = False + for line in output.split("\n"): + if re.search(r"Running \d+ items in this shard", line) or re.search( + r"\[pytest-split\] Running group", line + ): + found_running_line = True + continue + if found_running_line and "======================" in line: + found_running_line = False + continue + if found_running_line and "::" in line: + shard_tests.append(line.strip()) + + print(f"Filtering complete. shard_tests size: {len(shard_tests)}") + + # Split into regular and isolate + regular_tests = [] + isolate_tests = [] + + if perf_mode: + regular_tests = [t for t in cleaned_lines if t.strip()] + else: + for test in shard_tests: + trimmed = test.strip() + if not trimmed: + continue + # Handle test_unittests.py::test_unittests_v2[xxxx] pattern + if trimmed.startswith("test_unittests.py::test_unittests_v2[") and trimmed.endswith( + "]" + ): + start = trimmed.index("[") + 1 + end = trimmed.rindex("]") + trimmed = trimmed[start:end] + + # Check if this test is in the isolation set + isolation_match = next((t for t in isolation_tests if trimmed in t), None) + if isolation_match: + isolate_tests.append(isolation_match) + else: + cleaned_match = next((t for t in cleaned_lines if trimmed in t), None) + if cleaned_match: + regular_tests.append(cleaned_match) + + # Write regular and isolate list files + regular_file = str(test_db_path.with_stem(test_db_path.stem + "_regular")) + isolate_file = str(test_db_path.with_stem(test_db_path.stem + "_isolate")) + + Path(regular_file).write_text("\n".join(regular_tests) + "\n" if regular_tests else "") + Path(isolate_file).write_text("\n".join(isolate_tests) + "\n" if isolate_tests else "") + + print(f"Created {regular_file} with {len(regular_tests)} regular tests") + print(f"Created {isolate_file} with {len(isolate_tests)} isolate tests") + + return regular_file, isolate_file, len(regular_tests), len(isolate_tests) + + +# --------------------------------------------------------------------------- +# Core pytest execution +# --------------------------------------------------------------------------- +def run_pytest(pytest_cmd, working_dir, tail_lines=0): + """Execute a pytest command and return the exit code and duration. + + Args: + pytest_cmd: The full pytest command string. + working_dir: Working directory for pytest execution. + tail_lines: When > 0, stream output line-by-line via Popen, keep the + last N lines in a buffer, and return them as the third tuple + element. Use for isolation tests so collection errors that + disappear between the session header and the summary line are + captured and can be re-printed in the [FAST FAILURE] banner. + + Returns: + Tuple of (exit_code, duration_seconds, captured_tail: list[str]). + captured_tail is an empty list when tail_lines == 0. + """ + print(f"Running pytest: {pytest_cmd}") + sys.stdout.flush() + t0 = time.monotonic() + + if tail_lines > 0: + buf = collections.deque(maxlen=tail_lines) + # PYTHONUNBUFFERED=1 prevents pytest from block-buffering stdout when + # it detects a pipe (non-tty), which would otherwise delay or lose + # collection-error output before the process exits. + env = {**os.environ, "PYTHONUNBUFFERED": "1"} + proc = subprocess.Popen( + pytest_cmd, + shell=True, + cwd=working_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=env, + ) + for line in proc.stdout: + sys.stdout.write(line) + sys.stdout.flush() + buf.append(line.rstrip("\n")) + proc.wait() + rc = proc.returncode + elapsed = time.monotonic() - t0 + print(f"Pytest finished with exit code {rc} ({format_duration(elapsed)})") + return rc, elapsed, list(buf) + + result = subprocess.run(pytest_cmd, shell=True, cwd=working_dir, stderr=subprocess.STDOUT) + elapsed = time.monotonic() - t0 + print(f"Pytest finished with exit code {result.returncode} ({format_duration(elapsed)})") + return result.returncode, elapsed, [] + + +# Isolation tests that finish in under this many seconds almost certainly +# failed before the test body ran (server not ready, import error, etc.). +_FAST_FAILURE_THRESHOLD_SECS = 120 + + +def print_xml_failure_summary(xml_path, tail_lines=None): + """Parse a pytest result XML and print failure/error messages. + + Called after a fast isolation-test failure to surface the root cause + without requiring manual inspection of the SLURM job log. + + Args: + xml_path: Path to the pytest JUnit XML result file. + tail_lines: Optional list of captured pytest output lines. Printed + when no XML is found (collection error) so the error is visible + in the [FAST FAILURE] banner even without a result file. + """ + if not os.path.exists(xml_path): + print(" [No result XML — pytest exited before generating results]") + if tail_lines: + print(" [Last pytest output (collection error likely above):]") + for line in tail_lines: + print(f" {line}") + return + try: + root = ET.parse(xml_path).getroot() + except ET.ParseError as exc: + print(f" [Could not parse result XML: {exc}]") + return + + cases = root.findall(".//testcase") + if not cases: + print(" [Result XML contains no test cases — 0 tests were collected]") + return + + printed = 0 + for tc in cases: + for tag in ("failure", "error"): + node = tc.find(tag) + if node is None: + continue + name = f"{tc.get('classname', '')}.{tc.get('name', '')}" + msg = (node.get("message") or "").strip() + detail = (node.text or "").strip() + print(f" [{tag.upper()}] {name}") + if msg: + print(f" message : {msg[:600]}") + if detail: + lines = detail.splitlines() + excerpt = "\n ".join(lines[-30:]) # last 30 lines + print(f" detail :\n {excerpt}") + printed += 1 + if printed == 0: + print(" [No failure/error nodes found in result XML]") + + +def rebuild_pytest_command(base_cmd, drop_patterns, append_args): + """Drop args matching any prefix in drop_patterns, then append append_args. + + For tokens of the form ``--flag=value`` the single token is dropped; for + ``--flag value`` the following value token is also dropped (when it does + not look like another flag). + + Shell pipeline prefixes (e.g. ``unset VAR && ``) are preserved verbatim + so that shlex.join does not quote ``&&`` into a literal argument. + + Args: + base_cmd: The original pytest command string. + drop_patterns: Argument prefixes to strip from base_cmd. + append_args: Tokens to append after stripping (already individually + shell-safe; quoting is handled by ``shlex.join``). + + Returns: + Rebuilt pytest command string. + """ + # Preserve shell prefix (everything before the last " && ") verbatim so + # shlex.join never quotes && into '&&' and breaks the pipeline. + shell_prefix = "" + cmd_part = base_cmd + if " && " in base_cmd: + and_parts = base_cmd.split(" && ") + shell_prefix = " && ".join(and_parts[:-1]) + " && " + cmd_part = and_parts[-1] + + parts = [] + tokens = shlex.split(cmd_part) + skip_next = False + for i, token in enumerate(tokens): + if skip_next: + skip_next = False + continue + if any(token.startswith(p) for p in drop_patterns): + if "=" not in token and i + 1 < len(tokens) and not tokens[i + 1].startswith("-"): + skip_next = True + continue + parts.append(token) + parts.extend(append_args) + return shell_prefix + shlex.join(parts) + + +def build_rerun_command(base_cmd, test_list, xml_path, csv_path, reruns): + """Build a rerun pytest command by stripping split/cov args and replacing test-list/output args. + + Args: + base_cmd: The original pytest command string. + test_list: Path to the rerun test list file. + xml_path: Path for the rerun results XML. + csv_path: Path for the rerun results CSV. + reruns: Number of reruns (passed to --reruns flag). + + Returns: + Modified pytest command string. + """ + drop_patterns = [ + "--splitting-algorithm", + "--splits", + "--group", + "--cov", + "--test-list", + "--csv", + "--periodic-junit-xmlpath", + ] + append_args = [ + f"--test-list={test_list}", + f"--csv={csv_path}", + "--periodic-junit-xmlpath", + xml_path, + ] + if reruns > 0: + append_args.extend(["--reruns", str(reruns)]) + return rebuild_pytest_command(base_cmd, drop_patterns, append_args) + + +def build_isolated_command(base_cmd, test_list, xml_path, csv_path): + """Build an isolated test command by replacing test-list/output/prefix args. + + Args: + base_cmd: The original pytest command string. + test_list: Path to the single-test list file. + xml_path: Path for the isolated results XML. + csv_path: Path for the isolated results CSV. + + Returns: + Modified pytest command string. + """ + drop_patterns = ["--test-list", "--test-prefix", "--csv", "--periodic-junit-xmlpath"] + append_args = [ + f"--test-list={test_list}", + f"--csv={csv_path}", + "--periodic-junit-xmlpath", + xml_path, + "--cov-append", + ] + return rebuild_pytest_command(base_cmd, drop_patterns, append_args) + + +# --------------------------------------------------------------------------- +# Rerun logic: replaces Groovy rerunFailedTests() +# --------------------------------------------------------------------------- +def check_and_rerun( + result_xml, + base_cmd, + working_dir, + output_dir, + rerun_tag, + fail_signatures, + max_rerun_tests, + test_list_file, +): + """Analyze test failures and rerun eligible tests. + + Args: + result_xml: Path to the results XML to analyze. + base_cmd: The original pytest command string. + working_dir: Working directory for pytest execution. + output_dir: Base output directory for the stage. + rerun_tag: Tag for this rerun (e.g., "regular", "isolated_0"). + fail_signatures: List of failure signature strings. + max_rerun_tests: Max number of failed tests that can trigger rerun. + test_list_file: Path to the test list file that drove this pytest run + (used to detect tests that never ran). + + Returns: + Tuple of (is_rerun_failed: bool, rerun_xml_files: list of paths to rerun result XMLs). + """ + if not os.path.exists(result_xml): + print(f"No {result_xml} file found, skip the rerun step") + return True, [] + + rerun_dir = os.path.join(output_dir, "rerun", rerun_tag) + os.makedirs(rerun_dir, exist_ok=True) + + # Step 1: Generate rerun test lists + print(f"Generating rerun test lists for {rerun_tag}...") + unfinished_test_file = os.path.join(output_dir, "unfinished_test.txt") + test_rerun.generate_rerun_tests_list( + rerun_dir, + result_xml, + fail_signatures, + test_list_file, + unfinished_test_file, + ) + + # Step 2: Count total failed tests + valid_count = 0 + for times in [1, 2]: + rerun_file = os.path.join(rerun_dir, f"rerun_{times}.txt") + if os.path.exists(rerun_file): + lines = [line for line in Path(rerun_file).read_text().splitlines() if line.strip()] + count = len(lines) + print(f"Found {count} {rerun_tag} tests to rerun {times} time(s)") + valid_count += count + + # Step 3: Execute reruns + is_rerun_failed = False + rerun_xml_files = [] + + for times in [1, 2]: + rerun_list = os.path.join(rerun_dir, f"rerun_{times}.txt") + if not os.path.exists(rerun_list): + print(f"No failed {rerun_tag} tests need to be rerun {times} time(s)") + continue + + print(f"Rerun test list ({times}):") + print(Path(rerun_list).read_text()) + + xml_file = os.path.join(rerun_dir, f"rerun_results_{times}.xml") + csv_file = os.path.join(rerun_dir, f"rerun_report_{times}.csv") + + rerun_cmd = build_rerun_command(base_cmd, rerun_list, xml_file, csv_file, times - 1) + rc, _, _tail = run_pytest(rerun_cmd, working_dir) + + if os.path.exists(xml_file): + rerun_xml_files.append(xml_file) + + if rc != 0: + if not os.path.exists(xml_file): + print(f"The {rerun_tag} tests crashed during rerun attempt (no XML produced).") + raise RuntimeError(f"Rerun crashed for {rerun_tag}, no XML produced") + print(f"The {rerun_tag} tests still failed after rerun attempt.") + is_rerun_failed = True + + print(f"is_rerun_failed for {rerun_tag}: {is_rerun_failed}") + return is_rerun_failed, rerun_xml_files + + +# --------------------------------------------------------------------------- +# Regular test execution +# --------------------------------------------------------------------------- +def run_regular_tests( + pytest_cmd, regular_list, working_dir, output_dir, stage_name, fail_signatures, max_rerun_tests +): + """Run regular tests and rerun failures if applicable. + + Args: + pytest_cmd: The full pytest command string (already includes --test-list). + regular_list: Path to the regular test list file. + working_dir: Working directory for pytest execution. + output_dir: Output directory for the stage. + stage_name: Name of the test stage. + fail_signatures: List of failure signature strings. + max_rerun_tests: Max number of failed tests that can trigger rerun. + + Returns: + Tuple of (rerun_failed: bool, all_xml_files: list of result XML paths). + """ + regular_count = len( + [line for line in Path(regular_list).read_text().splitlines() if line.strip()] + ) + print_banner(f"REGULAR TESTS — {regular_count} test(s) for stage [{stage_name}]") + + result_xml = os.path.join(output_dir, "results.xml") + all_xml_files = [result_xml] + + rc, elapsed, _tail = run_pytest(pytest_cmd, working_dir) + + if rc != 0: + print_phase_summary("REGULAR TESTS", 0, 1, elapsed) + print_banner( + f"RERUN — checking failed regular tests for stage [{stage_name}]", + char="-", + ) + try: + is_rerun_failed, rerun_xmls = check_and_rerun( + result_xml, + pytest_cmd, + working_dir, + output_dir, + "regular", + fail_signatures, + max_rerun_tests, + regular_list, + ) + all_xml_files.extend(rerun_xmls) + return is_rerun_failed, all_xml_files + except RuntimeError as e: + print(f"Regular tests rerun crashed: {e}") + return True, all_xml_files + + print_phase_summary("REGULAR TESTS", regular_count, 0, elapsed) + return False, all_xml_files + + +# --------------------------------------------------------------------------- +# Isolated test execution: replaces Groovy runIsolatedTests() +# --------------------------------------------------------------------------- +def run_isolated_tests( + pytest_cmd, isolate_list, working_dir, output_dir, stage_name, fail_signatures, max_rerun_tests +): + """Run isolated tests one by one, rerunning each on failure. + + Args: + pytest_cmd: The base pytest command string. + isolate_list: Path to the isolate test list file. + working_dir: Working directory for pytest execution. + output_dir: Output directory for the stage. + stage_name: Name of the test stage. + fail_signatures: List of failure signature strings. + max_rerun_tests: Max number of failed tests that can trigger rerun. + + Returns: + Tuple of (rerun_failed: bool, all_xml_files: list of result XML paths). + """ + isolate_tests = [ + line.strip() for line in Path(isolate_list).read_text().splitlines() if line.strip() + ] + + print_banner(f"ISOLATION TESTS — {len(isolate_tests)} test(s) for stage [{stage_name}]") + + rerun_failed = False + all_xml_files = [] + iso_passed = 0 + iso_failed = 0 + t0 = time.monotonic() + + for i, test_name in enumerate(isolate_tests): + print(f"\n--- [ISOLATION {i + 1}/{len(isolate_tests)}] {test_name} ---") + + # Create a temporary file for this single test + single_test_file = os.path.join(output_dir, f"isolated_{i}.txt") + Path(single_test_file).write_text(test_name + "\n") + + xml_path = os.path.join(output_dir, f"results_isolated_{i}.xml") + csv_path = os.path.join(output_dir, f"report_isolated_{i}.csv") + + isolated_cmd = build_isolated_command(pytest_cmd, single_test_file, xml_path, csv_path) + rc, elapsed, pytest_tail = run_pytest(isolated_cmd, working_dir, tail_lines=200) + all_xml_files.append(xml_path) + + if rc != 0: + if elapsed < _FAST_FAILURE_THRESHOLD_SECS: + print( + f"\n [FAST FAILURE] Isolation test {i + 1} exited in " + f"{format_duration(elapsed)} (< {_FAST_FAILURE_THRESHOLD_SECS}s). " + f"Likely failed before the test body ran (server not ready, " + f"collection error, etc.)." + ) + print_xml_failure_summary(xml_path, tail_lines=pytest_tail) + print_banner( + f"RERUN — isolated test {i + 1}/{len(isolate_tests)}: {test_name}", + char="-", + ) + try: + is_rerun_failed, rerun_xmls = check_and_rerun( + xml_path, + isolated_cmd, + working_dir, + output_dir, + f"isolated_{i}", + fail_signatures, + max_rerun_tests, + single_test_file, + ) + all_xml_files.extend(rerun_xmls) + if is_rerun_failed: + print(f"Isolated test {i + 1} ({test_name}) failed after rerun attempt") + rerun_failed = True + iso_failed += 1 + else: + iso_passed += 1 + except RuntimeError as e: + print(f"Isolated test {i + 1} ({test_name}) rerun crashed: {e}") + rerun_failed = True + iso_failed += 1 + else: + iso_passed += 1 + + # Clean up temporary file + try: + os.remove(single_test_file) + except OSError: + pass + + elapsed = time.monotonic() - t0 + print_phase_summary("ISOLATION TESTS", iso_passed, iso_failed, elapsed) + + return rerun_failed, all_xml_files + + +# --------------------------------------------------------------------------- +# Result merging: replaces Groovy generateRerunReport() +# --------------------------------------------------------------------------- +def merge_results(output_dir, stage_name, all_xml_files): + """Merge all result XMLs and generate rerun report. + + Args: + output_dir: Output directory for the stage. + stage_name: Name of the test stage. + all_xml_files: List of all result XML file paths. + """ + # Fix testsuite names in all XMLs + for xml_file in all_xml_files: + if not os.path.exists(xml_file): + continue + try: + tree = ET.parse(xml_file) + except (OSError, ET.ParseError) as e: + print(f"Warning: Failed to parse {xml_file}: {e}") + continue + modified = False + for ts in tree.iter("testsuite"): + if ts.get("name") == "pytest": + ts.set("name", stage_name) + modified = True + if modified: + try: + tree.write(xml_file, encoding="utf-8", xml_declaration=True) + except OSError as e: + print(f"Warning: Failed to write {xml_file}: {e}") + + # Separate original results and rerun results + rerun_result_files = [] + for xml_file in all_xml_files: + if not os.path.exists(xml_file): + continue + # Rerun results live under rerun/ directory + if "/rerun/" in xml_file and "rerun_results_" in xml_file: + rerun_result_files.append(xml_file) + + # Also add original results that have corresponding reruns to the rerun report + rerun_base_dir = os.path.join(output_dir, "rerun") + original_results_with_reruns = [] + + # Check regular reruns + regular_rerun_dir = os.path.join(rerun_base_dir, "regular") + if os.path.isdir(regular_rerun_dir): + has_regular_reruns = any( + f.startswith("rerun_results_") and f.endswith(".xml") + for f in os.listdir(regular_rerun_dir) + ) + if has_regular_reruns: + results_xml = os.path.join(output_dir, "results.xml") + if os.path.exists(results_xml): + original_results_with_reruns.append(results_xml) + + # Check isolated reruns + if os.path.isdir(rerun_base_dir): + for d in os.listdir(rerun_base_dir): + if d.startswith("isolated_") and os.path.isdir(os.path.join(rerun_base_dir, d)): + iso_dir = os.path.join(rerun_base_dir, d) + has_iso_reruns = any( + f.startswith("rerun_results_") and f.endswith(".xml") + for f in os.listdir(iso_dir) + ) + if has_iso_reruns: + iso_num = d.replace("isolated_", "") + iso_result = os.path.join(output_dir, f"results_isolated_{iso_num}.xml") + if os.path.exists(iso_result): + original_results_with_reruns.append(iso_result) + + # Generate rerun report if any reruns occurred + rerun_report_inputs = original_results_with_reruns + rerun_result_files + if rerun_report_inputs: + print(f"Generating rerun report with input files: {rerun_report_inputs}") + rerun_report_xml = os.path.join(output_dir, "rerun_results.xml") + test_rerun.generate_rerun_report(rerun_report_xml, rerun_report_inputs) + + # Merge all XMLs into a single results.xml for junit + existing_xml_files = [f for f in all_xml_files if os.path.exists(f)] + if existing_xml_files: + merged_output = os.path.join(output_dir, "results.xml") + print(f"Merging {len(existing_xml_files)} XML files into {merged_output}") + test_rerun.merge_junit_xmls(merged_output, existing_xml_files, deduplicate=True) + + # Remove isolation results since they are merged into results.xml + for f in all_xml_files: + if "results_isolated_" in f and os.path.exists(f): + try: + os.remove(f) + except OSError: + pass + + print("Result merging completed") + + +# --------------------------------------------------------------------------- +# Create empty results XML +# --------------------------------------------------------------------------- +def create_empty_results_xml(output_dir, stage_name): + """Create an empty results.xml for stages with no tests. + + Args: + output_dir: Output directory for the stage. + stage_name: Name of the test stage. + """ + os.makedirs(output_dir, exist_ok=True) + content = ( + '\n' + "\n" + f'\n' + "\n" + "\n" + ) + Path(os.path.join(output_dir, "results.xml")).write_text(content) + + +# --------------------------------------------------------------------------- +# Main entry point +# --------------------------------------------------------------------------- +def parse_args(): + parser = argparse.ArgumentParser(description="Unified TensorRT-LLM test runner") + + # Render mode arguments + parser.add_argument( + "--render", + action="store_true", + help="Run render_test_list to split test-db list into regular/isolate shards", + ) + parser.add_argument("--test-db-list", help="Path to the test-db rendered test list file") + parser.add_argument("--splits", type=int, default=1, help="Total number of shards") + parser.add_argument("--group", type=int, default=1, help="This shard group number (1-based)") + parser.add_argument( + "--perf-mode", action="store_true", help="Performance mode: skip collection, run all tests" + ) + + # Run mode arguments + parser.add_argument("--pytest-base-cmd", help="Base pytest command string") + parser.add_argument( + "--regular-test-list", help="Path to regular test list (if not using --render)" + ) + parser.add_argument( + "--isolate-test-list", help="Path to isolate test list (if not using --render)" + ) + parser.add_argument("--stage-name", required=True, help="Name of the test stage") + parser.add_argument("--output-dir", required=True, help="Output directory for the stage") + parser.add_argument("--working-dir", help="Working directory for pytest (defaults to cwd)") + parser.add_argument( + "--fail-signatures", + default="", + help="Comma-separated list of failure signatures for rerun eligibility", + ) + parser.add_argument( + "--max-rerun-tests", type=int, default=5, help="Max failed tests to trigger rerun" + ) + parser.add_argument( + "--durations-path", + default=None, + help="Path to per-cluster .test_durations file for least_duration splitting. " + "Overrides the default .test_durations when set.", + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + + working_dir = args.working_dir or os.getcwd() + output_dir = args.output_dir + stage_name = args.stage_name + fail_signatures = [s for s in args.fail_signatures.split(",") if s] + max_rerun_tests = args.max_rerun_tests + + os.makedirs(output_dir, exist_ok=True) + + # Step 1: Render test list if requested + regular_list = args.regular_test_list + isolate_list = args.isolate_test_list + regular_count = 0 + isolate_count = 0 + + if args.render: + if not args.test_db_list: + print("Error: --test-db-list is required when using --render") + sys.exit(1) + regular_list, isolate_list, regular_count, isolate_count = render_test_list( + args.test_db_list, + working_dir, + args.splits, + args.group, + args.perf_mode, + args.durations_path, + ) + else: + if regular_list and os.path.exists(regular_list): + regular_count = len( + [line for line in Path(regular_list).read_text().splitlines() if line.strip()] + ) + if isolate_list and os.path.exists(isolate_list): + isolate_count = len( + [line for line in Path(isolate_list).read_text().splitlines() if line.strip()] + ) + + if not args.pytest_base_cmd: + # Render-only mode: just output the lists and exit + if args.render: + print(f"regular_list={regular_list}") + print(f"isolate_list={isolate_list}") + print(f"regular_count={regular_count}") + print(f"isolate_count={isolate_count}") + sys.exit(0) + else: + print("Error: --pytest-base-cmd is required for test execution") + sys.exit(1) + + pytest_base_cmd = args.pytest_base_cmd + total_t0 = time.monotonic() + + print_banner( + f"TEST EXECUTION START — stage [{stage_name}] " + f"({regular_count} regular, {isolate_count} isolation)" + ) + + # Step 2: Run regular tests + all_xml_files = [] + rerun_failed = False + + if regular_count > 0: + # Add --test-list to the base command for regular tests + regular_cmd = f"{pytest_base_cmd} --test-list={regular_list}" + failed, xml_files = run_regular_tests( + regular_cmd, + regular_list, + working_dir, + output_dir, + stage_name, + fail_signatures, + max_rerun_tests, + ) + rerun_failed = rerun_failed or failed + all_xml_files.extend(xml_files) + else: + print(f"No regular tests to run for stage {stage_name}") + create_empty_results_xml(output_dir, stage_name) + all_xml_files.append(os.path.join(output_dir, "results.xml")) + + # Step 3: Run isolated tests + if isolate_count > 0: + failed, xml_files = run_isolated_tests( + pytest_base_cmd, + isolate_list, + working_dir, + output_dir, + stage_name, + fail_signatures, + max_rerun_tests, + ) + rerun_failed = rerun_failed or failed + all_xml_files.extend(xml_files) + else: + print(f"No isolated tests to run for stage {stage_name}") + + # Step 4: Check that at least some tests were executed + if regular_count == 0 and isolate_count == 0: + print(f"Error: No tests were executed for stage {stage_name}") + sys.exit(1) + + # Step 5: Merge all results + print_banner("RESULT MERGING", char="-") + merge_results(output_dir, stage_name, all_xml_files) + + # Step 6: Final summary and exit + total_elapsed = time.monotonic() - total_t0 + print_banner(f"TEST EXECUTION COMPLETE — stage [{stage_name}]") + print(f" Regular tests : {regular_count}") + print(f" Isolation tests : {isolate_count}") + print(f" Total time : {format_duration(total_elapsed)}") + + if rerun_failed: + print(" Result : FAILED") + print("\nSome tests still failed after rerun attempts, please check the test report.") + sys.exit(1) + + print(" Result : PASSED") + print("\nAll tests passed.") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index 4aecf0d98864..b5d22f8a33ea 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -79,31 +79,41 @@ pytest_exit_code=0 perf_check_exit_code=0 perf_report_exit_code=0 -eval $pytestCommand -pytest_exit_code=$? -echo "Rank${SLURM_PROCID} Pytest finished execution with exit code $pytest_exit_code" -python3 "$llmSrcNode/tests/test_common/s3_output.py" \ - --drain-spool "$jobWorkspace" || true - -# DEBUG: Diagnose intermittent "unrecognized arguments" failure (Exit Code 4) -# Remove this after the issue is resolved -if [ $pytest_exit_code -eq 4 ]; then - echo "DEBUG: Pytest failed with usage error (exit code 4)" - echo "DEBUG: Directory state at $(pwd):" - ls -l - echo "DEBUG: Directory state at $llmSrcNode/tests/integration/defs:" - ls -l $llmSrcNode/tests/integration/defs - - echo "DEBUG: conftest.py content:" - md5sum $llmSrcNode/tests/integration/defs/conftest.py - - echo "DEBUG: pytest.ini content:" - md5sum $llmSrcNode/tests/integration/defs/pytest.ini - - echo "DEBUG: Check importability of conftest.py" - python3 -c "import sys; sys.path.insert(0, '.'); import conftest; print('DEBUG: conftest imported successfully')" +# Use unified run_tests.py for test execution, rerun, and result merging +runTestsScript="$llmSrcNode/jenkins/scripts/run_tests.py" + +# Build the run_tests.py arguments array for proper quoting +runTestsArgs=( + python3 "$runTestsScript" + --render + --test-db-list "$testListPathNode" + --splits "${testSplits:-1}" + --group "${testGroup:-1}" + --pytest-base-cmd "$pytestCommand" + --stage-name "$stageName" + --output-dir "$jobWorkspace" + --working-dir "$llmSrcNode/tests/integration/defs" + --fail-signatures "${failSignaturesList:-}" + --max-rerun-tests 5 + ${testDurationsPath:+--durations-path "$testDurationsPath"} +) +if [ "$perfMode" = "true" ]; then + runTestsArgs+=(--perf-mode) fi +# For multi-node runs, wrap run_tests.py with the MPI launcher so that +# worker nodes stay alive across all pytest invocations (regular, isolated, +# rerun). The launcher ensures rank 0 executes run_tests.py while workers +# run mgmn_worker_node for the entire duration. +if [ -n "${pytestUtil:-}" ] && [ "${SLURM_JOB_NUM_NODES:-1}" -gt 1 ]; then + echo "Multi-node mode: wrapping run_tests.py with $pytestUtil" + "$pytestUtil" "${runTestsArgs[@]}" +else + "${runTestsArgs[@]}" +fi +pytest_exit_code=$? +echo "Rank${SLURM_PROCID} run_tests.py finished execution with exit code $pytest_exit_code" + if [ $SLURM_PROCID -eq 0 ] && [ "$perfMode" = "true" ]; then # Only PyTorch perf stages remain; the TensorRT perf baseline was removed. basePerfFilename="base_perf_pytorch.csv" From 1685f757a25ac9aaf82007157cf3d724cf4d6b8b Mon Sep 17 00:00:00 2001 From: EmmaQiaoCh Date: Wed, 12 Aug 2026 11:31:54 +0300 Subject: [PATCH 2/5] Fix after resolving conflicts Signed-off-by: EmmaQiaoCh --- jenkins/L0_Test.groovy | 45 +++++++++++++++++++++--------------- jenkins/scripts/run_tests.py | 15 ++++++++---- jenkins/scripts/slurm_run.sh | 6 ++++- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index b464194879da..81b87d2eed9f 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1535,7 +1535,7 @@ def getPytestBaseCommandLine( if (stageName.contains("-Ray-")) { testCmdLine += ["--run-ray"] } - def unittestMarkExpr = (stageName.startsWith("CPU-")) ? "cpu_only and not disabled" : "not cpu_only" + def unittestMarkExpr = (stageName.startsWith("CPU-")) ? "cpu_only" : "not cpu_only" testCmdLine += ["--unittest-markexpr=\"${unittestMarkExpr}\""] if (ENABLE_UPLOAD_TEST_RESULTS) { testCmdLine += ["-o console_output_style=progress-even-when-capture-no"] @@ -4533,23 +4533,32 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def failSignaturesList = trtllm_utils.getFailSignaturesList().join(",") // Use unified run_tests.py for render + regular + isolated + rerun + merge - sh """ - rm -rf ${stageName}/ && \ - cd ${llmSrc}/tests/integration/defs && \ - python3 ${llmSrc}/jenkins/scripts/run_tests.py \ - --render \ - --test-db-list ${testDBList} \ - --splits ${splits} \ - --group ${splitId} \ - ${perfMode ? '--perf-mode' : ''} \ - --pytest-base-cmd '${pytestCommand.join(" ")}' \ - --stage-name ${stageName} \ - --output-dir ${WORKSPACE}/${stageName} \ - --working-dir ${llmSrc}/tests/integration/defs \ - --fail-signatures '${failSignaturesList}' \ - --max-rerun-tests 5 \ - ${clusterDurationsPath ? "--durations-path ${clusterDurationsPath}" : ''} - """ + try { + sh """ + rm -rf ${stageName}/ && \ + cd ${llmSrc}/tests/integration/defs && \ + python3 ${llmSrc}/jenkins/scripts/run_tests.py \ + --render \ + --test-db-list ${testDBList} \ + --splits ${splits} \ + --group ${splitId} \ + ${perfMode ? '--perf-mode' : ''} \ + --pytest-base-cmd '${pytestCommand.join(" ")}' \ + --stage-name ${stageName} \ + --output-dir ${WORKSPACE}/${stageName} \ + --working-dir ${llmSrc}/tests/integration/defs \ + --fail-signatures '${failSignaturesList}' \ + --max-rerun-tests 5 \ + ${clusterDurationsPath ? "--durations-path ${clusterDurationsPath}" : ''} + """ + } finally { + if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { + sh """ + python3 ${llmSrc}/tests/test_common/s3_output.py \ + --drain-spool "${WORKSPACE}/${stageName}" || true + """ + } + } } // CBTS coverage liveness signal: log this stage's touch counts (no artifacts); never fails the stage (|| true). diff --git a/jenkins/scripts/run_tests.py b/jenkins/scripts/run_tests.py index 2ec12eed97de..a0c82562c50f 100644 --- a/jenkins/scripts/run_tests.py +++ b/jenkins/scripts/run_tests.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -170,7 +170,7 @@ def render_test_list(test_db_list, working_dir, splits, group, perf_mode, durati if result.stderr: print(f"pytest --collect-only stderr:\n{result.stderr}") - if result.returncode != 0 and not output: + if result.returncode != 0: print(f"Error: pytest --collect-only failed with exit code {result.returncode}") print(f"stderr: {result.stderr}") sys.exit(1) @@ -313,7 +313,7 @@ def print_xml_failure_summary(xml_path, tail_lines=None): return try: root = ET.parse(xml_path).getroot() - except ET.ParseError as exc: + except (OSError, ET.ParseError) as exc: print(f" [Could not parse result XML: {exc}]") return @@ -490,7 +490,7 @@ def check_and_rerun( unfinished_test_file, ) - # Step 2: Count total failed tests + # Step 2: Count total failed tests and enforce the cap valid_count = 0 for times in [1, 2]: rerun_file = os.path.join(rerun_dir, f"rerun_{times}.txt") @@ -500,6 +500,13 @@ def check_and_rerun( print(f"Found {count} {rerun_tag} tests to rerun {times} time(s)") valid_count += count + if valid_count > max_rerun_tests: + print( + f"Too many {rerun_tag} tests to rerun ({valid_count} > max_rerun_tests={max_rerun_tests}). " + "Skipping rerun." + ) + return False + # Step 3: Execute reruns is_rerun_failed = False rerun_xml_files = [] diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index b5d22f8a33ea..539fc6d2479a 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -95,8 +95,10 @@ runTestsArgs=( --working-dir "$llmSrcNode/tests/integration/defs" --fail-signatures "${failSignaturesList:-}" --max-rerun-tests 5 - ${testDurationsPath:+--durations-path "$testDurationsPath"} ) +if [ -n "${testDurationsPath:-}" ]; then + runTestsArgs+=(--durations-path "$testDurationsPath") +fi if [ "$perfMode" = "true" ]; then runTestsArgs+=(--perf-mode) fi @@ -113,6 +115,8 @@ else fi pytest_exit_code=$? echo "Rank${SLURM_PROCID} run_tests.py finished execution with exit code $pytest_exit_code" +python3 "$llmSrcNode/tests/test_common/s3_output.py" \ + --drain-spool "$jobWorkspace" || true if [ $SLURM_PROCID -eq 0 ] && [ "$perfMode" = "true" ]; then # Only PyTorch perf stages remain; the TensorRT perf baseline was removed. From 8c293b582b7df65d94166c3b5667c877fd9e26e5 Mon Sep 17 00:00:00 2001 From: EmmaQiaoCh Date: Wed, 12 Aug 2026 11:41:03 +0300 Subject: [PATCH 3/5] TEST: Update a multinode stage Signed-off-by: EmmaQiaoCh --- ...lti_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml index a2c11d454409..30375cdbb492 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml @@ -15,3 +15,8 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2: backend: pytorch tests: - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90), ISOLATION + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90), ISOLATION + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90), ISOLATION + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90), ISOLATION + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90), ISOLATION From 54a5936f665dbcf7472351aaeb2f7044f086a215 Mon Sep 17 00:00:00 2001 From: EmmaQiaoCh Date: Mon, 17 Aug 2026 12:19:49 +0300 Subject: [PATCH 4/5] 2 minor fixes due to comments Signed-off-by: EmmaQiaoCh --- jenkins/scripts/run_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jenkins/scripts/run_tests.py b/jenkins/scripts/run_tests.py index a0c82562c50f..ff0334a69277 100644 --- a/jenkins/scripts/run_tests.py +++ b/jenkins/scripts/run_tests.py @@ -505,7 +505,7 @@ def check_and_rerun( f"Too many {rerun_tag} tests to rerun ({valid_count} > max_rerun_tests={max_rerun_tests}). " "Skipping rerun." ) - return False + return True, [] # Step 3: Execute reruns is_rerun_failed = False @@ -536,7 +536,6 @@ def check_and_rerun( print(f"The {rerun_tag} tests still failed after rerun attempt.") is_rerun_failed = True - print(f"is_rerun_failed for {rerun_tag}: {is_rerun_failed}") return is_rerun_failed, rerun_xml_files From 6f2d212a4fc77061b7c3ba080e36df90b7b8c2b5 Mon Sep 17 00:00:00 2001 From: EmmaQiaoCh Date: Mon, 17 Aug 2026 13:16:15 +0300 Subject: [PATCH 5/5] fix for comments Signed-off-by: EmmaQiaoCh --- jenkins/L0_Test.groovy | 21 ++++++++++++--------- jenkins/scripts/slurm_run.sh | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index e7b4e651d751..3c77f693f0d6 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4629,8 +4629,12 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO // Build fail signatures list for rerun eligibility def failSignaturesList = trtllm_utils.getFailSignaturesList().join(",") - // Use unified run_tests.py for render + regular + isolated + rerun + merge - try { + // Use unified run_tests.py for render + regular + isolated + rerun + merge. + // catchError lets execution continue to the report-upload and timeout-XML steps + // below even when run_tests.py exits non-zero (test failures), preserving the + // same behaviour as main where those steps ran inside a catch block before the + // terminal error. + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { sh """ rm -rf ${stageName}/ && \ cd ${llmSrc}/tests/integration/defs && \ @@ -4648,13 +4652,12 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO --max-rerun-tests 5 \ ${clusterDurationsPath ? "--durations-path ${clusterDurationsPath}" : ''} """ - } finally { - if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { - sh """ - python3 ${llmSrc}/tests/test_common/s3_output.py \ - --drain-spool "${WORKSPACE}/${stageName}" || true - """ - } + } + if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { + sh """ + python3 ${llmSrc}/tests/test_common/s3_output.py \ + --drain-spool "${WORKSPACE}/${stageName}" || true + """ } } diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index 539fc6d2479a..68e436aaf4b9 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -1,4 +1,18 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Set up error handling set -xEeuo pipefail