From e4241e65abff1d911c5283e6a9033d549dcb782a Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:02:19 -0700 Subject: [PATCH 01/34] ci: add infrastructure dry-run benchmark Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/scripts/infra_dry_run_benchmark.py | 341 ++++++++++++++++++ .../tools/test_infra_dry_run_benchmark.py | 325 +++++++++++++++++ 2 files changed, 666 insertions(+) create mode 100644 jenkins/scripts/infra_dry_run_benchmark.py create mode 100644 tests/unittest/tools/test_infra_dry_run_benchmark.py diff --git a/jenkins/scripts/infra_dry_run_benchmark.py b/jenkins/scripts/infra_dry_run_benchmark.py new file mode 100644 index 000000000000..f399f4b64175 --- /dev/null +++ b/jenkins/scripts/infra_dry_run_benchmark.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run a small CUDA/NCCL smoke test and emit CI-friendly result artifacts.""" + +import argparse +import json +import math +import os +import sys +import xml.etree.ElementTree as ET +from datetime import timedelta +from pathlib import Path +from typing import Any, Mapping + +NAME = "infra_dry_run" +MATRIX_SIZE = 64 +JUNIT_FILE = "results-infra_dry_run.xml" +MANIFEST_FILE = "infra_dry_run_manifest.json" + + +class BenchmarkError(RuntimeError): + """Raised after failure artifacts have been written.""" + + +def _load_runtime_modules() -> tuple[Any, Any]: + # Normal imports intentionally exercise the installed package and PyTorch. + import tensorrt_llm + import torch + + return tensorrt_llm, torch + + +def _rank_context(environ: Mapping[str, str]) -> dict[str, int]: + try: + rank = int(environ.get("RANK", "0")) + local_rank = int(environ.get("LOCAL_RANK", "0")) + world_size = int(environ.get("WORLD_SIZE", "1")) + except ValueError as error: + raise BenchmarkError("RANK, LOCAL_RANK, and WORLD_SIZE must be integers") from error + if world_size < 1 or rank < 0 or rank >= world_size or local_rank < 0: + raise BenchmarkError( + f"invalid rank context: rank={rank}, local_rank={local_rank}, world_size={world_size}" + ) + return {"rank": rank, "local_rank": local_rank, "world_size": world_size} + + +def _new_result( + context: Mapping[str, int], + stage: str | None, + commit: str | None, + environ: Mapping[str, str], +) -> dict[str, Any]: + return { + "name": NAME, + "status": "failed", + "product_tests_executed": 0, + **context, + "stage": stage or environ.get("STAGE_NAME") or environ.get("stageName") or "", + "commit": commit or environ.get("GIT_COMMIT") or environ.get("gitlabCommit") or "", + "tensorrt_llm_version": "unknown", + "tensorrt_llm_module": "unknown", + } + + +def _select_cuda_device(torch: Any, local_rank: int) -> str: + if not torch.cuda.is_available(): + raise BenchmarkError("CUDA is required") + device_count = int(torch.cuda.device_count()) + if local_rank >= device_count: + raise BenchmarkError( + f"LOCAL_RANK {local_rank} cannot select from {device_count} visible CUDA device(s)" + ) + torch.cuda.set_device(local_rank) + return f"cuda:{local_rank}" + + +def _cuda_matmul(torch: Any, local_rank: int, device: str) -> dict[str, Any]: + torch.manual_seed(0) + torch.cuda.manual_seed_all(0) + left = torch.full((MATRIX_SIZE, MATRIX_SIZE), 0.5, dtype=torch.float16, device=device) + right = torch.full((MATRIX_SIZE, MATRIX_SIZE), 0.25, dtype=torch.float16, device=device) + output = torch.matmul(left, right) + torch.cuda.synchronize(local_rank) + if int(output.numel()) == 0: + raise BenchmarkError("CUDA matrix multiplication returned an empty tensor") + if not bool(torch.isfinite(output).all().item()): + raise BenchmarkError("CUDA matrix multiplication returned non-finite values") + checksum = float(output.float().sum().item()) + if not math.isfinite(checksum): + raise BenchmarkError("CUDA matrix multiplication checksum is non-finite") + return { + "device": device, + "matrix_size": MATRIX_SIZE, + "dtype": "float16", + "checksum": checksum, + } + + +def _initialize_distributed(torch: Any, context: Mapping[str, int], timeout_seconds: int) -> bool: + if context["world_size"] == 1: + return False + if not torch.distributed.is_available() or not torch.distributed.is_nccl_available(): + raise BenchmarkError("torch.distributed with NCCL is required for WORLD_SIZE > 1") + torch.distributed.init_process_group( + backend="nccl", + init_method="env://", + rank=context["rank"], + world_size=context["world_size"], + timeout=timedelta(seconds=timeout_seconds), + ) + return True + + +def _summary(result: Mapping[str, Any]) -> dict[str, Any]: + return { + "rank": int(result["rank"]), + "world_size": int(result["world_size"]), + "status": str(result["status"]), + "checksum": result.get("cuda", {}).get("checksum"), + "error": str(result.get("error", "")), + } + + +def _gather_summaries(torch: Any, result: Mapping[str, Any]) -> list[dict[str, Any]]: + world_size = int(result["world_size"]) + local = torch.tensor( + [ + float(result["rank"]), + float(world_size), + float(result["status"] == "passed"), + float(result.get("cuda", {}).get("checksum", 0.0)), + ], + dtype=torch.float64, + device=f"cuda:{result['local_rank']}", + ) + gathered = [torch.empty_like(local) for _ in range(world_size)] + torch.distributed.all_gather(gathered, local) + return [ + { + "rank": int(values[0]), + "world_size": int(values[1]), + "status": "passed" if bool(values[2]) else "failed", + "checksum": float(values[3]), + "error": "" if bool(values[2]) else "CUDA work failed on this rank", + } + for values in (item.cpu().tolist() for item in gathered) + ] + + +def _validate(summaries: list[Mapping[str, Any]], world_size: int) -> list[str]: + errors: list[str] = [] + ranks = [int(item["rank"]) for item in summaries] + expected_ranks = list(range(world_size)) + if len(summaries) != world_size or sorted(ranks) != expected_ranks: + errors.append(f"observed ranks {sorted(ranks)} do not match expected {expected_ranks}") + if any(int(item["world_size"]) != world_size for item in summaries): + errors.append("rank results contain a world-size mismatch") + failed = [int(item["rank"]) for item in summaries if item["status"] != "passed"] + if failed: + errors.append(f"rank(s) {sorted(failed)} reported failure") + try: + checksums = [ + float(item["checksum"]) for item in summaries if item["status"] == "passed" + ] + except (TypeError, ValueError): + errors.append("passed rank result is missing a valid CUDA checksum") + else: + if any(not math.isfinite(value) for value in checksums): + errors.append("rank results contain a non-finite CUDA checksum") + if checksums and any( + not math.isclose(value, checksums[0], rel_tol=1e-6, abs_tol=1e-6) + for value in checksums[1:] + ): + errors.append("rank results contain inconsistent CUDA checksums") + return errors + + +def _write_json(path: Path, value: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + with temporary.open("w", encoding="utf-8") as output: + json.dump(value, output, indent=2, sort_keys=True) + output.write("\n") + os.replace(temporary, path) + + +def _write_reports( + output_dir: Path, + result: Mapping[str, Any], + summaries: list[Mapping[str, Any]], + errors: list[str], +) -> dict[str, Any]: + world_size = int(result["world_size"]) + manifest: dict[str, Any] = { + "name": NAME, + "status": "failed" if errors else "passed", + "product_tests_executed": 0, + "stage": result["stage"], + "commit": result["commit"], + "world_size": world_size, + "observed_ranks": sorted(int(item["rank"]) for item in summaries), + "validation_errors": errors, + "rank_results": summaries, + "junit_file": JUNIT_FILE, + } + _write_json(output_dir / MANIFEST_FILE, manifest) + + by_rank = {int(item["rank"]): item for item in summaries} + cases: list[tuple[str, str | None]] = [] + for rank in range(world_size): + item = by_rank.get(rank) + failure = None + if item is None: + failure = f"missing result for rank {rank}" + elif item["status"] != "passed": + failure = str(item.get("error") or f"rank {rank} failed") + cases.append((f"{NAME}_rank_{rank}_cuda_matmul", failure)) + if world_size > 1: + cases.append((f"{NAME}_nccl_collective", "; ".join(errors) or None)) + + suite = ET.Element( + "testsuite", + { + "name": NAME, + "tests": str(len(cases)), + "failures": str(sum(failure is not None for _, failure in cases)), + "errors": "0", + "skipped": "0", + "time": "0", + }, + ) + properties = ET.SubElement(suite, "properties") + for name in ("product_tests_executed", "stage", "commit", "world_size"): + ET.SubElement(properties, "property", {"name": name, "value": str(manifest[name])}) + for name, failure in cases: + case = ET.SubElement( + suite, + "testcase", + {"classname": NAME, "name": name, "time": "0"}, + ) + if failure: + node = ET.SubElement(case, "failure", {"message": failure}) + node.text = failure + + root = ET.Element("testsuites") + root.append(suite) + tree = ET.ElementTree(root) + ET.indent(tree, space=" ") + tree.write(output_dir / JUNIT_FILE, encoding="utf-8", xml_declaration=True) + return manifest + + +def run_benchmark( + output_dir: Path, + *, + stage: str | None = None, + commit: str | None = None, + timeout_seconds: int = 120, + environ: Mapping[str, str] | None = None, +) -> dict[str, Any]: + environment = os.environ if environ is None else environ + context = _rank_context(environment) + result = _new_result(context, stage, commit, environment) + summaries = [_summary(result)] + torch = None + distributed = False + + try: + trtllm, torch = _load_runtime_modules() + result["tensorrt_llm_version"] = str(getattr(trtllm, "__version__", "unknown")) + result["tensorrt_llm_module"] = str(getattr(trtllm, "__file__", "unknown")) + device = _select_cuda_device(torch, context["local_rank"]) + distributed = _initialize_distributed(torch, context, timeout_seconds) + try: + result["cuda"] = _cuda_matmul(torch, context["local_rank"], device) + result["status"] = "passed" + except Exception as error: + result["error"] = f"{type(error).__name__}: {error}" + summaries = _gather_summaries(torch, result) if distributed else [_summary(result)] + except Exception as error: + result["status"] = "failed" + result["error"] = f"{type(error).__name__}: {error}" + summaries = [_summary(result)] + finally: + if distributed and torch.distributed.is_initialized(): + try: + torch.distributed.destroy_process_group() + except Exception as error: + result["status"] = "failed" + result["error"] = f"distributed cleanup failed: {error}" + summaries = [ + item for item in summaries if int(item["rank"]) != context["rank"] + ] + [_summary(result)] + + errors = _validate(summaries, context["world_size"]) + result["overall_status"] = "failed" if errors else "passed" + _write_json(output_dir / f"{NAME}_rank_{context['rank']}.json", result) + if context["rank"] == 0: + _write_reports(output_dir, result, summaries, errors) + if errors: + raise BenchmarkError("; ".join(errors)) + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, default=Path(".")) + parser.add_argument("--stage") + parser.add_argument("--commit") + parser.add_argument("--distributed-timeout-seconds", type=int, default=120) + args = parser.parse_args(sys.argv[1:] if argv is None else argv) + try: + result = run_benchmark( + args.output_dir, + stage=args.stage, + commit=args.commit, + timeout_seconds=args.distributed_timeout_seconds, + ) + except Exception as error: + print(f"{NAME} failed: {error}", file=sys.stderr) + return 1 + print(f"{NAME} passed on rank {result['rank']}/{result['world_size']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unittest/tools/test_infra_dry_run_benchmark.py b/tests/unittest/tools/test_infra_dry_run_benchmark.py new file mode 100644 index 000000000000..6fcec4c4c6e8 --- /dev/null +++ b/tests/unittest/tools/test_infra_dry_run_benchmark.py @@ -0,0 +1,325 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import builtins +import importlib.util +import json +import tempfile +import unittest +import xml.etree.ElementTree as ET +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +SCRIPT_PATH = REPO_ROOT / "jenkins" / "scripts" / "infra_dry_run_benchmark.py" +SPEC = importlib.util.spec_from_file_location("infra_dry_run_benchmark", SCRIPT_PATH) +BENCHMARK = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(BENCHMARK) + + +class _Scalar: + def __init__(self, value): + self.value = value + + def item(self): + return self.value + + +class _Output: + def __init__(self, *, finite=True, numel=16): + self.finite = finite + self._numel = numel + + def numel(self): + return self._numel + + def all(self): + return _Scalar(self.finite) + + def float(self): + return self + + def sum(self): + return _Scalar(128.0) + + +class _Cuda: + def __init__(self, available=True): + self.available = available + self.selected = None + self.synchronized = None + + def is_available(self): + return self.available + + def device_count(self): + return 1 + + def set_device(self, device): + self.selected = device + + def manual_seed_all(self, _seed): + return None + + def synchronize(self, device): + self.synchronized = device + + +class _Probe: + def __init__(self, values): + self.values = values + + def cpu(self): + return self + + def tolist(self): + return self.values + + +class _Distributed: + def __init__(self, remote): + self.remote = remote + self.initialized = False + self.destroyed = False + self.gather_calls = 0 + + def is_available(self): + return True + + def is_nccl_available(self): + return True + + def init_process_group(self, **kwargs): + assert kwargs["backend"] == "nccl" + assert kwargs["init_method"] == "env://" + self.initialized = True + + def is_initialized(self): + return self.initialized + + def all_gather(self, gathered, local): + self.gather_calls += 1 + gathered[0].values = local.values + gathered[1].values = [ + float(self.remote["rank"]), + float(self.remote["world_size"]), + float(self.remote["status"] == "passed"), + float(self.remote["checksum"]), + ] + + def destroy_process_group(self): + self.destroyed = True + self.initialized = False + + +class _Torch: + float16 = "float16" + float64 = "float64" + + def __init__(self, *, cuda=True, finite=True, numel=16, remote=None): + self.cuda = _Cuda(cuda) + self.output = _Output(finite=finite, numel=numel) + self.distributed = _Distributed(remote) if remote else SimpleNamespace() + + def manual_seed(self, _seed): + return None + + def full(self, *_args, **_kwargs): + return object() + + def matmul(self, _left, _right): + return self.output + + def isfinite(self, output): + return output + + def tensor(self, values, **_kwargs): + return _Probe(values) + + def empty_like(self, probe): + return _Probe([0.0] * len(probe.values)) + + +TRTLLM = SimpleNamespace(__version__="1.2.3", __file__="/installed/tensorrt_llm/__init__.py") + + +def _run(output_dir, torch, **kwargs): + with mock.patch.object(BENCHMARK, "_load_runtime_modules", return_value=(TRTLLM, torch)): + return BENCHMARK.run_benchmark(output_dir, **kwargs) + + +def _read_outputs(output_dir): + rank = json.loads((output_dir / "infra_dry_run_rank_0.json").read_text()) + manifest = json.loads((output_dir / "infra_dry_run_manifest.json").read_text()) + junit = ET.parse(output_dir / "results-infra_dry_run.xml") + return rank, manifest, junit + + +def _result(rank, *, world_size=2, status="passed", checksum=128.0): + return { + "rank": rank, + "world_size": world_size, + "status": status, + "checksum": checksum, + "error": "" if status == "passed" else "CUDA work failed", + } + + +class InfraDryRunBenchmarkTest(unittest.TestCase): + def test_runtime_loader_performs_real_package_imports(self): + imported = [] + real_import = builtins.__import__ + modules = {"tensorrt_llm": SimpleNamespace(), "torch": SimpleNamespace()} + + def record_import(name, *args, **kwargs): + if name in modules: + imported.append(name) + return modules[name] + return real_import(name, *args, **kwargs) + + with mock.patch.object(builtins, "__import__", side_effect=record_import): + self.assertEqual( + BENCHMARK._load_runtime_modules(), + (modules["tensorrt_llm"], modules["torch"]), + ) + self.assertEqual(imported, ["tensorrt_llm", "torch"]) + + def test_single_rank_success_writes_rank_manifest_and_junit(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = Path(temp_dir) + torch = _Torch() + result = _run( + output_dir, + torch, + stage="Single-GPU", + commit="deadbeef", + environ={}, + ) + rank, manifest, junit = _read_outputs(output_dir) + + self.assertEqual(result["overall_status"], "passed") + self.assertEqual(rank["tensorrt_llm_module"], TRTLLM.__file__) + self.assertEqual(manifest["product_tests_executed"], 0) + self.assertEqual((manifest["stage"], manifest["commit"]), ("Single-GPU", "deadbeef")) + self.assertEqual(manifest["observed_ranks"], [0]) + self.assertEqual( + junit.find(".//testcase").attrib["name"], + "infra_dry_run_rank_0_cuda_matmul", + ) + self.assertIsNone(junit.find(".//failure")) + self.assertEqual((torch.cuda.selected, torch.cuda.synchronized), (0, 0)) + + def test_cuda_failures_return_nonzero_and_write_failure_junit(self): + scenarios = [ + (_Torch(cuda=False), "CUDA is required"), + (_Torch(finite=False), "non-finite values"), + (_Torch(numel=0), "empty tensor"), + ] + for torch, expected in scenarios: + with self.subTest(expected=expected), tempfile.TemporaryDirectory() as temp_dir: + output_dir = Path(temp_dir) + with self.assertRaises(BENCHMARK.BenchmarkError): + _run(output_dir, torch, environ={}) + rank, manifest, junit = _read_outputs(output_dir) + self.assertIn(expected, rank["error"]) + self.assertEqual(manifest["status"], "failed") + self.assertIsNotNone(junit.find(".//failure")) + + def test_import_failure_writes_failure_artifacts(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = Path(temp_dir) + with mock.patch.object( + BENCHMARK, + "_load_runtime_modules", + side_effect=ImportError("tensorrt_llm is not installed"), + ): + with self.assertRaises(BENCHMARK.BenchmarkError): + BENCHMARK.run_benchmark(output_dir, environ={}) + rank, manifest, junit = _read_outputs(output_dir) + + self.assertIn("is not installed", rank["error"]) + self.assertEqual(manifest["status"], "failed") + self.assertIsNotNone(junit.find(".//failure")) + + def test_multi_rank_uses_one_nccl_gather_and_cleans_up(self): + remote = _result(1) + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = Path(temp_dir) + torch = _Torch(remote=remote) + result = _run( + output_dir, + torch, + stage="Multi-GPU", + environ={"RANK": "0", "LOCAL_RANK": "0", "WORLD_SIZE": "2"}, + ) + _, manifest, junit = _read_outputs(output_dir) + + self.assertEqual(result["overall_status"], "passed") + self.assertEqual(manifest["observed_ranks"], [0, 1]) + self.assertEqual(len(junit.findall(".//testcase")), 3) + self.assertEqual(torch.distributed.gather_calls, 1) + self.assertTrue(torch.distributed.destroyed) + + def test_remote_rank_failure_makes_process_fail_and_cleans_up(self): + remote = _result(1, status="failed", checksum=0.0) + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = Path(temp_dir) + torch = _Torch(remote=remote) + with self.assertRaises(BENCHMARK.BenchmarkError): + _run( + output_dir, + torch, + environ={"RANK": "0", "LOCAL_RANK": "0", "WORLD_SIZE": "2"}, + ) + rank, manifest, junit = _read_outputs(output_dir) + + self.assertEqual(rank["overall_status"], "failed") + self.assertEqual(manifest["status"], "failed") + self.assertIsNotNone(junit.find(".//failure")) + self.assertEqual(torch.distributed.gather_calls, 1) + self.assertTrue(torch.distributed.destroyed) + + def test_manifest_rejects_missing_mismatched_and_inconsistent_ranks(self): + scenarios = [ + ([_result(0)], "observed ranks [0] do not match expected [0, 1]"), + ( + [_result(0), _result(1, world_size=3)], + "rank results contain a world-size mismatch", + ), + ( + [_result(0), _result(1, checksum=256.0)], + "rank results contain inconsistent CUDA checksums", + ), + ] + for summaries, expected in scenarios: + with self.subTest(expected=expected), tempfile.TemporaryDirectory() as temp_dir: + output_dir = Path(temp_dir) + result = { + "world_size": 2, + "stage": "Multi-GPU", + "commit": "abc123", + } + errors = BENCHMARK._validate(summaries, 2) + manifest = BENCHMARK._write_reports(output_dir, result, summaries, errors) + junit = ET.parse(output_dir / "results-infra_dry_run.xml") + self.assertIn(expected, errors) + self.assertEqual(manifest["status"], "failed") + self.assertIsNotNone(junit.find(".//failure")) + + +if __name__ == "__main__": + unittest.main() From 886ac4842cf1aa21782f9cb811fc4f218d21a87e Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:02:45 -0700 Subject: [PATCH 02/34] ci: integrate infrastructure dry run into L0 tests Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 151 ++++++++++++++---- jenkins/scripts/slurm_run.sh | 33 +++- .../tools/test_infra_dry_run_pipeline.py | 95 +++++++++++ 3 files changed, 244 insertions(+), 35 deletions(-) create mode 100644 tests/unittest/tools/test_infra_dry_run_pipeline.py diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 321cb817838a..766148517cf1 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -228,7 +228,14 @@ COMMON_SSH_OPTIONS = Utils.DEFAULT_CUSTOM_SSH_OPTIONS // Per-stage CBTS coverage exclusions applied on top of the upstream eligibility decision. CBTS_EXCLUDE_STAGES = [] as Set +def isInfraDryRun() { + return testFilter[(INFRA_DRY_RUN)] ?: false +} + def isCbtsStage(String stageName) { + if (isInfraDryRun()) { + return false + } // Pipeline-level eligibility (post-merge gate + kill switch) is decided in L0_MergeRequest.groovy and propagated via testFilter. if (!(testFilter[(CBTS_COVERAGE)] ?: false)) { return false @@ -390,43 +397,55 @@ def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String def hasTimeoutTest = false def downloadResultSucceed = false def downloadPerfResultSucceed = false + def downloadInfraResultSucceed = false pipeline.stage('Submit Test Result') { sh "mkdir -p ${stageName}" // Download timeout test results - def timeoutTestFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/unfinished_test.txt" - def downloadTimeoutTestSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, timeoutTestFilePath, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 - if (downloadTimeoutTestSucceed) { - if (stageIsInterrupted) { - echo "Stage is interrupted, skip to generate terminated unexpectedly test result." - } else { - sh "ls -al ${stageName}/" - // Generate timeout test result xml if there are terminated unexpectedly tests - hasTimeoutTest = generateTimeoutTestResultXml(pipeline, stageName) + if (!isInfraDryRun()) { + def timeoutTestFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/unfinished_test.txt" + def downloadTimeoutTestSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, timeoutTestFilePath, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 + if (downloadTimeoutTestSucceed) { + if (stageIsInterrupted) { + echo "Stage is interrupted, skip to generate terminated unexpectedly test result." + } else { + sh "ls -al ${stageName}/" + // Generate timeout test result xml if there are terminated unexpectedly tests + hasTimeoutTest = generateTimeoutTestResultXml(pipeline, stageName) + } } } // Download normal test results def resultsFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/results*.xml" downloadResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, resultsFilePath, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 + if (isInfraDryRun()) { + def infraResultPath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/infra_dry_run*.json" + downloadInfraResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, infraResultPath, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 + if (!downloadInfraResultSucceed) { + error "Failed to collect infrastructure dry-run JSON results for ${stageName}" + } + } // Download perf test results - def perfResultsBasePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}" - def folderListOutput = Utils.exec( - pipeline, - script: Utils.sshUserCmd( - remote, - "\"find '${perfResultsBasePath}' -maxdepth 1 -type d \\( -name 'aggr*' -o -name 'disagg*' \\) -printf '%f\\n' || true\"" - ), - returnStdout: true, - numRetries: 3 - )?.trim() ?: "" - def perfFolders = folderListOutput.split(/\s+/).collect { it.trim().replaceAll(/\/$/, '') }.findAll { it } - echo "Perf Result Folders: ${perfFolders}" - if (perfFolders) { - def scpSources = perfFolders.size() == 1 - ? "${perfResultsBasePath}/${perfFolders[0]}" - : "{${perfFolders.collect { "${perfResultsBasePath}/${it}" }.join(',')}}" - downloadPerfResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, scpSources, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 + if (!isInfraDryRun()) { + def perfResultsBasePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}" + def folderListOutput = Utils.exec( + pipeline, + script: Utils.sshUserCmd( + remote, + "\"find '${perfResultsBasePath}' -maxdepth 1 -type d \\( -name 'aggr*' -o -name 'disagg*' \\) -printf '%f\\n' || true\"" + ), + returnStdout: true, + numRetries: 3 + )?.trim() ?: "" + def perfFolders = folderListOutput.split(/\s+/).collect { it.trim().replaceAll(/\/$/, '') }.findAll { it } + echo "Perf Result Folders: ${perfFolders}" + if (perfFolders) { + def scpSources = perfFolders.size() == 1 + ? "${perfResultsBasePath}/${perfFolders[0]}" + : "{${perfFolders.collect { "${perfResultsBasePath}/${it}" }.join(',')}}" + downloadPerfResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, scpSources, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 + } } // Pull this stage's per-process .cbtscov files as one archive into ${stageName}/cbts/; bounded and non-fatal. @@ -499,7 +518,11 @@ def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String } if ((hasTimeoutTest || downloadResultSucceed) && !suppressTestReporting) { - junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml") + if (isInfraDryRun()) { + junit(testResults: "${stageName}/results-infra_dry_run*.xml") + } else { + junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml") + } } else if (suppressTestReporting) { echo "[INFRA-RETRY] ${stageName}${postTag}: suppressing junit() because a retry is still planned" } @@ -1543,6 +1566,35 @@ def getNodeArgs(int nodeCount, int gpuCount, boolean setSegment = false) { return args } +def getInfraDryRunNodeArgs(int nodeCount, int gpuCount) { + int gpusPerNode = ((gpuCount / nodeCount) as BigDecimal).setScale(0, BigDecimal.ROUND_CEILING).intValue() + return [ + "--nodes=${nodeCount}", + "--ntasks=${gpuCount}", + "--ntasks-per-node=${gpusPerNode}", + "--gpus-per-node=${gpusPerNode}", + ] +} + +def getInfraDryRunDirectCommand(String llmSrc, String outputPath, String stageName, String commit) { + def benchmarkArgs = [ + "${llmSrc}/jenkins/scripts/infra_dry_run_benchmark.py", + "--output-dir '${outputPath}'", + "--stage '${stageName}'", + "--commit '${commit}'", + ].join(" ") + return """ + set -euo pipefail + mkdir -p '${outputPath}' + gpu_count=\$(python3 -c 'import torch; print(torch.cuda.device_count())') + if [ "\$gpu_count" -gt 1 ]; then + torchrun --standalone --nproc-per-node="\$gpu_count" ${benchmarkArgs} + else + python3 ${benchmarkArgs} + fi + """ +} + def getPytestBaseCommandLine( String llmSrc, String stageName, @@ -1926,7 +1978,9 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG .replace("${ARTIFACTORY_DOCKER_HOST}/", "${ARTIFACTORY_DOCKER_HOST}#") } def mounts = getMountListForSlurmTest(cluster, true).join(",") - String[] taskArgs = getNodeArgs(nodeCount, gpuCount, disaggMultiNodeMode) + String[] taskArgs = isInfraDryRun() + ? getInfraDryRunNodeArgs(nodeCount, gpuCount) + : getNodeArgs(nodeCount, gpuCount, disaggMultiNodeMode) if (taskArgs == null) { error "Invalid Slurm test stage name is set" } @@ -2047,6 +2101,10 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG if (ENABLE_UPLOAD_TEST_RESULTS) { srunArgs.add("--container-env=S3_SECRET_KEY") } + if (isInfraDryRun()) { + srunArgs.add("--container-env=MASTER_ADDR") + srunArgs.add("--container-env=MASTER_PORT") + } envVarsToExport.each { varName, varValue -> srunArgs.add("--container-env=${varName}") } @@ -2091,6 +2149,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG export llmSrcNode=$llmSrcNode export stageName=$stageName export perfMode=$perfMode + export infraDryRun=${isInfraDryRun()} export resourcePathNode=$resourcePathNode export pytestCommand="$pytestCommand" export coverageConfigFile="$coverageConfigFile" @@ -2107,10 +2166,15 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG echo "Env NVIDIA_IMEX_CHANNELS: \$NVIDIA_IMEX_CHANNELS" echo "Env NVIDIA_VISIBLE_DEVICES: \$NVIDIA_VISIBLE_DEVICES" + if [ "\$infraDryRun" = "true" ]; then + export MASTER_ADDR=\$(scontrol show hostnames "\$SLURM_JOB_NODELIST" | head -n 1) + export MASTER_PORT=\$((20000 + SLURM_JOB_ID % 20000)) + fi + ${srunPrologue} """.replaceAll("(?m)^\\s*", "") - if (disaggMultiNodeMode || aggMultiNodeMode) { + if (!isInfraDryRun() && (disaggMultiNodeMode || aggMultiNodeMode)) { def scriptLaunchPrefixPathLocal = Utils.createTempLocation(pipeline, "./slurm_launch_prefix.sh") def scriptLaunchSrunArgsPathLocal = Utils.createTempLocation(pipeline, "./slurm_srun_args.txt") // The unified submit.py handles both agg and disagg; only the @@ -2542,7 +2606,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG // CBTS Layer 2.5: rename narrowed stages (reuse-safety) and resize their splits to k. def cbtsResizeSplits(configs) { - def cbts = testFilter[(CBTS_RESULT)] + def cbts = isInfraDryRun() ? null : testFilter[(CBTS_RESULT)] if (cbts == null || !cbts.cbts_test_db_artifact_path) { return configs } @@ -2572,7 +2636,7 @@ def cbtsResizeSplits(configs) { // CBTS Layer 2: replace the normal stage set with the selector's affected // stages while retaining the baseline sanity and multi-GPU gates. def filterCbtsStageJobs(parallelJobs, parallelJobsFiltered, multiGpuJobs, testFilter) { - def cbts = testFilter[(CBTS_RESULT)] + def cbts = isInfraDryRun() ? null : testFilter[(CBTS_RESULT)] if (cbts == null) { return parallelJobsFiltered } @@ -2690,7 +2754,7 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p backoffMs: 60L * 1000L, ] - if (nodeCount > 1 || runWithSbatch) { + if (isInfraDryRun() || nodeCount > 1 || runWithSbatch) { runLLMTestlistWithSbatch(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, nodeCount, skipInstallWheel, cpver, postTag, useClusterDurations, attemptPlacementContext, slurmRetryContext) } else { runLLMTestlistWithAgent(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, skipInstallWheel, cpver, postTag, useClusterDurations, attemptPlacementContext, slurmRetryContext) @@ -2831,6 +2895,8 @@ def CBTS_RESULT = "cbts_result" // Pipeline-level CBTS coverage eligibility, decided in L0_MergeRequest.groovy. @Field def CBTS_COVERAGE = "cbts_coverage" +@Field +def INFRA_DRY_RUN = "infra_dry_run" // Suffix for CBTS-narrowed stages so their results aren't reused by non-CBTS runs. // A suffix (not prefix) keeps the GPU type as the first '-' token for positional parsers. @Field @@ -2858,6 +2924,7 @@ def testFilter = [ (DETAILED_LOG): false, (CBTS_RESULT): null, (CBTS_COVERAGE): false, + (INFRA_DRY_RUN): false, ] @Field @@ -4063,7 +4130,7 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu // If the download or extraction fails we swallow the error: the override // directory will be absent below, the overrideYaml check will fail, and // renderTestDB falls back to the source test-db. - def cbts = testFilter[(CBTS_RESULT)] + def cbts = isInfraDryRun() ? null : testFilter[(CBTS_RESULT)] if (cbts != null && cbts.test_db_dir_override && cbts.cbts_test_db_artifact_path) { try { // Always re-fetch: a reused workspace may hold a stale cbts_test_db/ shadowing this build's YAMLs. @@ -4913,6 +4980,16 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } containerLD_LIBRARY_PATH = containerLD_LIBRARY_PATH.replaceAll(':+$', '') withEnv(["LD_LIBRARY_PATH=${containerLD_LIBRARY_PATH}"]) { + if (isInfraDryRun()) { + def commit = env.artifactCommit ?: env.gitlabCommit ?: "" + sh getInfraDryRunDirectCommand( + llmSrc, + "${WORKSPACE}/${stageName}", + stageName, + commit, + ) + return + } withCredentials([ string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), @@ -4993,6 +5070,10 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } } + if (isInfraDryRun()) { + return + } + // Generate comprehensive rerun report if any reruns occurred stage ("Generate Report") { timeout(time: 15, unit: 'MINUTES'){ @@ -5070,6 +5151,10 @@ def runLLMTestlistOnPlatform(pipeline, platform, testList, config=VANILLA_CONFIG error("Error in post-debug session: ${e.message}") } } + if (isInfraDryRun()) { + sh "ls -al ${stageName}/" + return + } // If the execution test list is null, remove the test result xml sh """ ls -al ${stageName}/ diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index c587825fa167..6277129e9a16 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -1,4 +1,18 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Set up error handling set -xEeuo pipefail @@ -15,9 +29,9 @@ if [ $SLURM_PROCID -eq 0 ]; then fi fi -# Aggregated mode will run install together with pytest in slurm_run.sh +# Aggregated mode and infrastructure dry runs install in slurm_run.sh. # Disaggregated mode will run install separately in slurm_install.sh -if [[ "$stageName" != *Disagg* ]]; then +if [[ "${infraDryRun:-false}" == "true" || "$stageName" != *Disagg* ]]; then installScriptPath="$(dirname "${BASH_SOURCE[0]}")/$(basename "${BASH_SOURCE[0]}" | sed 's/slurm_run\.sh/slurm_install.sh/')" source "$installScriptPath" slurm_install_setup @@ -56,6 +70,21 @@ env | sort echo "Full Command: $pytestCommand" +if [[ "${infraDryRun:-false}" == "true" ]]; then + export RANK="$SLURM_PROCID" + export LOCAL_RANK="$SLURM_LOCALID" + export WORLD_SIZE="$SLURM_NTASKS" + export MASTER_ADDR="${MASTER_ADDR:?MASTER_ADDR must be set by the Slurm launch script}" + export MASTER_PORT="${MASTER_PORT:?MASTER_PORT must be set by the Slurm launch script}" + + python3 "$llmSrcNode/jenkins/scripts/infra_dry_run_benchmark.py" \ + --output-dir "$jobWorkspace" \ + --stage "$stageName" \ + --commit "${gitlabCommit:-}" \ + --distributed-timeout-seconds 900 + exit $? +fi + # For single-node test runs or disaggregated benchmark/server runs, clear all # environment variables related to Slurm and MPI. This prevents test processes # (e.g., pytest) from incorrectly initializing MPI when running under a diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py new file mode 100644 index 000000000000..7c6b3dec394c --- /dev/null +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +GROOVY = (REPO_ROOT / "jenkins" / "L0_Test.groovy").read_text() +SLURM_RUN = (REPO_ROOT / "jenkins" / "scripts" / "slurm_run.sh").read_text() + + +def _function_body(source, name, next_name): + start = source.index(f"def {name}") + end = source.index(f"def {next_name}", start + len(f"def {name}")) + return source[start:end] + + +class InfraDryRunPipelineTest(unittest.TestCase): + def test_direct_command_selects_python_or_torchrun(self): + body = _function_body( + GROOVY, + "getInfraDryRunDirectCommand", + "getPytestBaseCommandLine", + ) + self.assertIn("torch.cuda.device_count()", body) + self.assertIn('if [ "\\$gpu_count" -gt 1 ]', body) + self.assertIn('torchrun --standalone --nproc-per-node="\\$gpu_count"', body) + self.assertIn("python3 ${benchmarkArgs}", body) + + def test_slurm_command_allocates_one_task_per_gpu(self): + body = _function_body(GROOVY, "getInfraDryRunNodeArgs", "getInfraDryRunDirectCommand") + self.assertIn('"--nodes=${nodeCount}"', body) + self.assertIn('"--ntasks=${gpuCount}"', body) + self.assertIn('"--ntasks-per-node=${gpusPerNode}"', body) + self.assertIn('"--gpus-per-node=${gpusPerNode}"', body) + + def test_slurm_maps_ranks_and_uses_stable_rendezvous(self): + for assignment in ( + 'RANK="$SLURM_PROCID"', + 'LOCAL_RANK="$SLURM_LOCALID"', + 'WORLD_SIZE="$SLURM_NTASKS"', + 'MASTER_ADDR="${MASTER_ADDR:?MASTER_ADDR must be set by the Slurm launch script}"', + 'MASTER_PORT="${MASTER_PORT:?MASTER_PORT must be set by the Slurm launch script}"', + ): + self.assertIn(assignment, SLURM_RUN) + self.assertIn('scontrol show hostnames "\\$SLURM_JOB_NODELIST"', GROOVY) + self.assertIn("20000 + SLURM_JOB_ID % 20000", GROOVY) + self.assertIn("--container-env=MASTER_ADDR", GROOVY) + self.assertIn("--container-env=MASTER_PORT", GROOVY) + self.assertIn("--distributed-timeout-seconds 900", SLURM_RUN) + self.assertLess( + SLURM_RUN.index('if [[ "${infraDryRun:-false}" == "true" ]]'), + SLURM_RUN.index("eval $pytestCommand"), + ) + + def test_direct_branch_follows_existing_shard_setup(self): + body = _function_body( + GROOVY, + "runLLMTestlistOnPlatformImpl", + "runLLMTestlistOnPlatform", + ) + command_index = body.index("getInfraDryRunDirectCommand(") + self.assertLess(body.index("processShardTestList("), command_index) + self.assertGreater(body.index("withCredentials([", command_index), command_index) + self.assertGreater(body.index("No tests were executed", command_index), command_index) + + def test_infra_junit_is_required_and_cbts_is_disabled(self): + self.assertIn( + 'junit(testResults: "${stageName}/results-infra_dry_run*.xml")', + GROOVY, + ) + self.assertNotIn( + 'junit(allowEmptyResults: true, testResults: "${stageName}/results-infra_dry_run', + GROOVY, + ) + self.assertIn("Failed to collect infrastructure dry-run JSON results", GROOVY) + cbts_body = _function_body(GROOVY, "isCbtsStage", "scpFromRemoteCmd") + self.assertIn("if (isInfraDryRun())", cbts_body) + self.assertIn("return false", cbts_body) + + +if __name__ == "__main__": + unittest.main() From 2eedc15e4eb7d420893231c1116a21cde7da0289 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Thu, 30 Jul 2026 02:53:14 -0700 Subject: [PATCH 03/34] ci: orchestrate infrastructure dry-run helpers Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 76 +++++++-- .../tools/test_infra_dry_run_pipeline.py | 146 ++++++++++++++++++ 2 files changed, 208 insertions(+), 14 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 133931393234..9f38bc645efe 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -152,12 +152,16 @@ def CBTS_RESULT = "cbts_result" def CBTS_COVERAGE = "cbts_coverage" @Field def DISABLE_CBTS = "disable_cbts" +@Field +def INFRA_DRY_RUN = "infra_dry_run" // Kill switch for CBTS per-test coverage; official post-merge pipeline only, single-GPU stages only in Phase 1. @Field def ENABLE_CBTS_COVERAGE = true @Field def OSS_COMPLIANCE_FILE_CHANGED = "oss_compliance_file_changed" +boolean infraDryRun = params.InfraDryRun?.toString()?.toBoolean() ?: false + def testFilter = [ (REUSE_TEST): gitlabParamsFromBot.get(REUSE_TEST, null), (REUSE_STAGE_LIST): trimForStageList(gitlabParamsFromBot.get(REUSE_STAGE_LIST, null)?.tokenize(',')), @@ -178,6 +182,7 @@ def testFilter = [ (CBTS_RESULT): null, (CBTS_COVERAGE): false, (DISABLE_CBTS): gitlabParamsFromBot.get((DISABLE_CBTS), false), + (INFRA_DRY_RUN): infraDryRun, ] String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) @@ -213,6 +218,7 @@ if (runMode == "nightly_release") { // GenPostMergeBuilds pipelines do not update GitLab status. boolean enableUpdateGitlabStatus = !GEN_POST_MERGE_BUILDS_ONLY && + !testFilter[INFRA_DRY_RUN] && !testFilter[ENABLE_SKIP_TEST] && !testFilter[ONLY_MULTI_GPU_TEST] && !testFilter[DISABLE_MULTI_GPU_TEST] && @@ -353,16 +359,19 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars) } echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}." echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}." - if (!GEN_POST_MERGE_BUILDS_ONLY) { + if (!GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) { trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, 'running', GITLAB_PROJECT_ID, env.gitlabCommit) } testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) - testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars) - // Decide CBTS coverage eligibility here so L0_Test only consumes the propagated flag. - // Coverage runs only on the official post-merge pipeline. - testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE && (env.JOB_NAME ==~ /.*PostMerge.*/) + if (testFilter[INFRA_DRY_RUN]) { + pipeline.echo("CBTS is skipped for the infrastructure dry run.") + } else { + testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars) + // Decide CBTS coverage eligibility here so L0_Test only consumes the propagated flag. + testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE && (env.JOB_NAME ==~ /.*PostMerge.*/) + } pipeline.echo("CBTS coverage eligible: ${testFilter[(CBTS_COVERAGE)]}") testFilter[(OSS_COMPLIANCE_FILE_CHANGED)] = getOssComplianceFileChanged(pipeline, globalVars) getContainerURIs().each { k, v -> @@ -1688,6 +1697,26 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor return buildStatus } +def launchInfraDryRunTestJobs(pipeline, arch, testFilter, globalVars, platform, imageParameters) +{ + String testFilterJson = writeJSON returnText: true, json: testFilter + def additionalParameters = ['testFilter': testFilterJson] + imageParameters + def testJobs = [ + "[Test-${arch}-Single-GPU] Remote Run": { + stage("[Test-${arch}-Single-GPU] Remote Run") { + launchJob(pipeline, "L0_Test-${arch}-Single-GPU", false, false, globalVars, platform, additionalParameters) + } + }, + "[Test-${arch}-Multi-GPU] Remote Run": { + stage("[Test-${arch}-Multi-GPU] Remote Run") { + launchJob(pipeline, "L0_Test-${arch}-Multi-GPU", false, false, globalVars, platform, additionalParameters) + } + }, + ] + testJobs.failFast = false + pipeline.parallel testJobs +} + def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) { stages = [ @@ -1781,6 +1810,15 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "Skipping x86_64 tests (PLC container scanning)" return } + if (testFilter[INFRA_DRY_RUN]) { + def imageParameters = [ + 'dockerImage': globalVars["LLM_DOCKER_IMAGE"], + 'wheelDockerImagePy310': globalVars["LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE"], + 'wheelDockerImagePy312': globalVars["LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE"], + ] + launchInfraDryRunTestJobs(pipeline, "x86_64", testFilter, globalVars, "x86_64", imageParameters) + return + } testStageName = "[Test-x86_64-Single-GPU] Remote Run" def singleGpuTestFailed = false @@ -1933,6 +1971,14 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "Skipping SBSA tests (PLC container scanning)" return } + if (testFilter[INFRA_DRY_RUN]) { + def imageParameters = [ + "dockerImage": globalVars["LLM_SBSA_DOCKER_IMAGE"], + 'wheelDockerImage': globalVars["LLM_SBSA_WHEEL_DOCKER_IMAGE"], + ] + launchInfraDryRunTestJobs(pipeline, "SBSA", testFilter, globalVars, "SBSA", imageParameters) + return + } testStageName = "[Test-SBSA-Single-GPU] Remote Run" def singleGpuTestFailed = false @@ -2239,7 +2285,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } }]} - parallelJobs.failFast = enableFailFast + parallelJobs.failFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast pipeline.parallel parallelJobs } @@ -2263,24 +2309,26 @@ pipeline { post { unsuccessful { script { - if (!GEN_POST_MERGE_BUILDS_ONLY) { + if (!GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) { trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "failed", GITLAB_PROJECT_ID, env.gitlabCommit) } } } success { script { - if (enableUpdateGitlabStatus) { - trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "success", GITLAB_PROJECT_ID, env.gitlabCommit) - } else if (!GEN_POST_MERGE_BUILDS_ONLY) { - trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "canceled", GITLAB_PROJECT_ID, env.gitlabCommit) - trtllm_utils.updateGitlabStatus("Custom Jenkins build", "success", GITLAB_PROJECT_ID, env.gitlabCommit) + if (!testFilter[INFRA_DRY_RUN]) { + if (enableUpdateGitlabStatus) { + trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "success", GITLAB_PROJECT_ID, env.gitlabCommit) + } else if (!GEN_POST_MERGE_BUILDS_ONLY) { + trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "canceled", GITLAB_PROJECT_ID, env.gitlabCommit) + trtllm_utils.updateGitlabStatus("Custom Jenkins build", "success", GITLAB_PROJECT_ID, env.gitlabCommit) + } } } } aborted { script { - if (!GEN_POST_MERGE_BUILDS_ONLY) { + if (!GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) { trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, 'canceled', GITLAB_PROJECT_ID, env.gitlabCommit) } } @@ -2304,7 +2352,7 @@ pipeline { echo "Upload Build Info failed: ${e.toString()}" } } - if (!isReleaseCheckMode && !GEN_POST_MERGE_BUILDS_ONLY) { + if (!isReleaseCheckMode && !GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) { collectTestResults(this, testFilter, globalVars) } } diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 7c6b3dec394c..1d6efe8485a2 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -13,11 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re import unittest from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent GROOVY = (REPO_ROOT / "jenkins" / "L0_Test.groovy").read_text() +PARENT_GROOVY = (REPO_ROOT / "jenkins" / "L0_MergeRequest.groovy").read_text() SLURM_RUN = (REPO_ROOT / "jenkins" / "scripts" / "slurm_run.sh").read_text() @@ -27,6 +29,14 @@ def _function_body(source, name, next_name): return source[start:end] +def _map_keys(source, assignment_index): + start = source.index("[", assignment_index) + line_start = source.rindex("\n", 0, assignment_index) + 1 + indentation = source[line_start:assignment_index] + end = source.index(f"\n{indentation}]", start) + return set(re.findall(r"""['"]([^'"]+)['"]\s*:""", source[start:end])) + + class InfraDryRunPipelineTest(unittest.TestCase): def test_direct_command_selects_python_or_torchrun(self): body = _function_body( @@ -91,5 +101,141 @@ def test_infra_junit_is_required_and_cbts_is_disabled(self): self.assertIn("return false", cbts_body) +class InfraDryRunParentPipelineTest(unittest.TestCase): + def test_parameter_is_propagated_to_the_helper_filter(self): + filter_setup = PARENT_GROOVY[ + PARENT_GROOVY.index("boolean infraDryRun ="): + PARENT_GROOVY.index("String reuseBuild =") + ] + self.assertIn("params.InfraDryRun?.toString()?.toBoolean()", filter_setup) + self.assertIn("(INFRA_DRY_RUN): infraDryRun", filter_setup) + + def test_dry_run_helpers_share_parameters_and_do_not_fail_fast(self): + body = _function_body( + PARENT_GROOVY, + "launchInfraDryRunTestJobs", + "launchStages", + ) + self.assertEqual(body.count("def additionalParameters ="), 1) + self.assertIn('"L0_Test-${arch}-Single-GPU"', body) + self.assertIn('"L0_Test-${arch}-Multi-GPU"', body) + self.assertEqual(body.count("additionalParameters)"), 2) + self.assertEqual(body.count(", false, false, globalVars,"), 2) + self.assertLess( + body.index("testJobs.failFast = false"), + body.index("pipeline.parallel testJobs"), + ) + + def test_helper_failure_propagates_after_parallel_siblings_finish(self): + helper = _function_body( + PARENT_GROOVY, + "launchInfraDryRunTestJobs", + "launchStages", + ) + launch_job = _function_body( + PARENT_GROOVY, + "launchJob", + "launchInfraDryRunTestJobs", + ) + self.assertNotIn("catchError", helper) + self.assertNotIn("catch (", helper) + self.assertIn('if (buildStatus != "SUCCESS")', launch_job) + self.assertIn('error "Downstream job did not succeed"', launch_job) + self.assertIn("testJobs.failFast = false", helper) + + def test_shared_filter_and_image_parameters_are_read_only_and_match_normal_jobs(self): + helper = _function_body( + PARENT_GROOVY, + "launchInfraDryRunTestJobs", + "launchStages", + ) + launch_job = _function_body( + PARENT_GROOVY, + "launchJob", + "launchInfraDryRunTestJobs", + ) + self.assertNotIn("additionalParameters[", helper + launch_job) + self.assertNotIn("additionalParameters.put", helper + launch_job) + self.assertIn("parameters += [", launch_job) + + start = PARENT_GROOVY.index("def launchStages") + launch_stages = PARENT_GROOVY[ + start:PARENT_GROOVY.index("\npipeline {", start) + ] + expected_keys = { + "x86_64": { + "dockerImage", + "wheelDockerImagePy310", + "wheelDockerImagePy312", + }, + "SBSA": {"dockerImage", "wheelDockerImage"}, + } + for arch, expected in expected_keys.items(): + dry_call = launch_stages.index( + f'launchInfraDryRunTestJobs(pipeline, "{arch}"' + ) + dry_map = launch_stages.rindex("def imageParameters = [", 0, dry_call) + normal_stage = launch_stages.index( + f'testStageName = "[Test-{arch}-Single-GPU] Remote Run"', + dry_call, + ) + normal_map = launch_stages.index( + "def additionalParameters = [", + normal_stage, + ) + self.assertEqual(_map_keys(launch_stages, dry_map), expected) + self.assertEqual( + _map_keys(launch_stages, normal_map) - {"testFilter"}, + expected, + ) + + def test_dry_run_branch_precedes_normal_single_gpu_gating(self): + start = PARENT_GROOVY.index("def launchStages") + launch_stages = PARENT_GROOVY[ + start:PARENT_GROOVY.index("\npipeline {", start) + ] + for arch in ("x86_64", "SBSA"): + dry_run_call = launch_stages.index( + f'launchInfraDryRunTestJobs(pipeline, "{arch}"' + ) + build_call = launch_stages.rindex( + f'launchJob(pipeline, "/LLM/helpers/Build-{arch}"', + 0, + dry_run_call, + ) + normal_single = launch_stages.index( + f'testStageName = "[Test-{arch}-Single-GPU] Remote Run"', + dry_run_call, + ) + marker = launch_stages.index( + f'currentBuild.description?.contains("Require {arch} Multi-GPU Testing")', + normal_single, + ) + self.assertLess(build_call, dry_run_call) + self.assertLess(dry_run_call, normal_single) + self.assertLess(normal_single, marker) + self.assertIn( + "parallelJobs.failFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast", + launch_stages, + ) + + def test_product_reporting_is_excluded_from_dry_run(self): + setup = _function_body( + PARENT_GROOVY, + "setupPipelineEnvironment", + "mergeWaiveList", + ) + self.assertLess( + setup.index("if (testFilter[INFRA_DRY_RUN])"), + setup.index("getCbtsResult("), + ) + self.assertIn( + "!testFilter[INFRA_DRY_RUN]) {\n" + " collectTestResults(", + PARENT_GROOVY, + ) + self.assertNotIn("L0_Stability", PARENT_GROOVY) + + if __name__ == "__main__": unittest.main() From d64e24731df712f101b34aaf109627f8667acb3b Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:22:04 -0700 Subject: [PATCH 04/34] [TRTLLMINF-161] ci: harden infrastructure dry run Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 30 +++++-- jenkins/L0_Test.groovy | 2 + .../tools/test_infra_dry_run_pipeline.py | 90 +++++++++++++++---- 3 files changed, 100 insertions(+), 22 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 9f38bc645efe..97e05ba1e2c6 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -362,9 +362,16 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars) if (!GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) { trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, 'running', GITLAB_PROJECT_ID, env.gitlabCommit) } - testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) - testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) - testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) + if (testFilter[INFRA_DRY_RUN]) { + pipeline.echo("Changed-file analysis is skipped for the infrastructure dry run.") + testFilter[(MULTI_GPU_FILE_CHANGED)] = false + testFilter[(ONLY_ONE_GROUP_CHANGED)] = "" + testFilter[(AUTO_TRIGGER_TAG_LIST)] = [] + } else { + testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) + testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) + testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) + } if (testFilter[INFRA_DRY_RUN]) { pipeline.echo("CBTS is skipped for the infrastructure dry run.") } else { @@ -472,7 +479,11 @@ def preparation(pipeline, testFilter, globalVars) setupPipelineEnvironment(pipeline, testFilter, globalVars) } stage("Merge Test Waive List") { - mergeWaiveList(pipeline, globalVars) + if (testFilter[INFRA_DRY_RUN]) { + echo "Skipping Merge Test Waive List for the infrastructure dry run." + } else { + mergeWaiveList(pipeline, globalVars) + } } }) } @@ -1722,7 +1733,10 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) stages = [ "Release-Check": { script { - if (GEN_POST_MERGE_BUILDS_ONLY) { + if (testFilter[INFRA_DRY_RUN]) { + echo "Skipping Release-Check for the infrastructure dry run." + return + } else if (GEN_POST_MERGE_BUILDS_ONLY) { echo "Skipping Release-Check (GenPostMergeBuilds mode: builds only)" return } @@ -2381,7 +2395,11 @@ pipeline { if (isReleaseCheckMode) { stage("Release-Check") { script { - launchReleaseCheck(this, globalVars) + if (testFilter[INFRA_DRY_RUN]) { + echo "Skipping Release-Check for the infrastructure dry run." + } else { + launchReleaseCheck(this, globalVars) + } } } } else { diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 766148517cf1..56d366ebddf4 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -7020,6 +7020,8 @@ pipeline { } if (singleGpuJobs.size() > 0) { runBranchesWithInfraDefer(singleGpuJobs, params.enableFailFast, stageInfraScope) + } else if (isInfraDryRun()) { + error "Skip single-GPU testing. No test to run for infrastructure dry run." } else { echo "Skip single-GPU testing. No test to run." } diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 1d6efe8485a2..96a0bd2fae4f 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -100,12 +100,28 @@ def test_infra_junit_is_required_and_cbts_is_disabled(self): self.assertIn("if (isInfraDryRun())", cbts_body) self.assertIn("return false", cbts_body) + def test_empty_single_gpu_filter_fails_only_for_dry_run(self): + single_branch_start = GROOVY.index("if (env.JOB_NAME ==~ /.*Single-GPU.*/)") + single_branch_end = GROOVY.index( + "} else if (env.JOB_NAME ==~ /.*Multi-GPU.*/)", + single_branch_start, + ) + single_branch = GROOVY[single_branch_start:single_branch_end] + dry_guard = single_branch.index("else if (isInfraDryRun())") + dry_error = single_branch.index( + 'error "Skip single-GPU testing. No test to run for infrastructure dry run."' + ) + normal_skip = single_branch.index('echo "Skip single-GPU testing. No test to run."') + self.assertLess(dry_guard, dry_error) + self.assertLess(dry_error, normal_skip) + class InfraDryRunParentPipelineTest(unittest.TestCase): def test_parameter_is_propagated_to_the_helper_filter(self): filter_setup = PARENT_GROOVY[ - PARENT_GROOVY.index("boolean infraDryRun ="): - PARENT_GROOVY.index("String reuseBuild =") + PARENT_GROOVY.index("boolean infraDryRun =") : PARENT_GROOVY.index( + "String reuseBuild =" + ) ] self.assertIn("params.InfraDryRun?.toString()?.toBoolean()", filter_setup) self.assertIn("(INFRA_DRY_RUN): infraDryRun", filter_setup) @@ -159,9 +175,7 @@ def test_shared_filter_and_image_parameters_are_read_only_and_match_normal_jobs( self.assertIn("parameters += [", launch_job) start = PARENT_GROOVY.index("def launchStages") - launch_stages = PARENT_GROOVY[ - start:PARENT_GROOVY.index("\npipeline {", start) - ] + launch_stages = PARENT_GROOVY[start : PARENT_GROOVY.index("\npipeline {", start)] expected_keys = { "x86_64": { "dockerImage", @@ -171,9 +185,7 @@ def test_shared_filter_and_image_parameters_are_read_only_and_match_normal_jobs( "SBSA": {"dockerImage", "wheelDockerImage"}, } for arch, expected in expected_keys.items(): - dry_call = launch_stages.index( - f'launchInfraDryRunTestJobs(pipeline, "{arch}"' - ) + dry_call = launch_stages.index(f'launchInfraDryRunTestJobs(pipeline, "{arch}"') dry_map = launch_stages.rindex("def imageParameters = [", 0, dry_call) normal_stage = launch_stages.index( f'testStageName = "[Test-{arch}-Single-GPU] Remote Run"', @@ -191,13 +203,9 @@ def test_shared_filter_and_image_parameters_are_read_only_and_match_normal_jobs( def test_dry_run_branch_precedes_normal_single_gpu_gating(self): start = PARENT_GROOVY.index("def launchStages") - launch_stages = PARENT_GROOVY[ - start:PARENT_GROOVY.index("\npipeline {", start) - ] + launch_stages = PARENT_GROOVY[start : PARENT_GROOVY.index("\npipeline {", start)] for arch in ("x86_64", "SBSA"): - dry_run_call = launch_stages.index( - f'launchInfraDryRunTestJobs(pipeline, "{arch}"' - ) + dry_run_call = launch_stages.index(f'launchInfraDryRunTestJobs(pipeline, "{arch}"') build_call = launch_stages.rindex( f'launchJob(pipeline, "/LLM/helpers/Build-{arch}"', 0, @@ -230,12 +238,62 @@ def test_product_reporting_is_excluded_from_dry_run(self): setup.index("getCbtsResult("), ) self.assertIn( - "!testFilter[INFRA_DRY_RUN]) {\n" - " collectTestResults(", + "!testFilter[INFRA_DRY_RUN]) {\n collectTestResults(", PARENT_GROOVY, ) self.assertNotIn("L0_Stability", PARENT_GROOVY) + def test_dry_run_skips_changed_file_analysis(self): + setup = _function_body( + PARENT_GROOVY, + "setupPipelineEnvironment", + "mergeWaiveList", + ) + first_guard = setup.index("if (testFilter[INFRA_DRY_RUN])") + second_guard = setup.index("if (testFilter[INFRA_DRY_RUN])", first_guard + 1) + changed_file_block = setup[first_guard:second_guard] + normal_path = changed_file_block.index("} else {") + self.assertIn("Changed-file analysis is skipped", changed_file_block[:normal_path]) + self.assertIn("(MULTI_GPU_FILE_CHANGED)] = false", changed_file_block[:normal_path]) + self.assertIn('(ONLY_ONE_GROUP_CHANGED)] = ""', changed_file_block[:normal_path]) + self.assertIn("(AUTO_TRIGGER_TAG_LIST)] = []", changed_file_block[:normal_path]) + for call in ( + "getMultiGpuFileChanged(", + "getOnlyOneGroupChanged(", + "getAutoTriggerTagList(", + ): + self.assertGreater(changed_file_block.index(call), normal_path) + + def test_dry_run_skips_waive_merge_and_release_check(self): + preparation = _function_body(PARENT_GROOVY, "preparation", "launchReleaseCheck") + waive_stage = preparation[preparation.index('stage("Merge Test Waive List")') :] + waive_guard = waive_stage.index("if (testFilter[INFRA_DRY_RUN])") + waive_skip = waive_stage.index("Skipping Merge Test Waive List") + waive_normal = waive_stage.index("mergeWaiveList(") + self.assertLess(waive_guard, waive_skip) + self.assertLess(waive_skip, waive_normal) + + launch_stages_start = PARENT_GROOVY.index("def launchStages") + launch_stages = PARENT_GROOVY[ + launch_stages_start : PARENT_GROOVY.index("\npipeline {", launch_stages_start) + ] + release_branch = launch_stages[ + launch_stages.index('"Release-Check":') : launch_stages.index('"x86_64-Linux":') + ] + self.assertLess( + release_branch.index("if (testFilter[INFRA_DRY_RUN])"), + release_branch.index("launchReleaseCheck("), + ) + + release_mode = PARENT_GROOVY.index("if (isReleaseCheckMode)") + release_only = PARENT_GROOVY[ + release_mode : PARENT_GROOVY.index("launchStages(this", release_mode) + ] + self.assertLess( + release_only.index("if (testFilter[INFRA_DRY_RUN])"), + release_only.index("launchReleaseCheck("), + ) + if __name__ == "__main__": unittest.main() From 841393c45a883fc47897e4a962e512afd03f4b82 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Mon, 3 Aug 2026 00:50:41 -0700 Subject: [PATCH 05/34] [TRTLLMINF-161] ci: collect dry-run test results Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 2 +- tests/unittest/tools/test_infra_dry_run_pipeline.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 97e05ba1e2c6..b4a4da8161a8 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -2366,7 +2366,7 @@ pipeline { echo "Upload Build Info failed: ${e.toString()}" } } - if (!isReleaseCheckMode && !GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) { + if (!isReleaseCheckMode && !GEN_POST_MERGE_BUILDS_ONLY) { collectTestResults(this, testFilter, globalVars) } } diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 96a0bd2fae4f..8f384e679ada 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -227,7 +227,7 @@ def test_dry_run_branch_precedes_normal_single_gpu_gating(self): launch_stages, ) - def test_product_reporting_is_excluded_from_dry_run(self): + def test_product_reporting_is_excluded_but_test_results_are_collected(self): setup = _function_body( PARENT_GROOVY, "setupPipelineEnvironment", @@ -237,10 +237,16 @@ def test_product_reporting_is_excluded_from_dry_run(self): setup.index("if (testFilter[INFRA_DRY_RUN])"), setup.index("getCbtsResult("), ) + always_start = PARENT_GROOVY.index(" always {") + always_block = PARENT_GROOVY[ + always_start : PARENT_GROOVY.index(" stages {", always_start) + ] self.assertIn( - "!testFilter[INFRA_DRY_RUN]) {\n collectTestResults(", - PARENT_GROOVY, + "if (!isReleaseCheckMode && !GEN_POST_MERGE_BUILDS_ONLY) {", + always_block, ) + self.assertIn("collectTestResults(this, testFilter, globalVars)", always_block) + self.assertNotIn("testFilter[INFRA_DRY_RUN]", always_block) self.assertNotIn("L0_Stability", PARENT_GROOVY) def test_dry_run_skips_changed_file_analysis(self): From f9552834f6f751dfa4371d0e1dfa367dbdb1fdd1 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:02:11 -0700 Subject: [PATCH 06/34] [TRTLLMINF-161] ci: cover CPU and docs dry runs Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 33 ++++++-- jenkins/scripts/infra_dry_run_benchmark.py | 82 ++++++++++++++----- .../tools/test_infra_dry_run_benchmark.py | 27 +++++- .../tools/test_infra_dry_run_pipeline.py | 39 +++++++++ 4 files changed, 153 insertions(+), 28 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 56d366ebddf4..5f05301ccb0c 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1577,20 +1577,26 @@ def getInfraDryRunNodeArgs(int nodeCount, int gpuCount) { } def getInfraDryRunDirectCommand(String llmSrc, String outputPath, String stageName, String commit) { + def deviceType = stageName.startsWith("CPU-") ? "cpu" : "cuda" def benchmarkArgs = [ "${llmSrc}/jenkins/scripts/infra_dry_run_benchmark.py", "--output-dir '${outputPath}'", "--stage '${stageName}'", "--commit '${commit}'", + "--device '${deviceType}'", ].join(" ") return """ - set -euo pipefail + set -eu mkdir -p '${outputPath}' - gpu_count=\$(python3 -c 'import torch; print(torch.cuda.device_count())') - if [ "\$gpu_count" -gt 1 ]; then - torchrun --standalone --nproc-per-node="\$gpu_count" ${benchmarkArgs} - else + if [ '${deviceType}' = 'cpu' ]; then python3 ${benchmarkArgs} + else + gpu_count=\$(python3 -c 'import torch; print(torch.cuda.device_count())') + if [ "\$gpu_count" -gt 1 ]; then + torchrun --standalone --nproc-per-node="\$gpu_count" ${benchmarkArgs} + else + python3 ${benchmarkArgs} + fi fi """ } @@ -3772,7 +3778,7 @@ def echoNodeAndGpuInfo(pipeline, stageName) pipeline.echo "HOST_NODE_NAME = ${hostNodeName} ; GPU_UUIDS = ${gpuUuids} ; STAGE_NAME = ${stageName}" } -def runLLMDocBuild(pipeline, config) +def runLLMDocBuild(pipeline, config, stageName) { // Step 1: cloning source code sh "pwd && ls -alh" @@ -3797,6 +3803,17 @@ def runLLMDocBuild(pipeline, config) trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmSrc} && pip3 install -r requirements-dev.txt") trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmPath} && pip3 install --force-reinstall --no-deps TensorRT-LLM/tensorrt_llm-*.whl") + if (isInfraDryRun()) { + def commit = env.artifactCommit ?: env.gitlabCommit ?: "" + sh getInfraDryRunDirectCommand( + llmSrc, + "${WORKSPACE}/${stageName}", + stageName, + commit, + ) + return + } + // Step 3: build doc trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y doxygen python3-pip graphviz") @@ -6359,7 +6376,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) docBuildConfigs = [ "CPU-Build_Docs": [docBuildSpec, { sh "rm -rf **/*.xml *.tar.gz" - runLLMDocBuild(pipeline, config=VANILLA_CONFIG) + runLLMDocBuild(pipeline, VANILLA_CONFIG, "A10-Build_Docs") }], ] @@ -6374,7 +6391,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) // pod-launch attempt; isFinalAttempt suppresses synthetic stage-fail XML // and junit() on intermediate retryable infra failures. stage("[${key}] Run") { - cacheErrorAndUploadResult("${key}", values[1], {}, true, attemptTag, isFinalAttempt, retryContext) + cacheErrorAndUploadResult("${key}", values[1], {}, !isInfraDryRun(), attemptTag, isFinalAttempt, retryContext) } }]]} diff --git a/jenkins/scripts/infra_dry_run_benchmark.py b/jenkins/scripts/infra_dry_run_benchmark.py index f399f4b64175..d18462a2a8e7 100644 --- a/jenkins/scripts/infra_dry_run_benchmark.py +++ b/jenkins/scripts/infra_dry_run_benchmark.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Run a small CUDA/NCCL smoke test and emit CI-friendly result artifacts.""" +"""Run a small CPU or CUDA/NCCL smoke test and emit CI-friendly result artifacts.""" import argparse import json @@ -38,9 +38,10 @@ class BenchmarkError(RuntimeError): def _load_runtime_modules() -> tuple[Any, Any]: # Normal imports intentionally exercise the installed package and PyTorch. - import tensorrt_llm import torch + import tensorrt_llm + return tensorrt_llm, torch @@ -63,12 +64,17 @@ def _new_result( stage: str | None, commit: str | None, environ: Mapping[str, str], + device_type: str, ) -> dict[str, Any]: return { "name": NAME, "status": "failed", "product_tests_executed": 0, **context, + "device_type": device_type, + "distributed_backend": ( + "nccl" if device_type == "cuda" and context["world_size"] > 1 else "none" + ), "stage": stage or environ.get("STAGE_NAME") or environ.get("stageName") or "", "commit": commit or environ.get("GIT_COMMIT") or environ.get("gitlabCommit") or "", "tensorrt_llm_version": "unknown", @@ -110,6 +116,26 @@ def _cuda_matmul(torch: Any, local_rank: int, device: str) -> dict[str, Any]: } +def _cpu_matmul(torch: Any) -> dict[str, Any]: + torch.manual_seed(0) + left = torch.full((MATRIX_SIZE, MATRIX_SIZE), 0.5, dtype=torch.float32, device="cpu") + right = torch.full((MATRIX_SIZE, MATRIX_SIZE), 0.25, dtype=torch.float32, device="cpu") + output = torch.matmul(left, right) + if int(output.numel()) == 0: + raise BenchmarkError("CPU matrix multiplication returned an empty tensor") + if not bool(torch.isfinite(output).all().item()): + raise BenchmarkError("CPU matrix multiplication returned non-finite values") + checksum = float(output.float().sum().item()) + if not math.isfinite(checksum): + raise BenchmarkError("CPU matrix multiplication checksum is non-finite") + return { + "device": "cpu", + "matrix_size": MATRIX_SIZE, + "dtype": "float32", + "checksum": checksum, + } + + def _initialize_distributed(torch: Any, context: Mapping[str, int], timeout_seconds: int) -> bool: if context["world_size"] == 1: return False @@ -126,11 +152,12 @@ def _initialize_distributed(torch: Any, context: Mapping[str, int], timeout_seco def _summary(result: Mapping[str, Any]) -> dict[str, Any]: + device_type = str(result.get("device_type", "cuda")) return { "rank": int(result["rank"]), "world_size": int(result["world_size"]), "status": str(result["status"]), - "checksum": result.get("cuda", {}).get("checksum"), + "checksum": result.get(device_type, {}).get("checksum"), "error": str(result.get("error", "")), } @@ -173,11 +200,9 @@ def _validate(summaries: list[Mapping[str, Any]], world_size: int) -> list[str]: if failed: errors.append(f"rank(s) {sorted(failed)} reported failure") try: - checksums = [ - float(item["checksum"]) for item in summaries if item["status"] == "passed" - ] + checksums = [float(item["checksum"]) for item in summaries if item["status"] == "passed"] except (TypeError, ValueError): - errors.append("passed rank result is missing a valid CUDA checksum") + errors.append("passed rank result is missing a valid benchmark checksum") else: if any(not math.isfinite(value) for value in checksums): errors.append("rank results contain a non-finite CUDA checksum") @@ -212,6 +237,8 @@ def _write_reports( "stage": result["stage"], "commit": result["commit"], "world_size": world_size, + "device_type": result.get("device_type", "cuda"), + "distributed_backend": result.get("distributed_backend", "none"), "observed_ranks": sorted(int(item["rank"]) for item in summaries), "validation_errors": errors, "rank_results": summaries, @@ -221,6 +248,7 @@ def _write_reports( by_rank = {int(item["rank"]): item for item in summaries} cases: list[tuple[str, str | None]] = [] + device_type = str(result.get("device_type", "cuda")) for rank in range(world_size): item = by_rank.get(rank) failure = None @@ -228,7 +256,7 @@ def _write_reports( failure = f"missing result for rank {rank}" elif item["status"] != "passed": failure = str(item.get("error") or f"rank {rank} failed") - cases.append((f"{NAME}_rank_{rank}_cuda_matmul", failure)) + cases.append((f"{NAME}_rank_{rank}_{device_type}_matmul", failure)) if world_size > 1: cases.append((f"{NAME}_nccl_collective", "; ".join(errors) or None)) @@ -244,7 +272,14 @@ def _write_reports( }, ) properties = ET.SubElement(suite, "properties") - for name in ("product_tests_executed", "stage", "commit", "world_size"): + for name in ( + "product_tests_executed", + "stage", + "commit", + "world_size", + "device_type", + "distributed_backend", + ): ET.SubElement(properties, "property", {"name": name, "value": str(manifest[name])}) for name, failure in cases: case = ET.SubElement( @@ -270,11 +305,12 @@ def run_benchmark( stage: str | None = None, commit: str | None = None, timeout_seconds: int = 120, + device_type: str = "cuda", environ: Mapping[str, str] | None = None, ) -> dict[str, Any]: environment = os.environ if environ is None else environ context = _rank_context(environment) - result = _new_result(context, stage, commit, environment) + result = _new_result(context, stage, commit, environment, device_type) summaries = [_summary(result)] torch = None distributed = False @@ -283,13 +319,19 @@ def run_benchmark( trtllm, torch = _load_runtime_modules() result["tensorrt_llm_version"] = str(getattr(trtllm, "__version__", "unknown")) result["tensorrt_llm_module"] = str(getattr(trtllm, "__file__", "unknown")) - device = _select_cuda_device(torch, context["local_rank"]) - distributed = _initialize_distributed(torch, context, timeout_seconds) - try: - result["cuda"] = _cuda_matmul(torch, context["local_rank"], device) + if device_type == "cpu": + if context["world_size"] != 1: + raise BenchmarkError("CPU mode requires WORLD_SIZE=1") + result["cpu"] = _cpu_matmul(torch) result["status"] = "passed" - except Exception as error: - result["error"] = f"{type(error).__name__}: {error}" + else: + device = _select_cuda_device(torch, context["local_rank"]) + distributed = _initialize_distributed(torch, context, timeout_seconds) + try: + result["cuda"] = _cuda_matmul(torch, context["local_rank"], device) + result["status"] = "passed" + except Exception as error: + result["error"] = f"{type(error).__name__}: {error}" summaries = _gather_summaries(torch, result) if distributed else [_summary(result)] except Exception as error: result["status"] = "failed" @@ -302,9 +344,9 @@ def run_benchmark( except Exception as error: result["status"] = "failed" result["error"] = f"distributed cleanup failed: {error}" - summaries = [ - item for item in summaries if int(item["rank"]) != context["rank"] - ] + [_summary(result)] + summaries = [item for item in summaries if int(item["rank"]) != context["rank"]] + [ + _summary(result) + ] errors = _validate(summaries, context["world_size"]) result["overall_status"] = "failed" if errors else "passed" @@ -321,6 +363,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--output-dir", type=Path, default=Path(".")) parser.add_argument("--stage") parser.add_argument("--commit") + parser.add_argument("--device", choices=("cpu", "cuda"), default="cuda") parser.add_argument("--distributed-timeout-seconds", type=int, default=120) args = parser.parse_args(sys.argv[1:] if argv is None else argv) try: @@ -329,6 +372,7 @@ def main(argv: list[str] | None = None) -> int: stage=args.stage, commit=args.commit, timeout_seconds=args.distributed_timeout_seconds, + device_type=args.device, ) except Exception as error: print(f"{NAME} failed: {error}", file=sys.stderr) diff --git a/tests/unittest/tools/test_infra_dry_run_benchmark.py b/tests/unittest/tools/test_infra_dry_run_benchmark.py index 6fcec4c4c6e8..80a3e857b11f 100644 --- a/tests/unittest/tools/test_infra_dry_run_benchmark.py +++ b/tests/unittest/tools/test_infra_dry_run_benchmark.py @@ -128,6 +128,7 @@ def destroy_process_group(self): class _Torch: float16 = "float16" + float32 = "float32" float64 = "float64" def __init__(self, *, cuda=True, finite=True, numel=16, remote=None): @@ -196,7 +197,7 @@ def record_import(name, *args, **kwargs): BENCHMARK._load_runtime_modules(), (modules["tensorrt_llm"], modules["torch"]), ) - self.assertEqual(imported, ["tensorrt_llm", "torch"]) + self.assertCountEqual(imported, ["tensorrt_llm", "torch"]) def test_single_rank_success_writes_rank_manifest_and_junit(self): with tempfile.TemporaryDirectory() as temp_dir: @@ -239,6 +240,30 @@ def test_cuda_failures_return_nonzero_and_write_failure_junit(self): self.assertEqual(manifest["status"], "failed") self.assertIsNotNone(junit.find(".//failure")) + def test_cpu_mode_runs_without_cuda_and_writes_cpu_metadata(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = Path(temp_dir) + result = _run( + output_dir, + _Torch(cuda=False), + stage="CPU-Generic-x86-1", + device_type="cpu", + environ={}, + ) + rank, manifest, junit = _read_outputs(output_dir) + + self.assertEqual(result["overall_status"], "passed") + self.assertEqual(rank["cpu"]["device"], "cpu") + self.assertEqual(manifest["device_type"], "cpu") + self.assertEqual(manifest["distributed_backend"], "none") + self.assertEqual(manifest["product_tests_executed"], 0) + self.assertIsNone(result.get("cuda")) + self.assertEqual( + junit.find(".//testcase").attrib["name"], + "infra_dry_run_rank_0_cpu_matmul", + ) + self.assertIsNone(junit.find(".//failure")) + def test_import_failure_writes_failure_artifacts(self): with tempfile.TemporaryDirectory() as temp_dir: output_dir = Path(temp_dir) diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 8f384e679ada..8b37dfa98976 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -45,9 +45,48 @@ def test_direct_command_selects_python_or_torchrun(self): "getPytestBaseCommandLine", ) self.assertIn("torch.cuda.device_count()", body) + self.assertIn("if [ '${deviceType}' = 'cpu' ]", body) self.assertIn('if [ "\\$gpu_count" -gt 1 ]', body) self.assertIn('torchrun --standalone --nproc-per-node="\\$gpu_count"', body) self.assertIn("python3 ${benchmarkArgs}", body) + self.assertIn('stageName.startsWith("CPU-") ? "cpu" : "cuda"', body) + self.assertIn("\"--device '${deviceType}'\"", body) + + def test_direct_command_is_posix_shell_compatible(self): + body = _function_body( + GROOVY, + "getInfraDryRunDirectCommand", + "getPytestBaseCommandLine", + ) + self.assertIn("set -eu", body) + self.assertNotIn("pipefail", body) + + def test_docs_dry_run_bypasses_normal_doc_build_and_keeps_results(self): + body = _function_body(GROOVY, "runLLMDocBuild", "launchTestListCheck") + dry_guard = body.index("if (isInfraDryRun())") + benchmark = body.index("getInfraDryRunDirectCommand(", dry_guard) + early_return = body.index("return", benchmark) + sphinx = body.index("make html") + self.assertLess(dry_guard, benchmark) + self.assertLess(benchmark, early_return) + self.assertLess(early_return, sphinx) + self.assertIn('"${WORKSPACE}/${stageName}"', body[dry_guard:early_return]) + + doc_jobs = GROOVY[ + GROOVY.index("docBuildConfigs = [") : GROOVY.index("// Python version and OS") + ] + self.assertIn('runLLMDocBuild(pipeline, VANILLA_CONFIG, "A10-Build_Docs")', doc_jobs) + self.assertIn("{}, !isInfraDryRun(), attemptTag", doc_jobs) + + def test_package_sanity_uses_the_shared_direct_benchmark_path(self): + package_jobs = GROOVY[ + GROOVY.index("sanityCheckJobs =") : GROOVY.index( + "multiGpuJobs =", GROOVY.index("sanityCheckJobs =") + ) + ] + self.assertIn("runLLMTestlistOnPlatform(", package_jobs) + self.assertIn("toStageName(values[1], key)", package_jobs) + self.assertNotIn('"CPU-', package_jobs) def test_slurm_command_allocates_one_task_per_gpu(self): body = _function_body(GROOVY, "getInfraDryRunNodeArgs", "getInfraDryRunDirectCommand") From 8c0ddeab7847fdef3d9ee4c28021a14384c7abe5 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:06:04 -0700 Subject: [PATCH 07/34] TRTLLMINF-161 Simplify dry-run test orchestration Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 30 +++----- .../tools/test_infra_dry_run_pipeline.py | 70 ++++++++----------- 2 files changed, 42 insertions(+), 58 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index b4a4da8161a8..c0e2298303f0 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1663,7 +1663,7 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor ] } - if (env.testPhase2StageName) { + if (!additionalParameters.containsKey('testPhase2StageName') && env.testPhase2StageName) { parameters += [ 'testPhase2StageName': env.testPhase2StageName, ] @@ -1708,24 +1708,16 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor return buildStatus } -def launchInfraDryRunTestJobs(pipeline, arch, testFilter, globalVars, platform, imageParameters) +def launchInfraDryRunTestJob(pipeline, arch, testFilter, globalVars, platform, imageParameters) { String testFilterJson = writeJSON returnText: true, json: testFilter - def additionalParameters = ['testFilter': testFilterJson] + imageParameters - def testJobs = [ - "[Test-${arch}-Single-GPU] Remote Run": { - stage("[Test-${arch}-Single-GPU] Remote Run") { - launchJob(pipeline, "L0_Test-${arch}-Single-GPU", false, false, globalVars, platform, additionalParameters) - } - }, - "[Test-${arch}-Multi-GPU] Remote Run": { - stage("[Test-${arch}-Multi-GPU] Remote Run") { - launchJob(pipeline, "L0_Test-${arch}-Multi-GPU", false, false, globalVars, platform, additionalParameters) - } - }, - ] - testJobs.failFast = false - pipeline.parallel testJobs + def additionalParameters = [ + 'testFilter': testFilterJson, + 'testPhase2StageName': '', + ] + imageParameters + stage("[Test-${arch}-Single-GPU] Remote Run") { + launchJob(pipeline, "L0_Test-${arch}-Single-GPU", false, false, globalVars, platform, additionalParameters) + } } def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) @@ -1830,7 +1822,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) 'wheelDockerImagePy310': globalVars["LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE"], 'wheelDockerImagePy312': globalVars["LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE"], ] - launchInfraDryRunTestJobs(pipeline, "x86_64", testFilter, globalVars, "x86_64", imageParameters) + launchInfraDryRunTestJob(pipeline, "x86_64", testFilter, globalVars, "x86_64", imageParameters) return } @@ -1990,7 +1982,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) "dockerImage": globalVars["LLM_SBSA_DOCKER_IMAGE"], 'wheelDockerImage': globalVars["LLM_SBSA_WHEEL_DOCKER_IMAGE"], ] - launchInfraDryRunTestJobs(pipeline, "SBSA", testFilter, globalVars, "SBSA", imageParameters) + launchInfraDryRunTestJob(pipeline, "SBSA", testFilter, globalVars, "SBSA", imageParameters) return } diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 8b37dfa98976..a660845651ec 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -165,54 +165,41 @@ def test_parameter_is_propagated_to_the_helper_filter(self): self.assertIn("params.InfraDryRun?.toString()?.toBoolean()", filter_setup) self.assertIn("(INFRA_DRY_RUN): infraDryRun", filter_setup) - def test_dry_run_helpers_share_parameters_and_do_not_fail_fast(self): + def test_dry_run_uses_one_combined_helper_without_inner_parallel(self): body = _function_body( PARENT_GROOVY, - "launchInfraDryRunTestJobs", - "launchStages", - ) - self.assertEqual(body.count("def additionalParameters ="), 1) - self.assertIn('"L0_Test-${arch}-Single-GPU"', body) - self.assertIn('"L0_Test-${arch}-Multi-GPU"', body) - self.assertEqual(body.count("additionalParameters)"), 2) - self.assertEqual(body.count(", false, false, globalVars,"), 2) - self.assertLess( - body.index("testJobs.failFast = false"), - body.index("pipeline.parallel testJobs"), - ) - - def test_helper_failure_propagates_after_parallel_siblings_finish(self): - helper = _function_body( - PARENT_GROOVY, - "launchInfraDryRunTestJobs", + "launchInfraDryRunTestJob", "launchStages", ) launch_job = _function_body( PARENT_GROOVY, "launchJob", - "launchInfraDryRunTestJobs", + "launchInfraDryRunTestJob", ) - self.assertNotIn("catchError", helper) - self.assertNotIn("catch (", helper) - self.assertIn('if (buildStatus != "SUCCESS")', launch_job) - self.assertIn('error "Downstream job did not succeed"', launch_job) - self.assertIn("testJobs.failFast = false", helper) - - def test_shared_filter_and_image_parameters_are_read_only_and_match_normal_jobs(self): - helper = _function_body( - PARENT_GROOVY, - "launchInfraDryRunTestJobs", - "launchStages", + self.assertIn('"L0_Test-${arch}-Single-GPU"', body) + self.assertNotIn('"L0_Test-${arch}-Multi-GPU"', body) + self.assertIn(", false, false, globalVars,", body) + self.assertIn("'testFilter': testFilterJson", body) + self.assertIn("'testPhase2StageName': ''", body) + self.assertNotIn("pipeline.parallel", body) + self.assertIn( + "if (!additionalParameters.containsKey('testPhase2StageName') && " + "env.testPhase2StageName)", + launch_job, ) - launch_job = _function_body( - PARENT_GROOVY, - "launchJob", - "launchInfraDryRunTestJobs", + + selection = GROOVY[GROOVY.index("singleGpuJobs = parallelJobs") :] + phase2_guard = selection.index("if (testPhase2StageName)") + single_start = selection.index("if (env.JOB_NAME ==~ /.*Single-GPU.*/)") + single_end = selection.index("} else if (env.JOB_NAME ==~ /.*Multi-GPU.*/)") + self.assertLess(phase2_guard, selection.index("singleGpuJobs = parallelJobs.findAll")) + self.assertIn("dgxJobs = [:]", selection[:phase2_guard]) + self.assertIn( + "parallel singleGpuJobs", + selection[single_start:single_end], ) - self.assertNotIn("additionalParameters[", helper + launch_job) - self.assertNotIn("additionalParameters.put", helper + launch_job) - self.assertIn("parameters += [", launch_job) + def test_image_parameters_match_normal_jobs(self): start = PARENT_GROOVY.index("def launchStages") launch_stages = PARENT_GROOVY[start : PARENT_GROOVY.index("\npipeline {", start)] expected_keys = { @@ -224,7 +211,7 @@ def test_shared_filter_and_image_parameters_are_read_only_and_match_normal_jobs( "SBSA": {"dockerImage", "wheelDockerImage"}, } for arch, expected in expected_keys.items(): - dry_call = launch_stages.index(f'launchInfraDryRunTestJobs(pipeline, "{arch}"') + dry_call = launch_stages.index(f'launchInfraDryRunTestJob(pipeline, "{arch}"') dry_map = launch_stages.rindex("def imageParameters = [", 0, dry_call) normal_stage = launch_stages.index( f'testStageName = "[Test-{arch}-Single-GPU] Remote Run"', @@ -244,7 +231,7 @@ def test_dry_run_branch_precedes_normal_single_gpu_gating(self): start = PARENT_GROOVY.index("def launchStages") launch_stages = PARENT_GROOVY[start : PARENT_GROOVY.index("\npipeline {", start)] for arch in ("x86_64", "SBSA"): - dry_run_call = launch_stages.index(f'launchInfraDryRunTestJobs(pipeline, "{arch}"') + dry_run_call = launch_stages.index(f'launchInfraDryRunTestJob(pipeline, "{arch}"') build_call = launch_stages.rindex( f'launchJob(pipeline, "/LLM/helpers/Build-{arch}"', 0, @@ -258,9 +245,14 @@ def test_dry_run_branch_precedes_normal_single_gpu_gating(self): f'currentBuild.description?.contains("Require {arch} Multi-GPU Testing")', normal_single, ) + normal_multi = launch_stages.index( + f'launchJob(pipeline, "L0_Test-{arch}-Multi-GPU"', + marker, + ) self.assertLess(build_call, dry_run_call) self.assertLess(dry_run_call, normal_single) self.assertLess(normal_single, marker) + self.assertLess(marker, normal_multi) self.assertIn( "parallelJobs.failFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast", launch_stages, From a1a17d3a72b9b8b701b47407d03cc6b66e74620b Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:11:15 -0700 Subject: [PATCH 08/34] TRTLLMINF-161 Make artifact download idempotent Reused Slurm workspaces may retain TensorRT-LLM.tar.gz. wget then auto-renames the new archive to TensorRT-LLM.tar.gz.1, while extraction continues to read the stale or truncated original and fails with Unexpected EOF. Remove the old target, download through a job/node-specific temporary path, and atomically replace the deterministic archive before extraction. Add regression coverage that starts with a stale archive and verifies the newly downloaded payload is promoted and extracted without creating a .1 file. Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- .../tools/test_infra_dry_run_pipeline.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index a660845651ec..877fc93629f5 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -13,7 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import re +import subprocess +import tempfile import unittest from pathlib import Path @@ -21,6 +24,7 @@ GROOVY = (REPO_ROOT / "jenkins" / "L0_Test.groovy").read_text() PARENT_GROOVY = (REPO_ROOT / "jenkins" / "L0_MergeRequest.groovy").read_text() SLURM_RUN = (REPO_ROOT / "jenkins" / "scripts" / "slurm_run.sh").read_text() +SLURM_INSTALL_PATH = REPO_ROOT / "jenkins" / "scripts" / "slurm_install.sh" def _function_body(source, name, next_name): @@ -114,6 +118,84 @@ def test_slurm_maps_ranks_and_uses_stable_rendezvous(self): SLURM_RUN.index("eval $pytestCommand"), ) + def test_slurm_artifact_download_replaces_existing_archive(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + archive_path = temp_path / "TensorRT-LLM.tar.gz" + wget_record_path = temp_path / "wget-output-path" + tar_record_path = temp_path / "tar-input-path" + archive_path.write_text("stale\n") + + script = r''' +source "$SLURM_INSTALL_PATH" +retry_command() { + if [[ "$1" == "--timeout" ]]; then + shift 2 + fi + "$@" +} +wget() { + local output_path="" + while (( "$#" )); do + if [[ "$1" == "-O" ]]; then + output_path="$2" + shift 2 + else + shift + fi + done + if [[ -z "$output_path" ]]; then + output_path="$resourcePathNode/$tarName" + [[ ! -e "$output_path" ]] || output_path="${output_path}.1" + fi + printf 'fresh\n' > "$output_path" + printf '%s\n' "$output_path" > "$WGET_RECORD_PATH" +} +tar() { + [[ "$1" == "-zxf" ]] + [[ "$2" == "$EXPECTED_ARCHIVE_PATH" ]] + grep -qx fresh "$2" + mkdir -p "$resourcePathNode/TensorRT-LLM/src" + printf '%s\n' "$2" > "$TAR_RECORD_PATH" +} +apt-get() { :; } +nvidia-smi() { :; } +pip3() { :; } +python3() { :; } +export -f pip3 wget +slurm_install_setup +''' + env = { + **os.environ, + "SLURM_INSTALL_PATH": str(SLURM_INSTALL_PATH), + "resourcePathNode": temp_dir, + "tarName": archive_path.name, + "llmTarfile": "https://artifacts.example/TensorRT-LLM.tar.gz", + "SLURM_LOCALID": "0", + "SLURM_JOB_ID": "123", + "SLURM_NODEID": "0", + "pytestCommand": "pytest", + "stageName": "test-stage", + "HOST_NODE_NAME": "test-host", + "EXPECTED_ARCHIVE_PATH": str(archive_path), + "WGET_RECORD_PATH": str(wget_record_path), + "TAR_RECORD_PATH": str(tar_record_path), + } + subprocess.run( + ["bash", "-c", script], + check=True, + capture_output=True, + text=True, + env=env, + ) + + expected_tmp = f"{archive_path}.tmp.123.0" + self.assertEqual(wget_record_path.read_text(), f"{expected_tmp}\n") + self.assertEqual(archive_path.read_text(), "fresh\n") + self.assertEqual(tar_record_path.read_text(), f"{archive_path}\n") + self.assertFalse(Path(f"{archive_path}.1").exists()) + self.assertFalse(Path(expected_tmp).exists()) + def test_direct_branch_follows_existing_shard_setup(self): body = _function_body( GROOVY, From bba20a171735bdaff1566fb7e576fd2173cce48a Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Mon, 10 Aug 2026 04:13:20 -0700 Subject: [PATCH 09/34] ci: run dry-run benchmark through standard pytest flow Register the infrastructure dry-run benchmark in the test DB and reuse the normal render, pytest, JUnit, and upload flow.\n\nKeep InfraDryRun=false or missing behavior unchanged for MR and PostMerge paths. Stabilize Remote MPI and local spawn worker serialization so worker processes can import benchmark callables. Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 251 +++++++---- jenkins/scripts/infra_dry_run_benchmark.py | 385 ---------------- jenkins/scripts/slurm_run.sh | 24 +- .../defs/infra_dry_run_benchmark.py | 342 ++++++++++++++ .../test_lists/test-db/infra_dry_run.yml | 9 + .../tools/test_infra_dry_run_benchmark.py | 350 --------------- .../tools/test_infra_dry_run_pipeline.py | 143 ++++-- .../tools/test_infra_dry_run_pytest.py | 421 ++++++++++++++++++ 8 files changed, 1036 insertions(+), 889 deletions(-) delete mode 100644 jenkins/scripts/infra_dry_run_benchmark.py create mode 100644 tests/integration/defs/infra_dry_run_benchmark.py create mode 100644 tests/integration/test_lists/test-db/infra_dry_run.yml delete mode 100644 tests/unittest/tools/test_infra_dry_run_benchmark.py create mode 100644 tests/unittest/tools/test_infra_dry_run_pytest.py diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 5f05301ccb0c..e3b9da053108 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -102,6 +102,12 @@ def LLVM_CONFIG = "LLVM" @Field def LINUX_AARCH64_CONFIG = "linux_aarch64" +@Field +def INFRA_DRY_RUN_TEST_CONTEXT = "infra_dry_run" + +@Field +def INFRA_DRY_RUN_BENCHMARK = "infra_dry_run_benchmark.py" + @Field def BUILD_CONFIGS = [ // Vanilla TARNAME is used for packaging in runLLMPackage @@ -397,7 +403,6 @@ def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String def hasTimeoutTest = false def downloadResultSucceed = false def downloadPerfResultSucceed = false - def downloadInfraResultSucceed = false pipeline.stage('Submit Test Result') { sh "mkdir -p ${stageName}" @@ -418,14 +423,6 @@ def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String // Download normal test results def resultsFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/results*.xml" downloadResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, resultsFilePath, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 - if (isInfraDryRun()) { - def infraResultPath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/infra_dry_run*.json" - downloadInfraResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, infraResultPath, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 - if (!downloadInfraResultSucceed) { - error "Failed to collect infrastructure dry-run JSON results for ${stageName}" - } - } - // Download perf test results if (!isInfraDryRun()) { def perfResultsBasePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}" @@ -518,11 +515,7 @@ def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String } if ((hasTimeoutTest || downloadResultSucceed) && !suppressTestReporting) { - if (isInfraDryRun()) { - junit(testResults: "${stageName}/results-infra_dry_run*.xml") - } else { - junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml") - } + junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml") } else if (suppressTestReporting) { echo "[INFRA-RETRY] ${stageName}${postTag}: suppressing junit() because a retry is still planned" } @@ -585,7 +578,7 @@ def runIsolatedTests(preprocessedLists, testCmdLine, llmSrc, stageName) { return rerunFailed // Return the updated value } -def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, durationsPath="") { +def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, durationsPath="", positionalTest="") { // Preprocess testDBList to extract ISOLATION markers echo "Preprocessing testDBList to extract ISOLATION markers..." @@ -654,6 +647,9 @@ def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, du "--splits ${splits}", "--group ${splitId}", ] + if (positionalTest) { + testListCmd += [positionalTest] + } if (durationsPath) { testListCmd += ["--durations-path ${durationsPath}"] } @@ -1566,41 +1562,6 @@ def getNodeArgs(int nodeCount, int gpuCount, boolean setSegment = false) { return args } -def getInfraDryRunNodeArgs(int nodeCount, int gpuCount) { - int gpusPerNode = ((gpuCount / nodeCount) as BigDecimal).setScale(0, BigDecimal.ROUND_CEILING).intValue() - return [ - "--nodes=${nodeCount}", - "--ntasks=${gpuCount}", - "--ntasks-per-node=${gpusPerNode}", - "--gpus-per-node=${gpusPerNode}", - ] -} - -def getInfraDryRunDirectCommand(String llmSrc, String outputPath, String stageName, String commit) { - def deviceType = stageName.startsWith("CPU-") ? "cpu" : "cuda" - def benchmarkArgs = [ - "${llmSrc}/jenkins/scripts/infra_dry_run_benchmark.py", - "--output-dir '${outputPath}'", - "--stage '${stageName}'", - "--commit '${commit}'", - "--device '${deviceType}'", - ].join(" ") - return """ - set -eu - mkdir -p '${outputPath}' - if [ '${deviceType}' = 'cpu' ]; then - python3 ${benchmarkArgs} - else - gpu_count=\$(python3 -c 'import torch; print(torch.cuda.device_count())') - if [ "\$gpu_count" -gt 1 ]; then - torchrun --standalone --nproc-per-node="\$gpu_count" ${benchmarkArgs} - else - python3 ${benchmarkArgs} - fi - fi - """ -} - def getPytestBaseCommandLine( String llmSrc, String stageName, @@ -1755,6 +1716,11 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG def jobWorkspace = "/home/svc_tensorrt/bloom/scripts/${jobUID}" def disaggMultiNodeMode = stageName.contains("Disagg-PerfSanity") def aggMultiNodeMode = !disaggMultiNodeMode && nodeCount > 1 && stageName.contains("PerfSanity") + def infraDryRun = isInfraDryRun() + def effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList + def effectiveSplitId = infraDryRun ? 1 : splitId + def effectiveSplits = infraDryRun ? 1 : splits + def effectivePerfMode = infraDryRun ? false : perfMode Utils.exec(pipeline, script: "env | sort && pwd && ls -alh") @@ -1783,8 +1749,11 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG def scriptInstallPathNode = "${jobWorkspace}/${jobUID}-slurm_install.sh" def scriptBashUtilsLocalPath = "${llmSrcLocal}/jenkins/scripts/bash_utils.sh" def scriptBashUtilsPathNode = "${jobWorkspace}/${jobUID}-bash_utils.sh" - def testListPathNode = "${jobWorkspace}/${testList}.txt" + def testListPathNode = "${jobWorkspace}/${effectiveTestList}.txt" def waivesListPathNode = "${jobWorkspace}/waives.txt" + def waivesListPathLocal = infraDryRun + ? "${llmPath}/infra_dry_run_waives.txt" + : "${llmSrcLocal}/tests/integration/test_lists/waives.txt" def slurmJobLogPath = "${jobWorkspace}/job-output.log" def scriptLaunchPathLocal = Utils.createTempLocation(pipeline, "./slurm_launch.sh") def scriptLaunchPathNode = "${jobWorkspace}/${jobUID}-slurm_launch.sh" @@ -1842,7 +1811,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG // if the line cannot be split by "=", just ignore that line. def makoOptsJson = transformMakoArgsToJson(["Mako options:"] + makoArgs) String clusterNameForDurations = useClusterDurations ? partition.clusterName.replaceAll('[^a-zA-Z0-9]', '_') : null - def testListPathLocal = renderTestDB(pipeline, testList, llmSrcLocal, stageName, makoOptsJson, clusterNameForDurations) + def testListPathLocal = renderTestDB(pipeline, effectiveTestList, llmSrcLocal, stageName, makoOptsJson, clusterNameForDurations) // Copy the test list atomically. A retry that reuses a still-active job // re-copies over ${testListPathNode} while that job may be reading it via // --test-list; scp truncates-then-streams, so a concurrent read could see a @@ -1859,18 +1828,22 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG script: Utils.sshUserCmd(remote, "\"mv -f ${testListPathNode}.tmp ${testListPathNode}\"") ) - // Download and Merge waives.txt - mergeWaivesTxt(pipeline, llmSrcLocal, stageName) + if (infraDryRun) { + sh "mkdir -p ${llmPath} && : > ${waivesListPathLocal}" + } else { + // Download and Merge waives.txt + mergeWaivesTxt(pipeline, llmSrcLocal, stageName) - // Add passed test list from previous pipeline run to the waives.txt - if (testFilter[(REUSE_TEST)] != false) { - reusePassedTestResults(llmSrcLocal, stageName, "${llmSrcLocal}/tests/integration/test_lists/waives.txt", postTag) + // Add passed test list from previous pipeline run to the waives.txt + if (testFilter[(REUSE_TEST)] != false) { + reusePassedTestResults(llmSrcLocal, stageName, waivesListPathLocal, postTag) + } } Utils.copyFileToRemoteHost( pipeline, remote, - "${llmSrcLocal}/tests/integration/test_lists/waives.txt", + waivesListPathLocal, waivesListPathNode ) @@ -1955,10 +1928,13 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG def extraArgs = [ "--test-list=$testListPathNode", "--splitting-algorithm least_duration", - "--splits $splits", - "--group $splitId", + "--splits $effectiveSplits", + "--group $effectiveSplitId", *clusterDurationsArgsNode, ] + if (infraDryRun) { + extraArgs += ["${llmSrcNode}/tests/integration/defs/${INFRA_DRY_RUN_BENCHMARK}"] + } if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { extraArgs += [ "--capture=fd", @@ -1970,7 +1946,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG llmSrcNode, stageName, waivesListPathNode, - perfMode, + effectivePerfMode, jobWorkspace, "$jobWorkspace/.coveragerc", pytestUtil, @@ -1984,9 +1960,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG .replace("${ARTIFACTORY_DOCKER_HOST}/", "${ARTIFACTORY_DOCKER_HOST}#") } def mounts = getMountListForSlurmTest(cluster, true).join(",") - String[] taskArgs = isInfraDryRun() - ? getInfraDryRunNodeArgs(nodeCount, gpuCount) - : getNodeArgs(nodeCount, gpuCount, disaggMultiNodeMode) + String[] taskArgs = getNodeArgs(nodeCount, gpuCount, disaggMultiNodeMode) if (taskArgs == null) { error "Invalid Slurm test stage name is set" } @@ -2154,8 +2128,8 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG export llmTarfile=$llmTarfile export llmSrcNode=$llmSrcNode export stageName=$stageName - export perfMode=$perfMode - export infraDryRun=${isInfraDryRun()} + export perfMode=$effectivePerfMode + export infraDryRun=$infraDryRun export resourcePathNode=$resourcePathNode export pytestCommand="$pytestCommand" export coverageConfigFile="$coverageConfigFile" @@ -3778,6 +3752,74 @@ def echoNodeAndGpuInfo(pipeline, stageName) pipeline.echo "HOST_NODE_NAME = ${hostNodeName} ; GPU_UUIDS = ${gpuUuids} ; STAGE_NAME = ${stageName}" } +def runInfraDryRunInPreparedWorkspace(pipeline, String llmSrc, String stageName) +{ + def outputPath = "${WORKSPACE}/${stageName}" + def waivesFile = "${llmSrc}/infra_dry_run_waives.txt" + def coverageConfigFile = "${llmSrc}/infra_dry_run.coveragerc" + def benchmarkPath = "${llmSrc}/tests/integration/defs/${INFRA_DRY_RUN_BENCHMARK}" + + sh "rm -rf ${outputPath} && mkdir -p ${outputPath} && : > ${waivesFile} && : > ${coverageConfigFile}" + def testDBList = renderTestDB( + pipeline, + INFRA_DRY_RUN_TEST_CONTEXT, + llmSrc, + stageName, + ) + def preprocessedLists = processShardTestList( + llmSrc, + testDBList, + 1, + 1, + false, + "", + benchmarkPath, + ) + if (preprocessedLists.regularCount < 1) { + error "No infrastructure dry-run benchmark was selected for ${stageName}" + } + + def extraArgs = [] + if (ENABLE_UPLOAD_TEST_RESULTS) { + def uploadPath = UPLOAD_PATH.replaceFirst("sw-tensorrt-generic/llm-artifacts/LLM/", "") + extraArgs += [ + "-s", + "--s3-upload-path=${uploadPath}/${stageName}", + ] + if (ENABLE_S3_ECHO_STDOUT) { + extraArgs += [ + "--s3-echo-stdout", + "--s3-capture-mode=timestamped", + ] + } + } + def pytestCommand = getPytestBaseCommandLine( + llmSrc, + stageName, + waivesFile, + false, + outputPath, + coverageConfigFile, + "", + extraArgs, + ) + pytestCommand += [ + "--test-list=${preprocessedLists.regular}", + benchmarkPath, + ] + + withCredentials([ + string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), + string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), + string(credentialsId: 'llm_evaltool_repo_url', variable: 'EVALTOOL_REPO_URL') + ]) { + sh """ + cd ${llmSrc}/tests/integration/defs && \ + ${pytestCommand.join(" ")} + """ + } +} + def runLLMDocBuild(pipeline, config, stageName) { // Step 1: cloning source code @@ -3804,13 +3846,7 @@ def runLLMDocBuild(pipeline, config, stageName) trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmPath} && pip3 install --force-reinstall --no-deps TensorRT-LLM/tensorrt_llm-*.whl") if (isInfraDryRun()) { - def commit = env.artifactCommit ?: env.gitlabCommit ?: "" - sh getInfraDryRunDirectCommand( - llmSrc, - "${WORKSPACE}/${stageName}", - stageName, - commit, - ) + runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName) return } @@ -4903,6 +4939,14 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def noRegularTests = false def noIsolateTests = false def rerunFailed = false + def infraDryRun = isInfraDryRun() + def effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList + def effectiveSplitId = infraDryRun ? 1 : splitId + def effectiveSplits = infraDryRun ? 1 : splits + def effectivePerfMode = infraDryRun ? false : perfMode + def benchmarkPath = infraDryRun + ? "${llmSrc}/tests/integration/defs/${INFRA_DRY_RUN_BENCHMARK}" + : "" // When useClusterDurations is set, use a per-cluster durations file keyed on // partition.clusterName (e.g. "oci-hsg", "dlcluster"). This lets each cluster @@ -4919,18 +4963,33 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO clusterDurationsArgs = ["--durations-path ${clusterDurationsPath}"] } - def testDBList = renderTestDB(pipeline, testList, llmSrc, stageName, null, clusterNameForDurations) + def testDBList = renderTestDB(pipeline, effectiveTestList, llmSrc, stageName, null, clusterNameForDurations) + def waivesFilePath = infraDryRun + ? "${llmSrc}/infra_dry_run_waives.txt" + : "${llmSrc}/tests/integration/test_lists/waives.txt" - // Download and Merge waives.txt - mergeWaivesTxt(pipeline, llmSrc, stageName) + if (infraDryRun) { + sh ": > ${waivesFilePath}" + } else { + // Download and Merge waives.txt + mergeWaivesTxt(pipeline, llmSrc, stageName) - // Add passed test list from previous pipeline run to the waives.txt - if (testFilter[(REUSE_TEST)] != false) { - reusePassedTestResults(llmSrc, stageName, "${llmSrc}/tests/integration/test_lists/waives.txt", postTag) + // Add passed test list from previous pipeline run to the waives.txt + if (testFilter[(REUSE_TEST)] != false) { + reusePassedTestResults(llmSrc, stageName, waivesFilePath, postTag) + } } // Process shard test list and create separate files for regular and isolate tests - def preprocessedLists = processShardTestList(llmSrc, testDBList, splitId, splits, perfMode, clusterDurationsPath) + def preprocessedLists = processShardTestList( + llmSrc, + testDBList, + effectiveSplitId, + effectiveSplits, + effectivePerfMode, + clusterDurationsPath, + benchmarkPath, + ) // Test Coverage def TRTLLM_WHL_PATH = sh(returnStdout: true, script: "pip3 show tensorrt_llm | grep Location | cut -d ' ' -f 2").replaceAll("\\s","") @@ -4964,6 +5023,9 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO // Temporarily disable to reduce the log size // sh 'if [ "$(id -u)" -eq 0 ]; then dmesg -C || true; fi' def extraArgs = [*clusterDurationsArgs] + if (infraDryRun) { + extraArgs += [benchmarkPath] + } if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { extraArgs += [ "--capture=fd", @@ -4974,8 +5036,8 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def pytestCommand = getPytestBaseCommandLine( llmSrc, stageName, - "${llmSrc}/tests/integration/test_lists/waives.txt", - perfMode, + waivesFilePath, + effectivePerfMode, "${WORKSPACE}/${stageName}", coverageConfigFile, "", // pytestUtil @@ -4997,22 +5059,23 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } containerLD_LIBRARY_PATH = containerLD_LIBRARY_PATH.replaceAll(':+$', '') withEnv(["LD_LIBRARY_PATH=${containerLD_LIBRARY_PATH}"]) { - if (isInfraDryRun()) { - def commit = env.artifactCommit ?: env.gitlabCommit ?: "" - sh getInfraDryRunDirectCommand( - llmSrc, - "${WORKSPACE}/${stageName}", - stageName, - commit, - ) - return - } withCredentials([ string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), string(credentialsId: 'llm_evaltool_repo_url', variable: 'EVALTOOL_REPO_URL') ]) { sh "env | sort" + if (infraDryRun) { + if (preprocessedLists.regularCount < 1) { + error "No infrastructure dry-run benchmark was selected for ${stageName}" + } + sh """ + rm -rf ${stageName}/ && \ + cd ${llmSrc}/tests/integration/defs && \ + ${pytestCommand.join(" ")} + """ + return + } try { try { if (preprocessedLists.regularCount > 0) { @@ -5168,10 +5231,6 @@ def runLLMTestlistOnPlatform(pipeline, platform, testList, config=VANILLA_CONFIG error("Error in post-debug session: ${e.message}") } } - if (isInfraDryRun()) { - sh "ls -al ${stageName}/" - return - } // If the execution test list is null, remove the test result xml sh """ ls -al ${stageName}/ diff --git a/jenkins/scripts/infra_dry_run_benchmark.py b/jenkins/scripts/infra_dry_run_benchmark.py deleted file mode 100644 index d18462a2a8e7..000000000000 --- a/jenkins/scripts/infra_dry_run_benchmark.py +++ /dev/null @@ -1,385 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Run a small CPU or CUDA/NCCL smoke test and emit CI-friendly result artifacts.""" - -import argparse -import json -import math -import os -import sys -import xml.etree.ElementTree as ET -from datetime import timedelta -from pathlib import Path -from typing import Any, Mapping - -NAME = "infra_dry_run" -MATRIX_SIZE = 64 -JUNIT_FILE = "results-infra_dry_run.xml" -MANIFEST_FILE = "infra_dry_run_manifest.json" - - -class BenchmarkError(RuntimeError): - """Raised after failure artifacts have been written.""" - - -def _load_runtime_modules() -> tuple[Any, Any]: - # Normal imports intentionally exercise the installed package and PyTorch. - import torch - - import tensorrt_llm - - return tensorrt_llm, torch - - -def _rank_context(environ: Mapping[str, str]) -> dict[str, int]: - try: - rank = int(environ.get("RANK", "0")) - local_rank = int(environ.get("LOCAL_RANK", "0")) - world_size = int(environ.get("WORLD_SIZE", "1")) - except ValueError as error: - raise BenchmarkError("RANK, LOCAL_RANK, and WORLD_SIZE must be integers") from error - if world_size < 1 or rank < 0 or rank >= world_size or local_rank < 0: - raise BenchmarkError( - f"invalid rank context: rank={rank}, local_rank={local_rank}, world_size={world_size}" - ) - return {"rank": rank, "local_rank": local_rank, "world_size": world_size} - - -def _new_result( - context: Mapping[str, int], - stage: str | None, - commit: str | None, - environ: Mapping[str, str], - device_type: str, -) -> dict[str, Any]: - return { - "name": NAME, - "status": "failed", - "product_tests_executed": 0, - **context, - "device_type": device_type, - "distributed_backend": ( - "nccl" if device_type == "cuda" and context["world_size"] > 1 else "none" - ), - "stage": stage or environ.get("STAGE_NAME") or environ.get("stageName") or "", - "commit": commit or environ.get("GIT_COMMIT") or environ.get("gitlabCommit") or "", - "tensorrt_llm_version": "unknown", - "tensorrt_llm_module": "unknown", - } - - -def _select_cuda_device(torch: Any, local_rank: int) -> str: - if not torch.cuda.is_available(): - raise BenchmarkError("CUDA is required") - device_count = int(torch.cuda.device_count()) - if local_rank >= device_count: - raise BenchmarkError( - f"LOCAL_RANK {local_rank} cannot select from {device_count} visible CUDA device(s)" - ) - torch.cuda.set_device(local_rank) - return f"cuda:{local_rank}" - - -def _cuda_matmul(torch: Any, local_rank: int, device: str) -> dict[str, Any]: - torch.manual_seed(0) - torch.cuda.manual_seed_all(0) - left = torch.full((MATRIX_SIZE, MATRIX_SIZE), 0.5, dtype=torch.float16, device=device) - right = torch.full((MATRIX_SIZE, MATRIX_SIZE), 0.25, dtype=torch.float16, device=device) - output = torch.matmul(left, right) - torch.cuda.synchronize(local_rank) - if int(output.numel()) == 0: - raise BenchmarkError("CUDA matrix multiplication returned an empty tensor") - if not bool(torch.isfinite(output).all().item()): - raise BenchmarkError("CUDA matrix multiplication returned non-finite values") - checksum = float(output.float().sum().item()) - if not math.isfinite(checksum): - raise BenchmarkError("CUDA matrix multiplication checksum is non-finite") - return { - "device": device, - "matrix_size": MATRIX_SIZE, - "dtype": "float16", - "checksum": checksum, - } - - -def _cpu_matmul(torch: Any) -> dict[str, Any]: - torch.manual_seed(0) - left = torch.full((MATRIX_SIZE, MATRIX_SIZE), 0.5, dtype=torch.float32, device="cpu") - right = torch.full((MATRIX_SIZE, MATRIX_SIZE), 0.25, dtype=torch.float32, device="cpu") - output = torch.matmul(left, right) - if int(output.numel()) == 0: - raise BenchmarkError("CPU matrix multiplication returned an empty tensor") - if not bool(torch.isfinite(output).all().item()): - raise BenchmarkError("CPU matrix multiplication returned non-finite values") - checksum = float(output.float().sum().item()) - if not math.isfinite(checksum): - raise BenchmarkError("CPU matrix multiplication checksum is non-finite") - return { - "device": "cpu", - "matrix_size": MATRIX_SIZE, - "dtype": "float32", - "checksum": checksum, - } - - -def _initialize_distributed(torch: Any, context: Mapping[str, int], timeout_seconds: int) -> bool: - if context["world_size"] == 1: - return False - if not torch.distributed.is_available() or not torch.distributed.is_nccl_available(): - raise BenchmarkError("torch.distributed with NCCL is required for WORLD_SIZE > 1") - torch.distributed.init_process_group( - backend="nccl", - init_method="env://", - rank=context["rank"], - world_size=context["world_size"], - timeout=timedelta(seconds=timeout_seconds), - ) - return True - - -def _summary(result: Mapping[str, Any]) -> dict[str, Any]: - device_type = str(result.get("device_type", "cuda")) - return { - "rank": int(result["rank"]), - "world_size": int(result["world_size"]), - "status": str(result["status"]), - "checksum": result.get(device_type, {}).get("checksum"), - "error": str(result.get("error", "")), - } - - -def _gather_summaries(torch: Any, result: Mapping[str, Any]) -> list[dict[str, Any]]: - world_size = int(result["world_size"]) - local = torch.tensor( - [ - float(result["rank"]), - float(world_size), - float(result["status"] == "passed"), - float(result.get("cuda", {}).get("checksum", 0.0)), - ], - dtype=torch.float64, - device=f"cuda:{result['local_rank']}", - ) - gathered = [torch.empty_like(local) for _ in range(world_size)] - torch.distributed.all_gather(gathered, local) - return [ - { - "rank": int(values[0]), - "world_size": int(values[1]), - "status": "passed" if bool(values[2]) else "failed", - "checksum": float(values[3]), - "error": "" if bool(values[2]) else "CUDA work failed on this rank", - } - for values in (item.cpu().tolist() for item in gathered) - ] - - -def _validate(summaries: list[Mapping[str, Any]], world_size: int) -> list[str]: - errors: list[str] = [] - ranks = [int(item["rank"]) for item in summaries] - expected_ranks = list(range(world_size)) - if len(summaries) != world_size or sorted(ranks) != expected_ranks: - errors.append(f"observed ranks {sorted(ranks)} do not match expected {expected_ranks}") - if any(int(item["world_size"]) != world_size for item in summaries): - errors.append("rank results contain a world-size mismatch") - failed = [int(item["rank"]) for item in summaries if item["status"] != "passed"] - if failed: - errors.append(f"rank(s) {sorted(failed)} reported failure") - try: - checksums = [float(item["checksum"]) for item in summaries if item["status"] == "passed"] - except (TypeError, ValueError): - errors.append("passed rank result is missing a valid benchmark checksum") - else: - if any(not math.isfinite(value) for value in checksums): - errors.append("rank results contain a non-finite CUDA checksum") - if checksums and any( - not math.isclose(value, checksums[0], rel_tol=1e-6, abs_tol=1e-6) - for value in checksums[1:] - ): - errors.append("rank results contain inconsistent CUDA checksums") - return errors - - -def _write_json(path: Path, value: Mapping[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") - with temporary.open("w", encoding="utf-8") as output: - json.dump(value, output, indent=2, sort_keys=True) - output.write("\n") - os.replace(temporary, path) - - -def _write_reports( - output_dir: Path, - result: Mapping[str, Any], - summaries: list[Mapping[str, Any]], - errors: list[str], -) -> dict[str, Any]: - world_size = int(result["world_size"]) - manifest: dict[str, Any] = { - "name": NAME, - "status": "failed" if errors else "passed", - "product_tests_executed": 0, - "stage": result["stage"], - "commit": result["commit"], - "world_size": world_size, - "device_type": result.get("device_type", "cuda"), - "distributed_backend": result.get("distributed_backend", "none"), - "observed_ranks": sorted(int(item["rank"]) for item in summaries), - "validation_errors": errors, - "rank_results": summaries, - "junit_file": JUNIT_FILE, - } - _write_json(output_dir / MANIFEST_FILE, manifest) - - by_rank = {int(item["rank"]): item for item in summaries} - cases: list[tuple[str, str | None]] = [] - device_type = str(result.get("device_type", "cuda")) - for rank in range(world_size): - item = by_rank.get(rank) - failure = None - if item is None: - failure = f"missing result for rank {rank}" - elif item["status"] != "passed": - failure = str(item.get("error") or f"rank {rank} failed") - cases.append((f"{NAME}_rank_{rank}_{device_type}_matmul", failure)) - if world_size > 1: - cases.append((f"{NAME}_nccl_collective", "; ".join(errors) or None)) - - suite = ET.Element( - "testsuite", - { - "name": NAME, - "tests": str(len(cases)), - "failures": str(sum(failure is not None for _, failure in cases)), - "errors": "0", - "skipped": "0", - "time": "0", - }, - ) - properties = ET.SubElement(suite, "properties") - for name in ( - "product_tests_executed", - "stage", - "commit", - "world_size", - "device_type", - "distributed_backend", - ): - ET.SubElement(properties, "property", {"name": name, "value": str(manifest[name])}) - for name, failure in cases: - case = ET.SubElement( - suite, - "testcase", - {"classname": NAME, "name": name, "time": "0"}, - ) - if failure: - node = ET.SubElement(case, "failure", {"message": failure}) - node.text = failure - - root = ET.Element("testsuites") - root.append(suite) - tree = ET.ElementTree(root) - ET.indent(tree, space=" ") - tree.write(output_dir / JUNIT_FILE, encoding="utf-8", xml_declaration=True) - return manifest - - -def run_benchmark( - output_dir: Path, - *, - stage: str | None = None, - commit: str | None = None, - timeout_seconds: int = 120, - device_type: str = "cuda", - environ: Mapping[str, str] | None = None, -) -> dict[str, Any]: - environment = os.environ if environ is None else environ - context = _rank_context(environment) - result = _new_result(context, stage, commit, environment, device_type) - summaries = [_summary(result)] - torch = None - distributed = False - - try: - trtllm, torch = _load_runtime_modules() - result["tensorrt_llm_version"] = str(getattr(trtllm, "__version__", "unknown")) - result["tensorrt_llm_module"] = str(getattr(trtllm, "__file__", "unknown")) - if device_type == "cpu": - if context["world_size"] != 1: - raise BenchmarkError("CPU mode requires WORLD_SIZE=1") - result["cpu"] = _cpu_matmul(torch) - result["status"] = "passed" - else: - device = _select_cuda_device(torch, context["local_rank"]) - distributed = _initialize_distributed(torch, context, timeout_seconds) - try: - result["cuda"] = _cuda_matmul(torch, context["local_rank"], device) - result["status"] = "passed" - except Exception as error: - result["error"] = f"{type(error).__name__}: {error}" - summaries = _gather_summaries(torch, result) if distributed else [_summary(result)] - except Exception as error: - result["status"] = "failed" - result["error"] = f"{type(error).__name__}: {error}" - summaries = [_summary(result)] - finally: - if distributed and torch.distributed.is_initialized(): - try: - torch.distributed.destroy_process_group() - except Exception as error: - result["status"] = "failed" - result["error"] = f"distributed cleanup failed: {error}" - summaries = [item for item in summaries if int(item["rank"]) != context["rank"]] + [ - _summary(result) - ] - - errors = _validate(summaries, context["world_size"]) - result["overall_status"] = "failed" if errors else "passed" - _write_json(output_dir / f"{NAME}_rank_{context['rank']}.json", result) - if context["rank"] == 0: - _write_reports(output_dir, result, summaries, errors) - if errors: - raise BenchmarkError("; ".join(errors)) - return result - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output-dir", type=Path, default=Path(".")) - parser.add_argument("--stage") - parser.add_argument("--commit") - parser.add_argument("--device", choices=("cpu", "cuda"), default="cuda") - parser.add_argument("--distributed-timeout-seconds", type=int, default=120) - args = parser.parse_args(sys.argv[1:] if argv is None else argv) - try: - result = run_benchmark( - args.output_dir, - stage=args.stage, - commit=args.commit, - timeout_seconds=args.distributed_timeout_seconds, - device_type=args.device, - ) - except Exception as error: - print(f"{NAME} failed: {error}", file=sys.stderr) - return 1 - print(f"{NAME} passed on rank {result['rank']}/{result['world_size']}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index 6277129e9a16..f858a74edef5 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -71,18 +71,18 @@ env | sort echo "Full Command: $pytestCommand" if [[ "${infraDryRun:-false}" == "true" ]]; then - export RANK="$SLURM_PROCID" - export LOCAL_RANK="$SLURM_LOCALID" - export WORLD_SIZE="$SLURM_NTASKS" - export MASTER_ADDR="${MASTER_ADDR:?MASTER_ADDR must be set by the Slurm launch script}" - export MASTER_PORT="${MASTER_PORT:?MASTER_PORT must be set by the Slurm launch script}" - - python3 "$llmSrcNode/jenkins/scripts/infra_dry_run_benchmark.py" \ - --output-dir "$jobWorkspace" \ - --stage "$stageName" \ - --commit "${gitlabCommit:-}" \ - --distributed-timeout-seconds 900 - exit $? + if [[ "${SLURM_JOB_NUM_NODES:-1}" -gt 1 ]]; then + export RANK="$SLURM_PROCID" + export LOCAL_RANK="$SLURM_LOCALID" + export WORLD_SIZE="$SLURM_NTASKS" + export MASTER_ADDR="${MASTER_ADDR:?MASTER_ADDR must be set by the Slurm launch script}" + export MASTER_PORT="${MASTER_PORT:?MASTER_PORT must be set by the Slurm launch script}" + else + # A single-node dry run is one pytest controller which spawns local + # workers. Do not let ambient launcher variables select the external + # multi-node rank path. + unset RANK LOCAL_RANK WORLD_SIZE + fi fi # For single-node test runs or disaggregated benchmark/server runs, clear all diff --git a/tests/integration/defs/infra_dry_run_benchmark.py b/tests/integration/defs/infra_dry_run_benchmark.py new file mode 100644 index 000000000000..05e38b8e36c9 --- /dev/null +++ b/tests/integration/defs/infra_dry_run_benchmark.py @@ -0,0 +1,342 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Small, model-free infrastructure benchmark used only by CI dry runs. + +The filename intentionally does not match pytest's normal ``test_*.py`` +pattern. Dry-run jobs pass this module explicitly after selecting its node ID +through the dedicated ``infra_dry_run`` test-db context. +""" + +from __future__ import annotations + +import importlib.util +import os +import signal +import socket +import sys +import threading +from contextlib import contextmanager +from datetime import timedelta +from pathlib import Path +from typing import Mapping, Optional + +import torch + +_DISTRIBUTED_TIMEOUT_SECONDS = 900 +_MATRIX_SIZE = 32 + + +@contextmanager +def _bounded_wait(seconds: int): + if threading.current_thread() is not threading.main_thread(): + raise RuntimeError("distributed dry-run timeout requires the main thread") + + previous_timer = signal.getitimer(signal.ITIMER_REAL) + if previous_timer != (0.0, 0.0): + raise RuntimeError("distributed dry-run timeout cannot replace an active ITIMER_REAL") + + def raise_timeout(_signum, _frame): + raise TimeoutError(f"distributed dry-run operation exceeded {seconds} seconds") + + previous_handler = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, raise_timeout) + try: + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + finally: + signal.signal(signal.SIGALRM, previous_handler) + + +def _required_int(environ: Mapping[str, str], name: str) -> int: + value = environ.get(name) + if value is None: + raise RuntimeError(f"{name} must be set for an externally launched rank") + try: + return int(value) + except ValueError as error: + raise RuntimeError(f"{name} must be an integer, got {value!r}") from error + + +def _external_rank_context( + environ: Mapping[str, str], +) -> Optional[tuple[int, int, int]]: + rank_names = ("RANK", "LOCAL_RANK", "WORLD_SIZE") + present = [name in environ for name in rank_names] + if not any(present): + return None + if not all(present): + missing = [name for name, is_present in zip(rank_names, present) if not is_present] + raise RuntimeError(f"incomplete distributed rank environment; missing {missing}") + + rank = _required_int(environ, "RANK") + local_rank = _required_int(environ, "LOCAL_RANK") + world_size = _required_int(environ, "WORLD_SIZE") + if world_size < 1: + raise RuntimeError(f"WORLD_SIZE must be positive, got {world_size}") + if not 0 <= rank < world_size: + raise RuntimeError(f"RANK {rank} is outside WORLD_SIZE {world_size}") + if local_rank < 0: + raise RuntimeError(f"LOCAL_RANK must be non-negative, got {local_rank}") + return rank, local_rank, world_size + + +def _run_cpu(torch_module=torch) -> None: + torch_module.manual_seed(0) + left = torch_module.full( + (_MATRIX_SIZE, _MATRIX_SIZE), 0.25, dtype=torch_module.float32, device="cpu" + ) + right = torch_module.full( + (_MATRIX_SIZE, _MATRIX_SIZE), 0.5, dtype=torch_module.float32, device="cpu" + ) + output = torch_module.matmul(left, right) + expected = torch_module.full_like(output, _MATRIX_SIZE * 0.25 * 0.5) + if output.device.type != "cpu" or output.dtype != torch_module.float32: + raise RuntimeError("CPU benchmark did not produce a CPU FP32 tensor") + if not torch_module.isfinite(output).all().item(): + raise RuntimeError("CPU benchmark produced non-finite values") + if not torch_module.equal(output, expected): + raise RuntimeError("CPU benchmark produced an unexpected deterministic result") + + +def _run_cuda_matmul(local_rank: int, torch_module=torch) -> float: + if not torch_module.cuda.is_available(): + raise RuntimeError("CUDA is required for this infrastructure dry-run stage") + device_count = torch_module.cuda.device_count() + if not 0 <= local_rank < device_count: + raise RuntimeError( + f"LOCAL_RANK {local_rank} is outside the {device_count} visible CUDA devices" + ) + + torch_module.cuda.set_device(local_rank) + device = torch_module.device("cuda", local_rank) + torch_module.manual_seed(1000 + local_rank) + torch_module.cuda.manual_seed_all(1000 + local_rank) + left = torch_module.full( + (_MATRIX_SIZE, _MATRIX_SIZE), 0.25, dtype=torch_module.float16, device=device + ) + right = torch_module.full( + (_MATRIX_SIZE, _MATRIX_SIZE), 0.5, dtype=torch_module.float16, device=device + ) + output = torch_module.matmul(left, right) + expected = torch_module.full_like(output, _MATRIX_SIZE * 0.25 * 0.5) + if output.device.type != "cuda" or output.dtype != torch_module.float16: + raise RuntimeError("GPU benchmark did not produce a CUDA FP16 tensor") + if not torch_module.isfinite(output).all().item(): + raise RuntimeError("GPU benchmark produced non-finite values") + if not torch_module.equal(output, expected): + raise RuntimeError("GPU benchmark produced an unexpected deterministic result") + torch_module.cuda.synchronize(device) + return float(output.float().sum().item()) + + +def _validate_rank_summaries(summaries: list[list[float]], world_size: int) -> None: + expected_ranks = list(range(world_size)) + observed_ranks = sorted(int(summary[0]) for summary in summaries) + if observed_ranks != expected_ranks: + raise RuntimeError( + f"observed ranks {observed_ranks} do not match expected {expected_ranks}" + ) + if any(int(summary[1]) != world_size for summary in summaries): + raise RuntimeError("rank summaries contain inconsistent world sizes") + checksums = [summary[2] for summary in summaries] + if any(abs(checksum - checksums[0]) > 1e-3 for checksum in checksums[1:]): + raise RuntimeError("rank summaries contain inconsistent CUDA checksums") + + +def _run_distributed_rank( + rank: int, + local_rank: int, + world_size: int, + timeout_seconds: int = _DISTRIBUTED_TIMEOUT_SECONDS, + torch_module=torch, +) -> list[float]: + distributed = torch_module.distributed + if not distributed.is_available() or not distributed.is_nccl_available(): + raise RuntimeError("NCCL distributed support is required for multi-GPU dry runs") + + try: + if not distributed.is_initialized(): + distributed.init_process_group( + backend="nccl", + init_method="env://", + rank=rank, + world_size=world_size, + timeout=timedelta(seconds=timeout_seconds), + ) + + checksum = _run_cuda_matmul(local_rank, torch_module) + device = torch_module.device("cuda", local_rank) + local_summary = torch_module.tensor( + [float(rank), float(world_size), checksum], + dtype=torch_module.float64, + device=device, + ) + reduced_checksum = torch_module.tensor(checksum, dtype=torch_module.float64, device=device) + distributed.all_reduce(reduced_checksum) + expected_total = checksum * world_size + if abs(float(reduced_checksum.item()) - expected_total) > 1e-3: + raise RuntimeError("NCCL all-reduce produced an unexpected checksum") + + gathered = [torch_module.empty_like(local_summary) for _ in range(world_size)] + distributed.all_gather(gathered, local_summary) + summaries = [summary.cpu().tolist() for summary in gathered] + _validate_rank_summaries(summaries, world_size) + return [float(rank), float(world_size), checksum] + finally: + if distributed.is_initialized(): + distributed.destroy_process_group() + + +def _llmapi_rank_task(timeout_seconds: int) -> list[float]: + """Run on every rank already owned by ``trtllm-llmapi-launch``.""" + from mpi4py import MPI + + rank_context = _external_rank_context(os.environ) + if rank_context is None: + raise RuntimeError("LLMAPI worker is missing its distributed rank environment") + env_rank, local_rank, env_world_size = rank_context + rank = MPI.COMM_WORLD.Get_rank() + world_size = MPI.COMM_WORLD.Get_size() + if (env_rank, env_world_size) != (rank, world_size): + raise RuntimeError( + "LLMAPI worker rank environment does not match its MPI communicator: " + f"env=({env_rank}, {env_world_size}), mpi=({rank}, {world_size})" + ) + return _run_distributed_rank(rank, local_rank, world_size, timeout_seconds) + + +def _worker_import_module(): + """Load this file under the top-level name visible from the worker cwd. + + Pytest may collect this file as ``defs.infra_dry_run_benchmark``, while the + MGMN workers run from this file's directory and can import it only as + ``infra_dry_run_benchmark``. Both RemoteMpiCommSession and multiprocessing + serialize callables, so worker functions must come from that deterministic + top-level module. + """ + module_name = Path(__file__).stem + module = sys.modules.get(module_name) + if module is None: + spec = importlib.util.spec_from_file_location(module_name, __file__) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load worker task module from {__file__}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except BaseException: + sys.modules.pop(module_name, None) + raise + else: + module_file = getattr(module, "__file__", None) + if module_file is None or Path(module_file).resolve() != Path(__file__).resolve(): + raise RuntimeError(f"{module_name} resolves to {module_file}, expected {__file__}") + return module + + +def _pickleable_llmapi_rank_task(): + return _worker_import_module()._llmapi_rank_task + + +def _run_with_existing_llmapi_launcher( + world_size: int, + timeout_seconds: int = _DISTRIBUTED_TIMEOUT_SECONDS, + session_factory=None, +) -> None: + if session_factory is None: + from tensorrt_llm.executor.utils import create_mpi_comm_session + + session_factory = create_mpi_comm_session + + session = session_factory(world_size) + try: + with _bounded_wait(timeout_seconds + 60): + summaries = session.submit_sync(_pickleable_llmapi_rank_task(), timeout_seconds) + if isinstance(summaries, BaseException): + raise RuntimeError("LLMAPI rank task failed") from summaries + if not isinstance(summaries, list) or len(summaries) != world_size: + raise RuntimeError( + "LLMAPI launcher returned an incomplete rank result set: " + f"expected {world_size}, got {summaries!r}" + ) + _validate_rank_summaries(summaries, world_size) + finally: + # RemoteMpiCommSessionClient.shutdown() is intentionally a no-op. The + # outer launcher owns and stops the MGMN server after pytest exits. + session.shutdown() + + +def _reserve_local_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _local_rank_worker( + local_rank: int, world_size: int, master_port: int, timeout_seconds: int +) -> None: + os.environ.update( + { + "MASTER_ADDR": "127.0.0.1", + "MASTER_PORT": str(master_port), + "RANK": str(local_rank), + "LOCAL_RANK": str(local_rank), + "WORLD_SIZE": str(world_size), + } + ) + _run_distributed_rank(local_rank, local_rank, world_size, timeout_seconds) + + +def test_infra_dry_run_benchmark() -> None: + """Exercise CPU or every assigned GPU without downloading external data.""" + stage_name = os.environ.get("stageName", "") + if stage_name.startswith("CPU-"): + _run_cpu() + return + + rank_context = _external_rank_context(os.environ) + if rank_context is not None: + rank, local_rank, world_size = rank_context + if world_size == 1: + _run_cuda_matmul(local_rank) + elif os.environ.get("TLLM_SPAWN_PROXY_PROCESS") == "1": + if rank != 0: + raise RuntimeError( + "only LLMAPI rank 0 may run the infrastructure pytest controller" + ) + _run_with_existing_llmapi_launcher(world_size) + else: + _run_distributed_rank(rank, local_rank, world_size) + return + + if not torch.cuda.is_available(): + raise RuntimeError("CUDA is required for this infrastructure dry-run stage") + device_count = torch.cuda.device_count() + if device_count < 1: + raise RuntimeError("no CUDA devices are visible to the infrastructure dry run") + if device_count == 1: + _run_cuda_matmul(0) + return + + torch.multiprocessing.spawn( + _worker_import_module()._local_rank_worker, + args=(device_count, _reserve_local_port(), _DISTRIBUTED_TIMEOUT_SECONDS), + nprocs=device_count, + join=True, + ) diff --git a/tests/integration/test_lists/test-db/infra_dry_run.yml b/tests/integration/test_lists/test-db/infra_dry_run.yml new file mode 100644 index 000000000000..f51d5a907fb0 --- /dev/null +++ b/tests/integration/test_lists/test-db/infra_dry_run.yml @@ -0,0 +1,9 @@ +version: 0.0.1 +infra_dry_run: +- condition: + ranges: + system_gpu_count: + gte: 0 + lte: 1024 + tests: + - infra_dry_run_benchmark.py::test_infra_dry_run_benchmark diff --git a/tests/unittest/tools/test_infra_dry_run_benchmark.py b/tests/unittest/tools/test_infra_dry_run_benchmark.py deleted file mode 100644 index 80a3e857b11f..000000000000 --- a/tests/unittest/tools/test_infra_dry_run_benchmark.py +++ /dev/null @@ -1,350 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import builtins -import importlib.util -import json -import tempfile -import unittest -import xml.etree.ElementTree as ET -from pathlib import Path -from types import SimpleNamespace -from unittest import mock - -REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent -SCRIPT_PATH = REPO_ROOT / "jenkins" / "scripts" / "infra_dry_run_benchmark.py" -SPEC = importlib.util.spec_from_file_location("infra_dry_run_benchmark", SCRIPT_PATH) -BENCHMARK = importlib.util.module_from_spec(SPEC) -assert SPEC.loader is not None -SPEC.loader.exec_module(BENCHMARK) - - -class _Scalar: - def __init__(self, value): - self.value = value - - def item(self): - return self.value - - -class _Output: - def __init__(self, *, finite=True, numel=16): - self.finite = finite - self._numel = numel - - def numel(self): - return self._numel - - def all(self): - return _Scalar(self.finite) - - def float(self): - return self - - def sum(self): - return _Scalar(128.0) - - -class _Cuda: - def __init__(self, available=True): - self.available = available - self.selected = None - self.synchronized = None - - def is_available(self): - return self.available - - def device_count(self): - return 1 - - def set_device(self, device): - self.selected = device - - def manual_seed_all(self, _seed): - return None - - def synchronize(self, device): - self.synchronized = device - - -class _Probe: - def __init__(self, values): - self.values = values - - def cpu(self): - return self - - def tolist(self): - return self.values - - -class _Distributed: - def __init__(self, remote): - self.remote = remote - self.initialized = False - self.destroyed = False - self.gather_calls = 0 - - def is_available(self): - return True - - def is_nccl_available(self): - return True - - def init_process_group(self, **kwargs): - assert kwargs["backend"] == "nccl" - assert kwargs["init_method"] == "env://" - self.initialized = True - - def is_initialized(self): - return self.initialized - - def all_gather(self, gathered, local): - self.gather_calls += 1 - gathered[0].values = local.values - gathered[1].values = [ - float(self.remote["rank"]), - float(self.remote["world_size"]), - float(self.remote["status"] == "passed"), - float(self.remote["checksum"]), - ] - - def destroy_process_group(self): - self.destroyed = True - self.initialized = False - - -class _Torch: - float16 = "float16" - float32 = "float32" - float64 = "float64" - - def __init__(self, *, cuda=True, finite=True, numel=16, remote=None): - self.cuda = _Cuda(cuda) - self.output = _Output(finite=finite, numel=numel) - self.distributed = _Distributed(remote) if remote else SimpleNamespace() - - def manual_seed(self, _seed): - return None - - def full(self, *_args, **_kwargs): - return object() - - def matmul(self, _left, _right): - return self.output - - def isfinite(self, output): - return output - - def tensor(self, values, **_kwargs): - return _Probe(values) - - def empty_like(self, probe): - return _Probe([0.0] * len(probe.values)) - - -TRTLLM = SimpleNamespace(__version__="1.2.3", __file__="/installed/tensorrt_llm/__init__.py") - - -def _run(output_dir, torch, **kwargs): - with mock.patch.object(BENCHMARK, "_load_runtime_modules", return_value=(TRTLLM, torch)): - return BENCHMARK.run_benchmark(output_dir, **kwargs) - - -def _read_outputs(output_dir): - rank = json.loads((output_dir / "infra_dry_run_rank_0.json").read_text()) - manifest = json.loads((output_dir / "infra_dry_run_manifest.json").read_text()) - junit = ET.parse(output_dir / "results-infra_dry_run.xml") - return rank, manifest, junit - - -def _result(rank, *, world_size=2, status="passed", checksum=128.0): - return { - "rank": rank, - "world_size": world_size, - "status": status, - "checksum": checksum, - "error": "" if status == "passed" else "CUDA work failed", - } - - -class InfraDryRunBenchmarkTest(unittest.TestCase): - def test_runtime_loader_performs_real_package_imports(self): - imported = [] - real_import = builtins.__import__ - modules = {"tensorrt_llm": SimpleNamespace(), "torch": SimpleNamespace()} - - def record_import(name, *args, **kwargs): - if name in modules: - imported.append(name) - return modules[name] - return real_import(name, *args, **kwargs) - - with mock.patch.object(builtins, "__import__", side_effect=record_import): - self.assertEqual( - BENCHMARK._load_runtime_modules(), - (modules["tensorrt_llm"], modules["torch"]), - ) - self.assertCountEqual(imported, ["tensorrt_llm", "torch"]) - - def test_single_rank_success_writes_rank_manifest_and_junit(self): - with tempfile.TemporaryDirectory() as temp_dir: - output_dir = Path(temp_dir) - torch = _Torch() - result = _run( - output_dir, - torch, - stage="Single-GPU", - commit="deadbeef", - environ={}, - ) - rank, manifest, junit = _read_outputs(output_dir) - - self.assertEqual(result["overall_status"], "passed") - self.assertEqual(rank["tensorrt_llm_module"], TRTLLM.__file__) - self.assertEqual(manifest["product_tests_executed"], 0) - self.assertEqual((manifest["stage"], manifest["commit"]), ("Single-GPU", "deadbeef")) - self.assertEqual(manifest["observed_ranks"], [0]) - self.assertEqual( - junit.find(".//testcase").attrib["name"], - "infra_dry_run_rank_0_cuda_matmul", - ) - self.assertIsNone(junit.find(".//failure")) - self.assertEqual((torch.cuda.selected, torch.cuda.synchronized), (0, 0)) - - def test_cuda_failures_return_nonzero_and_write_failure_junit(self): - scenarios = [ - (_Torch(cuda=False), "CUDA is required"), - (_Torch(finite=False), "non-finite values"), - (_Torch(numel=0), "empty tensor"), - ] - for torch, expected in scenarios: - with self.subTest(expected=expected), tempfile.TemporaryDirectory() as temp_dir: - output_dir = Path(temp_dir) - with self.assertRaises(BENCHMARK.BenchmarkError): - _run(output_dir, torch, environ={}) - rank, manifest, junit = _read_outputs(output_dir) - self.assertIn(expected, rank["error"]) - self.assertEqual(manifest["status"], "failed") - self.assertIsNotNone(junit.find(".//failure")) - - def test_cpu_mode_runs_without_cuda_and_writes_cpu_metadata(self): - with tempfile.TemporaryDirectory() as temp_dir: - output_dir = Path(temp_dir) - result = _run( - output_dir, - _Torch(cuda=False), - stage="CPU-Generic-x86-1", - device_type="cpu", - environ={}, - ) - rank, manifest, junit = _read_outputs(output_dir) - - self.assertEqual(result["overall_status"], "passed") - self.assertEqual(rank["cpu"]["device"], "cpu") - self.assertEqual(manifest["device_type"], "cpu") - self.assertEqual(manifest["distributed_backend"], "none") - self.assertEqual(manifest["product_tests_executed"], 0) - self.assertIsNone(result.get("cuda")) - self.assertEqual( - junit.find(".//testcase").attrib["name"], - "infra_dry_run_rank_0_cpu_matmul", - ) - self.assertIsNone(junit.find(".//failure")) - - def test_import_failure_writes_failure_artifacts(self): - with tempfile.TemporaryDirectory() as temp_dir: - output_dir = Path(temp_dir) - with mock.patch.object( - BENCHMARK, - "_load_runtime_modules", - side_effect=ImportError("tensorrt_llm is not installed"), - ): - with self.assertRaises(BENCHMARK.BenchmarkError): - BENCHMARK.run_benchmark(output_dir, environ={}) - rank, manifest, junit = _read_outputs(output_dir) - - self.assertIn("is not installed", rank["error"]) - self.assertEqual(manifest["status"], "failed") - self.assertIsNotNone(junit.find(".//failure")) - - def test_multi_rank_uses_one_nccl_gather_and_cleans_up(self): - remote = _result(1) - with tempfile.TemporaryDirectory() as temp_dir: - output_dir = Path(temp_dir) - torch = _Torch(remote=remote) - result = _run( - output_dir, - torch, - stage="Multi-GPU", - environ={"RANK": "0", "LOCAL_RANK": "0", "WORLD_SIZE": "2"}, - ) - _, manifest, junit = _read_outputs(output_dir) - - self.assertEqual(result["overall_status"], "passed") - self.assertEqual(manifest["observed_ranks"], [0, 1]) - self.assertEqual(len(junit.findall(".//testcase")), 3) - self.assertEqual(torch.distributed.gather_calls, 1) - self.assertTrue(torch.distributed.destroyed) - - def test_remote_rank_failure_makes_process_fail_and_cleans_up(self): - remote = _result(1, status="failed", checksum=0.0) - with tempfile.TemporaryDirectory() as temp_dir: - output_dir = Path(temp_dir) - torch = _Torch(remote=remote) - with self.assertRaises(BENCHMARK.BenchmarkError): - _run( - output_dir, - torch, - environ={"RANK": "0", "LOCAL_RANK": "0", "WORLD_SIZE": "2"}, - ) - rank, manifest, junit = _read_outputs(output_dir) - - self.assertEqual(rank["overall_status"], "failed") - self.assertEqual(manifest["status"], "failed") - self.assertIsNotNone(junit.find(".//failure")) - self.assertEqual(torch.distributed.gather_calls, 1) - self.assertTrue(torch.distributed.destroyed) - - def test_manifest_rejects_missing_mismatched_and_inconsistent_ranks(self): - scenarios = [ - ([_result(0)], "observed ranks [0] do not match expected [0, 1]"), - ( - [_result(0), _result(1, world_size=3)], - "rank results contain a world-size mismatch", - ), - ( - [_result(0), _result(1, checksum=256.0)], - "rank results contain inconsistent CUDA checksums", - ), - ] - for summaries, expected in scenarios: - with self.subTest(expected=expected), tempfile.TemporaryDirectory() as temp_dir: - output_dir = Path(temp_dir) - result = { - "world_size": 2, - "stage": "Multi-GPU", - "commit": "abc123", - } - errors = BENCHMARK._validate(summaries, 2) - manifest = BENCHMARK._write_reports(output_dir, result, summaries, errors) - junit = ET.parse(output_dir / "results-infra_dry_run.xml") - self.assertIn(expected, errors) - self.assertEqual(manifest["status"], "failed") - self.assertIsNotNone(junit.find(".//failure")) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 877fc93629f5..42bcd06d36a1 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -24,7 +24,15 @@ GROOVY = (REPO_ROOT / "jenkins" / "L0_Test.groovy").read_text() PARENT_GROOVY = (REPO_ROOT / "jenkins" / "L0_MergeRequest.groovy").read_text() SLURM_RUN = (REPO_ROOT / "jenkins" / "scripts" / "slurm_run.sh").read_text() +LLMAPI_LAUNCHER = (REPO_ROOT / "tensorrt_llm" / "llmapi" / "trtllm-llmapi-launch").read_text() +EXECUTOR_UTILS = (REPO_ROOT / "tensorrt_llm" / "executor" / "utils.py").read_text() +MPI_SESSION = (REPO_ROOT / "tensorrt_llm" / "llmapi" / "mpi_session.py").read_text() SLURM_INSTALL_PATH = REPO_ROOT / "jenkins" / "scripts" / "slurm_install.sh" +BENCHMARK_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "infra_dry_run_benchmark.py" +BENCHMARK = BENCHMARK_PATH.read_text() +DRY_RUN_DB_PATH = ( + REPO_ROOT / "tests" / "integration" / "test_lists" / "test-db" / "infra_dry_run.yml" +) def _function_body(source, name, next_name): @@ -42,39 +50,53 @@ def _map_keys(source, assignment_index): class InfraDryRunPipelineTest(unittest.TestCase): - def test_direct_command_selects_python_or_torchrun(self): - body = _function_body( - GROOVY, - "getInfraDryRunDirectCommand", - "getPytestBaseCommandLine", + def test_dry_run_is_a_test_db_selected_positional_pytest_module(self): + process = _function_body(GROOVY, "processShardTestList", "isValidSlurmJobId") + platform = _function_body( + GROOVY, "runLLMTestlistOnPlatformImpl", "runLLMTestlistOnPlatform" ) - self.assertIn("torch.cuda.device_count()", body) - self.assertIn("if [ '${deviceType}' = 'cpu' ]", body) - self.assertIn('if [ "\\$gpu_count" -gt 1 ]', body) - self.assertIn('torchrun --standalone --nproc-per-node="\\$gpu_count"', body) - self.assertIn("python3 ${benchmarkArgs}", body) - self.assertIn('stageName.startsWith("CPU-") ? "cpu" : "cuda"', body) - self.assertIn("\"--device '${deviceType}'\"", body) - - def test_direct_command_is_posix_shell_compatible(self): - body = _function_body( - GROOVY, - "getInfraDryRunDirectCommand", - "getPytestBaseCommandLine", + self.assertTrue(BENCHMARK_PATH.is_file()) + self.assertFalse(BENCHMARK_PATH.name.startswith("test_")) + self.assertEqual(DRY_RUN_DB_PATH.read_text().splitlines()[1], "infra_dry_run:") + self.assertIn( + "infra_dry_run_benchmark.py::test_infra_dry_run_benchmark", + DRY_RUN_DB_PATH.read_text(), + ) + self.assertIn('positionalTest=""', process) + self.assertIn("if (positionalTest)", process) + self.assertIn("testListCmd += [positionalTest]", process) + self.assertIn( + "effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList", platform + ) + self.assertIn("extraArgs += [benchmarkPath]", platform) + self.assertIn("--test-list=${preprocessedLists.regular}", platform) + self.assertIn("rerunFailedTests(", platform) + self.assertIn("runIsolatedTests(", platform) + self.assertNotIn("getInfraDryRunDirectCommand", GROOVY) + self.assertIn("create_mpi_comm_session", BENCHMARK) + self.assertIn("session.submit_sync(", BENCHMARK) + self.assertIn("_pickleable_llmapi_rank_task(), timeout_seconds", BENCHMARK) + self.assertNotIn("torchrun", BENCHMARK) + self.assertNotIn("subprocess", BENCHMARK) + self.assertFalse( + (REPO_ROOT / "jenkins" / "scripts" / "infra_dry_run_benchmark.py").exists() ) - self.assertIn("set -eu", body) - self.assertNotIn("pipefail", body) def test_docs_dry_run_bypasses_normal_doc_build_and_keeps_results(self): + prepared = _function_body(GROOVY, "runInfraDryRunInPreparedWorkspace", "runLLMDocBuild") body = _function_body(GROOVY, "runLLMDocBuild", "launchTestListCheck") dry_guard = body.index("if (isInfraDryRun())") - benchmark = body.index("getInfraDryRunDirectCommand(", dry_guard) + benchmark = body.index("runInfraDryRunInPreparedWorkspace(", dry_guard) early_return = body.index("return", benchmark) sphinx = body.index("make html") self.assertLess(dry_guard, benchmark) self.assertLess(benchmark, early_return) self.assertLess(early_return, sphinx) - self.assertIn('"${WORKSPACE}/${stageName}"', body[dry_guard:early_return]) + self.assertIn("renderTestDB(", prepared) + self.assertIn("processShardTestList(", prepared) + self.assertIn("getPytestBaseCommandLine(", prepared) + self.assertIn("withCredentials([", prepared) + self.assertIn("benchmarkPath", prepared) doc_jobs = GROOVY[ GROOVY.index("docBuildConfigs = [") : GROOVY.index("// Python version and OS") @@ -82,7 +104,7 @@ def test_docs_dry_run_bypasses_normal_doc_build_and_keeps_results(self): self.assertIn('runLLMDocBuild(pipeline, VANILLA_CONFIG, "A10-Build_Docs")', doc_jobs) self.assertIn("{}, !isInfraDryRun(), attemptTag", doc_jobs) - def test_package_sanity_uses_the_shared_direct_benchmark_path(self): + def test_package_sanity_uses_the_shared_platform_pytest_path(self): package_jobs = GROOVY[ GROOVY.index("sanityCheckJobs =") : GROOVY.index( "multiGpuJobs =", GROOVY.index("sanityCheckJobs =") @@ -92,12 +114,22 @@ def test_package_sanity_uses_the_shared_direct_benchmark_path(self): self.assertIn("toStageName(values[1], key)", package_jobs) self.assertNotIn('"CPU-', package_jobs) - def test_slurm_command_allocates_one_task_per_gpu(self): - body = _function_body(GROOVY, "getInfraDryRunNodeArgs", "getInfraDryRunDirectCommand") - self.assertIn('"--nodes=${nodeCount}"', body) - self.assertIn('"--ntasks=${gpuCount}"', body) - self.assertIn('"--ntasks-per-node=${gpusPerNode}"', body) - self.assertIn('"--gpus-per-node=${gpusPerNode}"', body) + def test_slurm_uses_standard_resources_and_pytest_command(self): + body = GROOVY[ + GROOVY.index("def runLLMTestlistWithSbatch") : GROOVY.index("def runLLMTestlistOnSlurm") + ] + self.assertIn("String[] taskArgs = getNodeArgs(", body) + self.assertNotIn("getInfraDryRunNodeArgs", GROOVY) + self.assertIn( + "effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList", body + ) + self.assertIn("effectiveSplitId = infraDryRun ? 1 : splitId", body) + self.assertIn("effectiveSplits = infraDryRun ? 1 : splits", body) + self.assertIn("effectivePerfMode = infraDryRun ? false : perfMode", body) + self.assertIn("infra_dry_run_waives.txt", body) + self.assertIn("${INFRA_DRY_RUN_BENCHMARK}", body) + self.assertIn('pytestUtil = "$llmSrcNode/tensorrt_llm/llmapi/trtllm-llmapi-launch"', body) + self.assertIn("if(nodeCount > 1) {", body) def test_slurm_maps_ranks_and_uses_stable_rendezvous(self): for assignment in ( @@ -112,11 +144,30 @@ def test_slurm_maps_ranks_and_uses_stable_rendezvous(self): self.assertIn("20000 + SLURM_JOB_ID % 20000", GROOVY) self.assertIn("--container-env=MASTER_ADDR", GROOVY) self.assertIn("--container-env=MASTER_PORT", GROOVY) - self.assertIn("--distributed-timeout-seconds 900", SLURM_RUN) self.assertLess( - SLURM_RUN.index('if [[ "${infraDryRun:-false}" == "true" ]]'), + SLURM_RUN.index('if [[ "${infraDryRun:-false}" == "true"'), SLURM_RUN.index("eval $pytestCommand"), ) + self.assertNotIn("infra_dry_run_benchmark.py", SLURM_RUN) + self.assertNotIn("exit $?", SLURM_RUN) + self.assertIn("export TLLM_SPAWN_PROXY_PROCESS=1", LLMAPI_LAUNCHER) + self.assertIn('if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]', LLMAPI_LAUNCHER) + self.assertIn("python3 -m tensorrt_llm.llmapi.mgmn_worker_node", LLMAPI_LAUNCHER) + self.assertIn("unset RANK LOCAL_RANK WORLD_SIZE", SLURM_RUN) + + def test_llmapi_session_contract_matches_the_benchmark_adapter(self): + create_session = _function_body(EXECUTOR_UTILS, "create_mpi_comm_session", "has_event_loop") + remote_session = MPI_SESSION[ + MPI_SESSION.index("class RemoteMpiCommSessionClient") : MPI_SESSION.index( + "class RemoteMpiCommSessionServer" + ) + ] + self.assertIn("n_workers: int", create_session) + self.assertIn("RemoteMpiCommSessionClient(", create_session) + self.assertIn("def submit_sync(self, task, *args, **kwargs) -> List[T]", remote_session) + self.assertIn("return res", remote_session) + self.assertIn("pickle.dumps(obj)", (REPO_ROOT / "tensorrt_llm/executor/ipc.py").read_text()) + self.assertIn("_pickleable_llmapi_rank_task()", BENCHMARK) def test_slurm_artifact_download_replaces_existing_archive(self): with tempfile.TemporaryDirectory() as temp_dir: @@ -126,7 +177,7 @@ def test_slurm_artifact_download_replaces_existing_archive(self): tar_record_path = temp_path / "tar-input-path" archive_path.write_text("stale\n") - script = r''' + script = r""" source "$SLURM_INSTALL_PATH" retry_command() { if [[ "$1" == "--timeout" ]]; then @@ -164,7 +215,7 @@ def test_slurm_artifact_download_replaces_existing_archive(self): python3() { :; } export -f pip3 wget slurm_install_setup -''' +""" env = { **os.environ, "SLURM_INSTALL_PATH": str(SLURM_INSTALL_PATH), @@ -196,27 +247,27 @@ def test_slurm_artifact_download_replaces_existing_archive(self): self.assertFalse(Path(f"{archive_path}.1").exists()) self.assertFalse(Path(expected_tmp).exists()) - def test_direct_branch_follows_existing_shard_setup(self): + def test_dry_pytest_failure_propagates_without_rerun_or_isolation(self): body = _function_body( GROOVY, "runLLMTestlistOnPlatformImpl", "runLLMTestlistOnPlatform", ) - command_index = body.index("getInfraDryRunDirectCommand(") - self.assertLess(body.index("processShardTestList("), command_index) - self.assertGreater(body.index("withCredentials([", command_index), command_index) - self.assertGreater(body.index("No tests were executed", command_index), command_index) - - def test_infra_junit_is_required_and_cbts_is_disabled(self): + command_area = body.index('withEnv(["LD_LIBRARY_PATH=') + branch_start = body.index("if (infraDryRun) {", command_area) + dry_branch = body[branch_start : body.index("try {", branch_start)] + self.assertIn('${pytestCommand.join(" ")}', dry_branch) + self.assertNotIn("rerunFailedTests", dry_branch) + self.assertNotIn("runIsolatedTests", dry_branch) + self.assertNotIn("catch", dry_branch) + + def test_standard_junit_is_used_and_cbts_is_disabled(self): self.assertIn( - 'junit(testResults: "${stageName}/results-infra_dry_run*.xml")', - GROOVY, - ) - self.assertNotIn( - 'junit(allowEmptyResults: true, testResults: "${stageName}/results-infra_dry_run', + 'junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml")', GROOVY, ) - self.assertIn("Failed to collect infrastructure dry-run JSON results", GROOVY) + self.assertNotIn("results-infra_dry_run", GROOVY) + self.assertNotIn("infra_dry_run*.json", GROOVY) cbts_body = _function_body(GROOVY, "isCbtsStage", "scpFromRemoteCmd") self.assertIn("if (isInfraDryRun())", cbts_body) self.assertIn("return false", cbts_body) diff --git a/tests/unittest/tools/test_infra_dry_run_pytest.py b/tests/unittest/tools/test_infra_dry_run_pytest.py new file mode 100644 index 000000000000..857f574cfc58 --- /dev/null +++ b/tests/unittest/tools/test_infra_dry_run_pytest.py @@ -0,0 +1,421 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import os +import pickle +import signal +import subprocess +import sys +import tempfile +import textwrap +import threading +import types +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +BENCHMARK_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "infra_dry_run_benchmark.py" + +_IMPORT_TORCH = types.ModuleType("torch") +with mock.patch.dict(sys.modules, {"torch": _IMPORT_TORCH}): + # Pytest's default import mode treats defs/ as a package; exercise the + # package-qualified controller name rather than the worker's top-level name. + SPEC = importlib.util.spec_from_file_location("defs.infra_dry_run_benchmark", BENCHMARK_PATH) + BENCHMARK = importlib.util.module_from_spec(SPEC) + assert SPEC.loader is not None + SPEC.loader.exec_module(BENCHMARK) + + +class _Scalar: + def __init__(self, value): + self.value = value + + def item(self): + return self.value + + +class _Tensor: + def __init__(self, values, *, device="cpu", dtype="float32"): + self.values = values + self.device = SimpleNamespace(type=device) + self.dtype = dtype + + def all(self): + return _Scalar(True) + + def cpu(self): + return self + + def tolist(self): + return list(self.values) + + def item(self): + return self.values + + +class _CpuTorch: + float32 = "float32" + + def __init__(self): + self.seed = None + self.matmul_calls = 0 + + def manual_seed(self, seed): + self.seed = seed + + def full(self, _shape, value, *, dtype, device): + return _Tensor(value, device=device, dtype=dtype) + + def matmul(self, _left, _right): + self.matmul_calls += 1 + return _Tensor(4.0) + + def full_like(self, _tensor, value): + return _Tensor(value) + + def isfinite(self, tensor): + return tensor + + def equal(self, left, right): + return left.values == right.values + + +class InfraDryRunPytestTest(unittest.TestCase): + def test_explicit_module_and_test_list_execute_but_normal_collection_ignores_it(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + benchmark = root / BENCHMARK_PATH.name + benchmark.write_text(BENCHMARK_PATH.read_text()) + (root / "torch.py").write_text( + textwrap.dedent( + """ + float32 = "float32" + + class Tensor: + def __init__(self, value, dtype="float32", device="cpu"): + self.value = value + self.dtype = dtype + self.device = type("Device", (), {"type": device})() + def all(self): return self + def item(self): return self.value + + def manual_seed(_seed): pass + def full(_shape, value, *, dtype, device): + return Tensor(value, dtype, device) + def matmul(_left, _right): return Tensor(4.0) + def full_like(_tensor, value): return Tensor(value) + def isfinite(tensor): return tensor + def equal(left, right): return left.value == right.value + """ + ) + ) + (root / "conftest.py").write_text( + textwrap.dedent( + """ + def pytest_addoption(parser): + parser.addoption("--test-list") + parser.addoption("--test-prefix") + + def pytest_collection_modifyitems(config, items): + prefix = config.getoption("--test-prefix") + if prefix: + for item in items: + item._nodeid = f"{prefix}/{item.nodeid}" + test_list = config.getoption("--test-list") + if not test_list: + return + wanted = { + f"{prefix}/{line.strip()}" if prefix else line.strip() + for line in open(test_list) + if line.strip() + } + selected = [item for item in items if item.nodeid in wanted] + config.hook.pytest_deselected( + items=[item for item in items if item not in selected] + ) + items[:] = selected + """ + ) + ) + test_list = root / "infra_dry_run.txt" + test_list.write_text("infra_dry_run_benchmark.py::test_infra_dry_run_benchmark\n") + env = {**os.environ, "stageName": "CPU-Validation"} + explicit = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + f"--test-list={test_list}", + "--test-prefix=CPU-Validation", + str(benchmark), + ], + cwd=root, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + normal = subprocess.run( + [sys.executable, "-m", "pytest", "--collect-only", "-q", str(root)], + cwd=root, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + + self.assertEqual(explicit.returncode, 0, explicit.stdout) + self.assertIn("1 passed", explicit.stdout) + self.assertEqual(normal.returncode, 5, normal.stdout) + self.assertNotIn(BENCHMARK_PATH.name, normal.stdout) + + def test_bounded_wait_interrupts_and_restores_the_signal_handler(self): + previous_handler = signal.getsignal(signal.SIGALRM) + with self.assertRaisesRegex(TimeoutError, "exceeded 30 seconds"): + with BENCHMARK._bounded_wait(30): + signal.raise_signal(signal.SIGALRM) + self.assertIs(signal.getsignal(signal.SIGALRM), previous_handler) + + def test_bounded_wait_rejects_non_main_threads_and_existing_timers(self): + errors = [] + + def enter_wait(): + try: + with BENCHMARK._bounded_wait(30): + pass + except BaseException as error: + errors.append(error) + + thread = threading.Thread(target=enter_wait) + thread.start() + thread.join() + self.assertEqual(len(errors), 1) + self.assertRegex(str(errors[0]), "requires the main thread") + + with ( + mock.patch.object(BENCHMARK.signal, "getitimer", return_value=(1.0, 0.0)), + self.assertRaisesRegex(RuntimeError, "cannot replace an active ITIMER_REAL"), + ): + with BENCHMARK._bounded_wait(30): + pass + + def test_cpu_path_is_explicit_deterministic_fp32(self): + torch_module = _CpuTorch() + BENCHMARK._run_cpu(torch_module) + self.assertEqual(torch_module.seed, 0) + self.assertEqual(torch_module.matmul_calls, 1) + + def test_cpu_stage_routes_through_the_pytest_controller(self): + with ( + mock.patch.dict(os.environ, {"stageName": "CPU-Generic-x86-1"}, clear=True), + mock.patch.object(BENCHMARK, "_run_cpu") as run_cpu, + ): + BENCHMARK.test_infra_dry_run_benchmark() + run_cpu.assert_called_once_with() + + def test_gpu_stage_never_falls_back_to_cpu(self): + fake_torch = SimpleNamespace( + cuda=SimpleNamespace(is_available=lambda: False, device_count=lambda: 0) + ) + with ( + mock.patch.dict(os.environ, {"stageName": "A10-GPU"}, clear=True), + mock.patch.object(BENCHMARK, "torch", fake_torch), + mock.patch.object(BENCHMARK, "_run_cpu") as run_cpu, + self.assertRaisesRegex(RuntimeError, "CUDA is required"), + ): + BENCHMARK.test_infra_dry_run_benchmark() + run_cpu.assert_not_called() + + def test_rank_environment_must_be_complete_and_in_range(self): + self.assertIsNone(BENCHMARK._external_rank_context({})) + with self.assertRaisesRegex(RuntimeError, "missing"): + BENCHMARK._external_rank_context({"RANK": "0"}) + with self.assertRaisesRegex(RuntimeError, "outside WORLD_SIZE"): + BENCHMARK._external_rank_context({"RANK": "2", "LOCAL_RANK": "0", "WORLD_SIZE": "2"}) + self.assertEqual( + BENCHMARK._external_rank_context({"RANK": "1", "LOCAL_RANK": "1", "WORLD_SIZE": "2"}), + (1, 1, 2), + ) + + def test_rank_summary_validation_rejects_missing_or_inconsistent_ranks(self): + BENCHMARK._validate_rank_summaries([[0.0, 2.0, 10.0], [1.0, 2.0, 10.0]], 2) + with self.assertRaisesRegex(RuntimeError, "observed ranks"): + BENCHMARK._validate_rank_summaries([[0.0, 2.0, 10.0]], 2) + with self.assertRaisesRegex(RuntimeError, "world sizes"): + BENCHMARK._validate_rank_summaries([[0.0, 2.0, 10.0], [1.0, 3.0, 10.0]], 2) + with self.assertRaisesRegex(RuntimeError, "checksums"): + BENCHMARK._validate_rank_summaries([[0.0, 2.0, 10.0], [1.0, 2.0, 11.0]], 2) + + def test_distributed_failure_always_destroys_the_process_group(self): + class Distributed: + def __init__(self): + self.destroyed = False + + def is_available(self): + return True + + def is_nccl_available(self): + return True + + def is_initialized(self): + return True + + def destroy_process_group(self): + self.destroyed = True + + distributed = Distributed() + torch_module = SimpleNamespace(distributed=distributed) + with ( + mock.patch.object( + BENCHMARK, "_run_cuda_matmul", side_effect=RuntimeError("CUDA failed") + ), + self.assertRaisesRegex(RuntimeError, "CUDA failed"), + ): + BENCHMARK._run_distributed_rank(0, 0, 2, torch_module=torch_module) + self.assertTrue(distributed.destroyed) + + def test_multi_node_uses_the_existing_llmapi_session(self): + class Session: + def __init__(self): + self.shutdown_called = False + self.submission = None + + def submit_sync(self, task, timeout): + self.submission = (task, timeout) + return [[0.0, 2.0, 10.0], [1.0, 2.0, 10.0]] + + def shutdown(self): + self.shutdown_called = True + + session = Session() + with mock.patch.dict(sys.modules, {"torch": _IMPORT_TORCH}): + BENCHMARK._run_with_existing_llmapi_launcher( + 2, timeout_seconds=30, session_factory=lambda world_size: session + ) + task, timeout = session.submission + self.assertEqual(task.__module__, "infra_dry_run_benchmark") + self.assertEqual(Path(task.__code__.co_filename).resolve(), BENCHMARK_PATH) + self.assertEqual(timeout, 30) + self.assertTrue(session.shutdown_called) + + def test_llmapi_rank_task_pickle_is_importable_in_worker_directory(self): + with mock.patch.dict(sys.modules, {"torch": _IMPORT_TORCH}): + task = BENCHMARK._pickleable_llmapi_rank_task() + payload = pickle.dumps(task) + with tempfile.TemporaryDirectory() as temp_dir: + Path(temp_dir, "torch.py").write_text("# worker import stub\n") + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join( + value for value in (temp_dir, env.get("PYTHONPATH")) if value + ) + result = subprocess.run( + [ + sys.executable, + "-c", + "import pickle, sys; print(pickle.loads(sys.stdin.buffer.read()).__module__)", + ], + cwd=BENCHMARK_PATH.parent, + env=env, + input=payload, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr.decode()) + self.assertEqual(result.stdout.decode().strip(), "infra_dry_run_benchmark") + + def test_llmapi_worker_binds_rank_environment_to_the_mpi_communicator(self): + communicator = SimpleNamespace(Get_rank=lambda: 1, Get_size=lambda: 2) + mpi4py = SimpleNamespace(MPI=SimpleNamespace(COMM_WORLD=communicator)) + environ = {"RANK": "1", "LOCAL_RANK": "0", "WORLD_SIZE": "2"} + with ( + mock.patch.dict(os.environ, environ, clear=True), + mock.patch.dict(sys.modules, {"mpi4py": mpi4py}), + mock.patch.object( + BENCHMARK, "_run_distributed_rank", return_value=[1.0, 2.0, 10.0] + ) as run_rank, + ): + result = BENCHMARK._llmapi_rank_task(45) + self.assertEqual(result, [1.0, 2.0, 10.0]) + run_rank.assert_called_once_with(1, 0, 2, 45) + + def test_external_multi_node_rank_reuses_existing_launcher(self): + environ = { + "stageName": "GB300-MultiNode", + "RANK": "0", + "LOCAL_RANK": "0", + "WORLD_SIZE": "2", + "MASTER_ADDR": "host0", + "MASTER_PORT": "23456", + "TLLM_SPAWN_PROXY_PROCESS": "1", + } + with ( + mock.patch.dict(os.environ, environ, clear=True), + mock.patch.object(BENCHMARK, "_run_with_existing_llmapi_launcher") as run_launcher, + ): + BENCHMARK.test_infra_dry_run_benchmark() + run_launcher.assert_called_once_with(2) + + def test_nonzero_proxy_rank_cannot_be_the_pytest_controller(self): + environ = { + "stageName": "GB300-MultiNode", + "RANK": "1", + "LOCAL_RANK": "1", + "WORLD_SIZE": "2", + "MASTER_ADDR": "host0", + "MASTER_PORT": "23456", + "TLLM_SPAWN_PROXY_PROCESS": "1", + } + with ( + mock.patch.dict(os.environ, environ, clear=True), + mock.patch.object(BENCHMARK, "_run_with_existing_llmapi_launcher") as run_launcher, + self.assertRaisesRegex(RuntimeError, "only LLMAPI rank 0"), + ): + BENCHMARK.test_infra_dry_run_benchmark() + run_launcher.assert_not_called() + + def test_single_node_multi_gpu_spawns_one_worker_per_visible_gpu(self): + cuda = SimpleNamespace(is_available=lambda: True, device_count=lambda: 4) + multiprocessing = SimpleNamespace(spawn=mock.Mock()) + fake_torch = SimpleNamespace(cuda=cuda, multiprocessing=multiprocessing) + with ( + mock.patch.dict(os.environ, {"stageName": "H100-Multi-GPU"}, clear=True), + mock.patch.dict(sys.modules, {"torch": fake_torch}), + mock.patch.object(BENCHMARK, "torch", fake_torch), + mock.patch.object(BENCHMARK, "_reserve_local_port", return_value=23456), + ): + BENCHMARK.test_infra_dry_run_benchmark() + + multiprocessing.spawn.assert_called_once() + worker = multiprocessing.spawn.call_args.args[0] + self.assertEqual(worker.__module__, "infra_dry_run_benchmark") + self.assertEqual( + multiprocessing.spawn.call_args.kwargs, + { + "args": (4, 23456, BENCHMARK._DISTRIBUTED_TIMEOUT_SECONDS), + "nprocs": 4, + "join": True, + }, + ) + + +if __name__ == "__main__": + unittest.main() From 96de91874dd82a02d6a73a95f0653a7e8d6f0c73 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:31:33 -0700 Subject: [PATCH 10/34] ci: collect dry-run benchmark during test-list validation The L0 test-list check used pytest's default discovery patterns, so the intentionally non-test_*.py dry-run benchmark was absent from the collected node IDs and its test DB entry was rejected as invalid. Extend only the L0 validation collection patterns to retain pytest's defaults and include the dry-run module. Add focused coverage that validates normal and nonstandard node IDs together while normal pytest discovery continues to ignore the benchmark. Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- scripts/check_test_list.py | 7 +++-- .../tools/test_infra_dry_run_pipeline.py | 5 +++ .../tools/test_infra_dry_run_pytest.py | 31 ++++++++++++++++++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/scripts/check_test_list.py b/scripts/check_test_list.py index 2a5a44367052..5206c1de3d50 100755 --- a/scripts/check_test_list.py +++ b/scripts/check_test_list.py @@ -46,6 +46,8 @@ # AST validation defaults _DEFAULT_TEST_LISTS_DIR = "tests/integration/test_lists" _DEFAULT_TEST_BASE_DIR = "tests/integration/defs" +# Preserve pytest's defaults while collecting the intentionally nonstandard dry-run module. +_L0_PYTEST_FILE_PATTERNS = "test_*.py *_test.py infra_dry_run_benchmark.py" # Paths whose tests are generated dynamically — skip AST validation _EXCLUDED_PATH_PREFIXES = ("perf/", ) @@ -967,7 +969,8 @@ def verify_l0_test_lists(llm_src): subprocess.run( f"cd {llm_src}/tests/integration/defs && " - f"pytest --test-list={test_list} --output-dir={llm_src} -s --co -q", + f"pytest -o \"python_files={_L0_PYTEST_FILE_PATTERNS}\" " + f"--test-list={test_list} --output-dir={llm_src} -s --co -q", shell=True, check=True) @@ -1040,7 +1043,7 @@ def check_waive_duplicates(llm_src): f.write( f" Occurrence {i} at line {line_no}: '{original_line}'\n" ) - f.write(f"\n") + f.write("\n") def verify_waive_list(llm_src, args): diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 42bcd06d36a1..e912f7dcfd39 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -33,6 +33,7 @@ DRY_RUN_DB_PATH = ( REPO_ROOT / "tests" / "integration" / "test_lists" / "test-db" / "infra_dry_run.yml" ) +CHECK_TEST_LIST = (REPO_ROOT / "scripts" / "check_test_list.py").read_text() def _function_body(source, name, next_name): @@ -62,6 +63,10 @@ def test_dry_run_is_a_test_db_selected_positional_pytest_module(self): "infra_dry_run_benchmark.py::test_infra_dry_run_benchmark", DRY_RUN_DB_PATH.read_text(), ) + l0_validation = _function_body( + CHECK_TEST_LIST, "verify_l0_test_lists", "verify_qa_test_lists" + ) + self.assertIn('pytest -o \\"python_files={_L0_PYTEST_FILE_PATTERNS}\\"', l0_validation) self.assertIn('positionalTest=""', process) self.assertIn("if (positionalTest)", process) self.assertIn("testListCmd += [positionalTest]", process) diff --git a/tests/unittest/tools/test_infra_dry_run_pytest.py b/tests/unittest/tools/test_infra_dry_run_pytest.py index 857f574cfc58..e682536f97cb 100644 --- a/tests/unittest/tools/test_infra_dry_run_pytest.py +++ b/tests/unittest/tools/test_infra_dry_run_pytest.py @@ -154,6 +154,12 @@ def pytest_collection_modifyitems(config, items): ) test_list = root / "infra_dry_run.txt" test_list.write_text("infra_dry_run_benchmark.py::test_infra_dry_run_benchmark\n") + (root / "test_normal.py").write_text("def test_normal(): pass\n") + validation_list = root / "all_l0.txt" + validation_list.write_text( + "infra_dry_run_benchmark.py::test_infra_dry_run_benchmark\n" + "test_normal.py::test_normal\n" + ) env = {**os.environ, "stageName": "CPU-Validation"} explicit = subprocess.run( [ @@ -172,6 +178,24 @@ def pytest_collection_modifyitems(config, items): stderr=subprocess.STDOUT, check=False, ) + validation = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "--collect-only", + "-o", + "python_files=test_*.py *_test.py infra_dry_run_benchmark.py", + f"--test-list={validation_list}", + ], + cwd=root, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) normal = subprocess.run( [sys.executable, "-m", "pytest", "--collect-only", "-q", str(root)], cwd=root, @@ -184,7 +208,12 @@ def pytest_collection_modifyitems(config, items): self.assertEqual(explicit.returncode, 0, explicit.stdout) self.assertIn("1 passed", explicit.stdout) - self.assertEqual(normal.returncode, 5, normal.stdout) + self.assertEqual(validation.returncode, 0, validation.stdout) + self.assertIn("infra_dry_run_benchmark.py::test_infra_dry_run_benchmark", validation.stdout) + self.assertIn("test_normal.py::test_normal", validation.stdout) + self.assertIn("2 tests collected", validation.stdout) + self.assertEqual(normal.returncode, 0, normal.stdout) + self.assertIn("test_normal.py::test_normal", normal.stdout) self.assertNotIn(BENCHMARK_PATH.name, normal.stdout) def test_bounded_wait_interrupts_and_restores_the_signal_handler(self): From 1434369a289d5d47c29ed5b1670e3ebeb6c1de3d Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:21:42 -0700 Subject: [PATCH 11/34] ci: make dry-run spawn workers importable Single-node multi-GPU dry runs serialize a top-level benchmark worker from a pytest package-qualified controller. Spawn children inherited no import path for that top-level module, so they exited during unpickling before PyTorch could write a child error file. Add the benchmark module directory to the spawn preparation path before returning serialized workers. Cover the failure with a real package-qualified subprocess and multiprocessing spawn child while leaving external-rank and LLMAPI execution unchanged. Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- .../defs/infra_dry_run_benchmark.py | 9 ++- .../tools/test_infra_dry_run_pytest.py | 60 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/tests/integration/defs/infra_dry_run_benchmark.py b/tests/integration/defs/infra_dry_run_benchmark.py index 05e38b8e36c9..1a25d4e98d1b 100644 --- a/tests/integration/defs/infra_dry_run_benchmark.py +++ b/tests/integration/defs/infra_dry_run_benchmark.py @@ -228,8 +228,13 @@ def _worker_import_module(): MGMN workers run from this file's directory and can import it only as ``infra_dry_run_benchmark``. Both RemoteMpiCommSession and multiprocessing serialize callables, so worker functions must come from that deterministic - top-level module. + top-level module and its directory must be inherited by spawn children. """ + module_path = Path(__file__).resolve() + module_dir = str(module_path.parent) + if module_dir not in sys.path: + sys.path.insert(0, module_dir) + module_name = Path(__file__).stem module = sys.modules.get(module_name) if module is None: @@ -245,7 +250,7 @@ def _worker_import_module(): raise else: module_file = getattr(module, "__file__", None) - if module_file is None or Path(module_file).resolve() != Path(__file__).resolve(): + if module_file is None or Path(module_file).resolve() != module_path: raise RuntimeError(f"{module_name} resolves to {module_file}, expected {__file__}") return module diff --git a/tests/unittest/tools/test_infra_dry_run_pytest.py b/tests/unittest/tools/test_infra_dry_run_pytest.py index e682536f97cb..faf1ae75e5e4 100644 --- a/tests/unittest/tools/test_infra_dry_run_pytest.py +++ b/tests/unittest/tools/test_infra_dry_run_pytest.py @@ -371,6 +371,66 @@ def test_llmapi_rank_task_pickle_is_importable_in_worker_directory(self): self.assertEqual(result.returncode, 0, result.stderr.decode()) self.assertEqual(result.stdout.decode().strip(), "infra_dry_run_benchmark") + def test_top_level_worker_module_is_importable_by_a_real_spawn_child(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + (temp_path / "torch.py").write_text("# import-only torch stub\n") + driver = temp_path / "spawn_driver.py" + driver.write_text( + textwrap.dedent( + f""" + import importlib.util + import multiprocessing + import sys + from pathlib import Path + + benchmark_path = Path({str(BENCHMARK_PATH)!r}) + + def main(): + spec = importlib.util.spec_from_file_location( + "defs.infra_dry_run_benchmark", benchmark_path + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + + module_dir = benchmark_path.parent.resolve() + sys.path[:] = [ + entry + for entry in sys.path + if Path(entry or ".").resolve() != module_dir + ] + worker_module = module._worker_import_module() + process = multiprocessing.get_context("spawn").Process( + target=worker_module._required_int, + args=({{"VALUE": "7"}}, "VALUE"), + ) + process.start() + process.join(30) + if process.is_alive(): + process.terminate() + process.join() + raise RuntimeError("spawn child did not finish") + raise SystemExit(process.exitcode) + + if __name__ == "__main__": + main() + """ + ) + ) + env = {**os.environ, "PYTHONPATH": temp_dir} + result = subprocess.run( + [sys.executable, str(driver)], + cwd=temp_dir, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stdout) + def test_llmapi_worker_binds_rank_environment_to_the_mpi_communicator(self): communicator = SimpleNamespace(Get_rank=lambda: 1, Get_size=lambda: 2) mpi4py = SimpleNamespace(MPI=SimpleNamespace(COMM_WORLD=communicator)) From 145febc8d7f55d81f9541138522b6ccb67a2b667 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Tue, 11 Aug 2026 04:31:45 -0700 Subject: [PATCH 12/34] ci: simplify infrastructure dry run Register the infrastructure benchmark as a standard pytest case in its dedicated test DB context and reuse the normal collection, JUnit, upload, and reporting flow. Remove positional-module collection overrides and custom multiprocessing, Remote MPI, rendezvous, and timeout orchestration while preserving non-dry-run behavior. Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 2 +- jenkins/L0_Test.groovy | 140 ++--- jenkins/scripts/slurm_run.sh | 15 - scripts/check_test_list.py | 7 +- .../defs/infra_dry_run_benchmark.py | 347 ------------ .../defs/test_infra_dry_run_benchmark.py | 58 ++ .../test_lists/test-db/infra_dry_run.yml | 2 +- .../tools/test_infra_dry_run_pipeline.py | 458 ++++------------ .../tools/test_infra_dry_run_pytest.py | 516 ++++-------------- 9 files changed, 304 insertions(+), 1241 deletions(-) delete mode 100644 tests/integration/defs/infra_dry_run_benchmark.py create mode 100644 tests/integration/defs/test_infra_dry_run_benchmark.py diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index c0e2298303f0..b3a041f59b22 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -2291,7 +2291,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } }]} - parallelJobs.failFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast + parallelJobs.failFast = enableFailFast pipeline.parallel parallelJobs } diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index e3b9da053108..95cced2a8e91 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -105,9 +105,6 @@ def LINUX_AARCH64_CONFIG = "linux_aarch64" @Field def INFRA_DRY_RUN_TEST_CONTEXT = "infra_dry_run" -@Field -def INFRA_DRY_RUN_BENCHMARK = "infra_dry_run_benchmark.py" - @Field def BUILD_CONFIGS = [ // Vanilla TARNAME is used for packaging in runLLMPackage @@ -239,9 +236,6 @@ def isInfraDryRun() { } def isCbtsStage(String stageName) { - if (isInfraDryRun()) { - return false - } // Pipeline-level eligibility (post-merge gate + kill switch) is decided in L0_MergeRequest.groovy and propagated via testFilter. if (!(testFilter[(CBTS_COVERAGE)] ?: false)) { return false @@ -407,42 +401,39 @@ def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String pipeline.stage('Submit Test Result') { sh "mkdir -p ${stageName}" // Download timeout test results - if (!isInfraDryRun()) { - def timeoutTestFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/unfinished_test.txt" - def downloadTimeoutTestSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, timeoutTestFilePath, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 - if (downloadTimeoutTestSucceed) { - if (stageIsInterrupted) { - echo "Stage is interrupted, skip to generate terminated unexpectedly test result." - } else { - sh "ls -al ${stageName}/" - // Generate timeout test result xml if there are terminated unexpectedly tests - hasTimeoutTest = generateTimeoutTestResultXml(pipeline, stageName) - } + def timeoutTestFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/unfinished_test.txt" + def downloadTimeoutTestSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, timeoutTestFilePath, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 + if (downloadTimeoutTestSucceed) { + if (stageIsInterrupted) { + echo "Stage is interrupted, skip to generate terminated unexpectedly test result." + } else { + sh "ls -al ${stageName}/" + // Generate timeout test result xml if there are terminated unexpectedly tests + hasTimeoutTest = generateTimeoutTestResultXml(pipeline, stageName) } } // Download normal test results def resultsFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/results*.xml" downloadResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, resultsFilePath, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 + // Download perf test results - if (!isInfraDryRun()) { - def perfResultsBasePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}" - def folderListOutput = Utils.exec( - pipeline, - script: Utils.sshUserCmd( - remote, - "\"find '${perfResultsBasePath}' -maxdepth 1 -type d \\( -name 'aggr*' -o -name 'disagg*' \\) -printf '%f\\n' || true\"" - ), - returnStdout: true, - numRetries: 3 - )?.trim() ?: "" - def perfFolders = folderListOutput.split(/\s+/).collect { it.trim().replaceAll(/\/$/, '') }.findAll { it } - echo "Perf Result Folders: ${perfFolders}" - if (perfFolders) { - def scpSources = perfFolders.size() == 1 - ? "${perfResultsBasePath}/${perfFolders[0]}" - : "{${perfFolders.collect { "${perfResultsBasePath}/${it}" }.join(',')}}" - downloadPerfResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, scpSources, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 - } + def perfResultsBasePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}" + def folderListOutput = Utils.exec( + pipeline, + script: Utils.sshUserCmd( + remote, + "\"find '${perfResultsBasePath}' -maxdepth 1 -type d \\( -name 'aggr*' -o -name 'disagg*' \\) -printf '%f\\n' || true\"" + ), + returnStdout: true, + numRetries: 3 + )?.trim() ?: "" + def perfFolders = folderListOutput.split(/\s+/).collect { it.trim().replaceAll(/\/$/, '') }.findAll { it } + echo "Perf Result Folders: ${perfFolders}" + if (perfFolders) { + def scpSources = perfFolders.size() == 1 + ? "${perfResultsBasePath}/${perfFolders[0]}" + : "{${perfFolders.collect { "${perfResultsBasePath}/${it}" }.join(',')}}" + downloadPerfResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, scpSources, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 } // Pull this stage's per-process .cbtscov files as one archive into ${stageName}/cbts/; bounded and non-fatal. @@ -578,7 +569,7 @@ def runIsolatedTests(preprocessedLists, testCmdLine, llmSrc, stageName) { return rerunFailed // Return the updated value } -def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, durationsPath="", positionalTest="") { +def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, durationsPath="") { // Preprocess testDBList to extract ISOLATION markers echo "Preprocessing testDBList to extract ISOLATION markers..." @@ -647,9 +638,6 @@ def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, du "--splits ${splits}", "--group ${splitId}", ] - if (positionalTest) { - testListCmd += [positionalTest] - } if (durationsPath) { testListCmd += ["--durations-path ${durationsPath}"] } @@ -1932,9 +1920,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "--group $effectiveSplitId", *clusterDurationsArgsNode, ] - if (infraDryRun) { - extraArgs += ["${llmSrcNode}/tests/integration/defs/${INFRA_DRY_RUN_BENCHMARK}"] - } if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { extraArgs += [ "--capture=fd", @@ -2081,10 +2066,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG if (ENABLE_UPLOAD_TEST_RESULTS) { srunArgs.add("--container-env=S3_SECRET_KEY") } - if (isInfraDryRun()) { - srunArgs.add("--container-env=MASTER_ADDR") - srunArgs.add("--container-env=MASTER_PORT") - } envVarsToExport.each { varName, varValue -> srunArgs.add("--container-env=${varName}") } @@ -2146,11 +2127,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG echo "Env NVIDIA_IMEX_CHANNELS: \$NVIDIA_IMEX_CHANNELS" echo "Env NVIDIA_VISIBLE_DEVICES: \$NVIDIA_VISIBLE_DEVICES" - if [ "\$infraDryRun" = "true" ]; then - export MASTER_ADDR=\$(scontrol show hostnames "\$SLURM_JOB_NODELIST" | head -n 1) - export MASTER_PORT=\$((20000 + SLURM_JOB_ID % 20000)) - fi - ${srunPrologue} """.replaceAll("(?m)^\\s*", "") @@ -2586,7 +2562,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG // CBTS Layer 2.5: rename narrowed stages (reuse-safety) and resize their splits to k. def cbtsResizeSplits(configs) { - def cbts = isInfraDryRun() ? null : testFilter[(CBTS_RESULT)] + def cbts = testFilter[(CBTS_RESULT)] if (cbts == null || !cbts.cbts_test_db_artifact_path) { return configs } @@ -2616,7 +2592,7 @@ def cbtsResizeSplits(configs) { // CBTS Layer 2: replace the normal stage set with the selector's affected // stages while retaining the baseline sanity and multi-GPU gates. def filterCbtsStageJobs(parallelJobs, parallelJobsFiltered, multiGpuJobs, testFilter) { - def cbts = isInfraDryRun() ? null : testFilter[(CBTS_RESULT)] + def cbts = testFilter[(CBTS_RESULT)] if (cbts == null) { return parallelJobsFiltered } @@ -3757,7 +3733,6 @@ def runInfraDryRunInPreparedWorkspace(pipeline, String llmSrc, String stageName) def outputPath = "${WORKSPACE}/${stageName}" def waivesFile = "${llmSrc}/infra_dry_run_waives.txt" def coverageConfigFile = "${llmSrc}/infra_dry_run.coveragerc" - def benchmarkPath = "${llmSrc}/tests/integration/defs/${INFRA_DRY_RUN_BENCHMARK}" sh "rm -rf ${outputPath} && mkdir -p ${outputPath} && : > ${waivesFile} && : > ${coverageConfigFile}" def testDBList = renderTestDB( @@ -3772,8 +3747,6 @@ def runInfraDryRunInPreparedWorkspace(pipeline, String llmSrc, String stageName) 1, 1, false, - "", - benchmarkPath, ) if (preprocessedLists.regularCount < 1) { error "No infrastructure dry-run benchmark was selected for ${stageName}" @@ -3805,18 +3778,19 @@ def runInfraDryRunInPreparedWorkspace(pipeline, String llmSrc, String stageName) ) pytestCommand += [ "--test-list=${preprocessedLists.regular}", - benchmarkPath, ] - withCredentials([ - string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), - string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), - string(credentialsId: 'llm_evaltool_repo_url', variable: 'EVALTOOL_REPO_URL') - ]) { - sh """ - cd ${llmSrc}/tests/integration/defs && \ - ${pytestCommand.join(" ")} - """ + withEnv(["stageName=${stageName}"]) { + withCredentials([ + string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), + string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), + string(credentialsId: 'llm_evaltool_repo_url', variable: 'EVALTOOL_REPO_URL') + ]) { + sh """ + cd ${llmSrc}/tests/integration/defs && \ + ${pytestCommand.join(" ")} + """ + } } } @@ -4183,7 +4157,7 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu // If the download or extraction fails we swallow the error: the override // directory will be absent below, the overrideYaml check will fail, and // renderTestDB falls back to the source test-db. - def cbts = isInfraDryRun() ? null : testFilter[(CBTS_RESULT)] + def cbts = testFilter[(CBTS_RESULT)] if (cbts != null && cbts.test_db_dir_override && cbts.cbts_test_db_artifact_path) { try { // Always re-fetch: a reused workspace may hold a stale cbts_test_db/ shadowing this build's YAMLs. @@ -4944,9 +4918,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def effectiveSplitId = infraDryRun ? 1 : splitId def effectiveSplits = infraDryRun ? 1 : splits def effectivePerfMode = infraDryRun ? false : perfMode - def benchmarkPath = infraDryRun - ? "${llmSrc}/tests/integration/defs/${INFRA_DRY_RUN_BENCHMARK}" - : "" // When useClusterDurations is set, use a per-cluster durations file keyed on // partition.clusterName (e.g. "oci-hsg", "dlcluster"). This lets each cluster @@ -4988,7 +4959,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO effectiveSplits, effectivePerfMode, clusterDurationsPath, - benchmarkPath, ) // Test Coverage @@ -5023,9 +4993,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO // Temporarily disable to reduce the log size // sh 'if [ "$(id -u)" -eq 0 ]; then dmesg -C || true; fi' def extraArgs = [*clusterDurationsArgs] - if (infraDryRun) { - extraArgs += [benchmarkPath] - } if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { extraArgs += [ "--capture=fd", @@ -5058,24 +5025,17 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO containerLD_LIBRARY_PATH = "${containerPIP_LLM_LIB_PATH}:${containerLD_LIBRARY_PATH}" } containerLD_LIBRARY_PATH = containerLD_LIBRARY_PATH.replaceAll(':+$', '') - withEnv(["LD_LIBRARY_PATH=${containerLD_LIBRARY_PATH}"]) { + def testEnvironment = ["LD_LIBRARY_PATH=${containerLD_LIBRARY_PATH}"] + if (infraDryRun) { + testEnvironment += ["stageName=${stageName}"] + } + withEnv(testEnvironment) { withCredentials([ string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), string(credentialsId: 'llm_evaltool_repo_url', variable: 'EVALTOOL_REPO_URL') ]) { sh "env | sort" - if (infraDryRun) { - if (preprocessedLists.regularCount < 1) { - error "No infrastructure dry-run benchmark was selected for ${stageName}" - } - sh """ - rm -rf ${stageName}/ && \ - cd ${llmSrc}/tests/integration/defs && \ - ${pytestCommand.join(" ")} - """ - return - } try { try { if (preprocessedLists.regularCount > 0) { @@ -5150,10 +5110,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } } - if (isInfraDryRun()) { - return - } - // Generate comprehensive rerun report if any reruns occurred stage ("Generate Report") { timeout(time: 15, unit: 'MINUTES'){ @@ -5169,7 +5125,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO error "Some tests terminated unexpectedly, please check the test report." } - if (perfMode) { + if (effectivePerfMode) { // Only PyTorch perf stages remain; the TensorRT perf baseline was removed. basePerfFilename = "base_perf_pytorch.csv" basePerfPath = "${llmSrc}/tests/integration/defs/perf/${basePerfFilename}" diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index f858a74edef5..6097eca724ed 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -70,21 +70,6 @@ env | sort echo "Full Command: $pytestCommand" -if [[ "${infraDryRun:-false}" == "true" ]]; then - if [[ "${SLURM_JOB_NUM_NODES:-1}" -gt 1 ]]; then - export RANK="$SLURM_PROCID" - export LOCAL_RANK="$SLURM_LOCALID" - export WORLD_SIZE="$SLURM_NTASKS" - export MASTER_ADDR="${MASTER_ADDR:?MASTER_ADDR must be set by the Slurm launch script}" - export MASTER_PORT="${MASTER_PORT:?MASTER_PORT must be set by the Slurm launch script}" - else - # A single-node dry run is one pytest controller which spawns local - # workers. Do not let ambient launcher variables select the external - # multi-node rank path. - unset RANK LOCAL_RANK WORLD_SIZE - fi -fi - # For single-node test runs or disaggregated benchmark/server runs, clear all # environment variables related to Slurm and MPI. This prevents test processes # (e.g., pytest) from incorrectly initializing MPI when running under a diff --git a/scripts/check_test_list.py b/scripts/check_test_list.py index 5206c1de3d50..2a5a44367052 100755 --- a/scripts/check_test_list.py +++ b/scripts/check_test_list.py @@ -46,8 +46,6 @@ # AST validation defaults _DEFAULT_TEST_LISTS_DIR = "tests/integration/test_lists" _DEFAULT_TEST_BASE_DIR = "tests/integration/defs" -# Preserve pytest's defaults while collecting the intentionally nonstandard dry-run module. -_L0_PYTEST_FILE_PATTERNS = "test_*.py *_test.py infra_dry_run_benchmark.py" # Paths whose tests are generated dynamically — skip AST validation _EXCLUDED_PATH_PREFIXES = ("perf/", ) @@ -969,8 +967,7 @@ def verify_l0_test_lists(llm_src): subprocess.run( f"cd {llm_src}/tests/integration/defs && " - f"pytest -o \"python_files={_L0_PYTEST_FILE_PATTERNS}\" " - f"--test-list={test_list} --output-dir={llm_src} -s --co -q", + f"pytest --test-list={test_list} --output-dir={llm_src} -s --co -q", shell=True, check=True) @@ -1043,7 +1040,7 @@ def check_waive_duplicates(llm_src): f.write( f" Occurrence {i} at line {line_no}: '{original_line}'\n" ) - f.write("\n") + f.write(f"\n") def verify_waive_list(llm_src, args): diff --git a/tests/integration/defs/infra_dry_run_benchmark.py b/tests/integration/defs/infra_dry_run_benchmark.py deleted file mode 100644 index 1a25d4e98d1b..000000000000 --- a/tests/integration/defs/infra_dry_run_benchmark.py +++ /dev/null @@ -1,347 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Small, model-free infrastructure benchmark used only by CI dry runs. - -The filename intentionally does not match pytest's normal ``test_*.py`` -pattern. Dry-run jobs pass this module explicitly after selecting its node ID -through the dedicated ``infra_dry_run`` test-db context. -""" - -from __future__ import annotations - -import importlib.util -import os -import signal -import socket -import sys -import threading -from contextlib import contextmanager -from datetime import timedelta -from pathlib import Path -from typing import Mapping, Optional - -import torch - -_DISTRIBUTED_TIMEOUT_SECONDS = 900 -_MATRIX_SIZE = 32 - - -@contextmanager -def _bounded_wait(seconds: int): - if threading.current_thread() is not threading.main_thread(): - raise RuntimeError("distributed dry-run timeout requires the main thread") - - previous_timer = signal.getitimer(signal.ITIMER_REAL) - if previous_timer != (0.0, 0.0): - raise RuntimeError("distributed dry-run timeout cannot replace an active ITIMER_REAL") - - def raise_timeout(_signum, _frame): - raise TimeoutError(f"distributed dry-run operation exceeded {seconds} seconds") - - previous_handler = signal.getsignal(signal.SIGALRM) - signal.signal(signal.SIGALRM, raise_timeout) - try: - signal.setitimer(signal.ITIMER_REAL, seconds) - try: - yield - finally: - signal.setitimer(signal.ITIMER_REAL, 0) - finally: - signal.signal(signal.SIGALRM, previous_handler) - - -def _required_int(environ: Mapping[str, str], name: str) -> int: - value = environ.get(name) - if value is None: - raise RuntimeError(f"{name} must be set for an externally launched rank") - try: - return int(value) - except ValueError as error: - raise RuntimeError(f"{name} must be an integer, got {value!r}") from error - - -def _external_rank_context( - environ: Mapping[str, str], -) -> Optional[tuple[int, int, int]]: - rank_names = ("RANK", "LOCAL_RANK", "WORLD_SIZE") - present = [name in environ for name in rank_names] - if not any(present): - return None - if not all(present): - missing = [name for name, is_present in zip(rank_names, present) if not is_present] - raise RuntimeError(f"incomplete distributed rank environment; missing {missing}") - - rank = _required_int(environ, "RANK") - local_rank = _required_int(environ, "LOCAL_RANK") - world_size = _required_int(environ, "WORLD_SIZE") - if world_size < 1: - raise RuntimeError(f"WORLD_SIZE must be positive, got {world_size}") - if not 0 <= rank < world_size: - raise RuntimeError(f"RANK {rank} is outside WORLD_SIZE {world_size}") - if local_rank < 0: - raise RuntimeError(f"LOCAL_RANK must be non-negative, got {local_rank}") - return rank, local_rank, world_size - - -def _run_cpu(torch_module=torch) -> None: - torch_module.manual_seed(0) - left = torch_module.full( - (_MATRIX_SIZE, _MATRIX_SIZE), 0.25, dtype=torch_module.float32, device="cpu" - ) - right = torch_module.full( - (_MATRIX_SIZE, _MATRIX_SIZE), 0.5, dtype=torch_module.float32, device="cpu" - ) - output = torch_module.matmul(left, right) - expected = torch_module.full_like(output, _MATRIX_SIZE * 0.25 * 0.5) - if output.device.type != "cpu" or output.dtype != torch_module.float32: - raise RuntimeError("CPU benchmark did not produce a CPU FP32 tensor") - if not torch_module.isfinite(output).all().item(): - raise RuntimeError("CPU benchmark produced non-finite values") - if not torch_module.equal(output, expected): - raise RuntimeError("CPU benchmark produced an unexpected deterministic result") - - -def _run_cuda_matmul(local_rank: int, torch_module=torch) -> float: - if not torch_module.cuda.is_available(): - raise RuntimeError("CUDA is required for this infrastructure dry-run stage") - device_count = torch_module.cuda.device_count() - if not 0 <= local_rank < device_count: - raise RuntimeError( - f"LOCAL_RANK {local_rank} is outside the {device_count} visible CUDA devices" - ) - - torch_module.cuda.set_device(local_rank) - device = torch_module.device("cuda", local_rank) - torch_module.manual_seed(1000 + local_rank) - torch_module.cuda.manual_seed_all(1000 + local_rank) - left = torch_module.full( - (_MATRIX_SIZE, _MATRIX_SIZE), 0.25, dtype=torch_module.float16, device=device - ) - right = torch_module.full( - (_MATRIX_SIZE, _MATRIX_SIZE), 0.5, dtype=torch_module.float16, device=device - ) - output = torch_module.matmul(left, right) - expected = torch_module.full_like(output, _MATRIX_SIZE * 0.25 * 0.5) - if output.device.type != "cuda" or output.dtype != torch_module.float16: - raise RuntimeError("GPU benchmark did not produce a CUDA FP16 tensor") - if not torch_module.isfinite(output).all().item(): - raise RuntimeError("GPU benchmark produced non-finite values") - if not torch_module.equal(output, expected): - raise RuntimeError("GPU benchmark produced an unexpected deterministic result") - torch_module.cuda.synchronize(device) - return float(output.float().sum().item()) - - -def _validate_rank_summaries(summaries: list[list[float]], world_size: int) -> None: - expected_ranks = list(range(world_size)) - observed_ranks = sorted(int(summary[0]) for summary in summaries) - if observed_ranks != expected_ranks: - raise RuntimeError( - f"observed ranks {observed_ranks} do not match expected {expected_ranks}" - ) - if any(int(summary[1]) != world_size for summary in summaries): - raise RuntimeError("rank summaries contain inconsistent world sizes") - checksums = [summary[2] for summary in summaries] - if any(abs(checksum - checksums[0]) > 1e-3 for checksum in checksums[1:]): - raise RuntimeError("rank summaries contain inconsistent CUDA checksums") - - -def _run_distributed_rank( - rank: int, - local_rank: int, - world_size: int, - timeout_seconds: int = _DISTRIBUTED_TIMEOUT_SECONDS, - torch_module=torch, -) -> list[float]: - distributed = torch_module.distributed - if not distributed.is_available() or not distributed.is_nccl_available(): - raise RuntimeError("NCCL distributed support is required for multi-GPU dry runs") - - try: - if not distributed.is_initialized(): - distributed.init_process_group( - backend="nccl", - init_method="env://", - rank=rank, - world_size=world_size, - timeout=timedelta(seconds=timeout_seconds), - ) - - checksum = _run_cuda_matmul(local_rank, torch_module) - device = torch_module.device("cuda", local_rank) - local_summary = torch_module.tensor( - [float(rank), float(world_size), checksum], - dtype=torch_module.float64, - device=device, - ) - reduced_checksum = torch_module.tensor(checksum, dtype=torch_module.float64, device=device) - distributed.all_reduce(reduced_checksum) - expected_total = checksum * world_size - if abs(float(reduced_checksum.item()) - expected_total) > 1e-3: - raise RuntimeError("NCCL all-reduce produced an unexpected checksum") - - gathered = [torch_module.empty_like(local_summary) for _ in range(world_size)] - distributed.all_gather(gathered, local_summary) - summaries = [summary.cpu().tolist() for summary in gathered] - _validate_rank_summaries(summaries, world_size) - return [float(rank), float(world_size), checksum] - finally: - if distributed.is_initialized(): - distributed.destroy_process_group() - - -def _llmapi_rank_task(timeout_seconds: int) -> list[float]: - """Run on every rank already owned by ``trtllm-llmapi-launch``.""" - from mpi4py import MPI - - rank_context = _external_rank_context(os.environ) - if rank_context is None: - raise RuntimeError("LLMAPI worker is missing its distributed rank environment") - env_rank, local_rank, env_world_size = rank_context - rank = MPI.COMM_WORLD.Get_rank() - world_size = MPI.COMM_WORLD.Get_size() - if (env_rank, env_world_size) != (rank, world_size): - raise RuntimeError( - "LLMAPI worker rank environment does not match its MPI communicator: " - f"env=({env_rank}, {env_world_size}), mpi=({rank}, {world_size})" - ) - return _run_distributed_rank(rank, local_rank, world_size, timeout_seconds) - - -def _worker_import_module(): - """Load this file under the top-level name visible from the worker cwd. - - Pytest may collect this file as ``defs.infra_dry_run_benchmark``, while the - MGMN workers run from this file's directory and can import it only as - ``infra_dry_run_benchmark``. Both RemoteMpiCommSession and multiprocessing - serialize callables, so worker functions must come from that deterministic - top-level module and its directory must be inherited by spawn children. - """ - module_path = Path(__file__).resolve() - module_dir = str(module_path.parent) - if module_dir not in sys.path: - sys.path.insert(0, module_dir) - - module_name = Path(__file__).stem - module = sys.modules.get(module_name) - if module is None: - spec = importlib.util.spec_from_file_location(module_name, __file__) - if spec is None or spec.loader is None: - raise RuntimeError(f"cannot load worker task module from {__file__}") - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - try: - spec.loader.exec_module(module) - except BaseException: - sys.modules.pop(module_name, None) - raise - else: - module_file = getattr(module, "__file__", None) - if module_file is None or Path(module_file).resolve() != module_path: - raise RuntimeError(f"{module_name} resolves to {module_file}, expected {__file__}") - return module - - -def _pickleable_llmapi_rank_task(): - return _worker_import_module()._llmapi_rank_task - - -def _run_with_existing_llmapi_launcher( - world_size: int, - timeout_seconds: int = _DISTRIBUTED_TIMEOUT_SECONDS, - session_factory=None, -) -> None: - if session_factory is None: - from tensorrt_llm.executor.utils import create_mpi_comm_session - - session_factory = create_mpi_comm_session - - session = session_factory(world_size) - try: - with _bounded_wait(timeout_seconds + 60): - summaries = session.submit_sync(_pickleable_llmapi_rank_task(), timeout_seconds) - if isinstance(summaries, BaseException): - raise RuntimeError("LLMAPI rank task failed") from summaries - if not isinstance(summaries, list) or len(summaries) != world_size: - raise RuntimeError( - "LLMAPI launcher returned an incomplete rank result set: " - f"expected {world_size}, got {summaries!r}" - ) - _validate_rank_summaries(summaries, world_size) - finally: - # RemoteMpiCommSessionClient.shutdown() is intentionally a no-op. The - # outer launcher owns and stops the MGMN server after pytest exits. - session.shutdown() - - -def _reserve_local_port() -> int: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("127.0.0.1", 0)) - return int(sock.getsockname()[1]) - - -def _local_rank_worker( - local_rank: int, world_size: int, master_port: int, timeout_seconds: int -) -> None: - os.environ.update( - { - "MASTER_ADDR": "127.0.0.1", - "MASTER_PORT": str(master_port), - "RANK": str(local_rank), - "LOCAL_RANK": str(local_rank), - "WORLD_SIZE": str(world_size), - } - ) - _run_distributed_rank(local_rank, local_rank, world_size, timeout_seconds) - - -def test_infra_dry_run_benchmark() -> None: - """Exercise CPU or every assigned GPU without downloading external data.""" - stage_name = os.environ.get("stageName", "") - if stage_name.startswith("CPU-"): - _run_cpu() - return - - rank_context = _external_rank_context(os.environ) - if rank_context is not None: - rank, local_rank, world_size = rank_context - if world_size == 1: - _run_cuda_matmul(local_rank) - elif os.environ.get("TLLM_SPAWN_PROXY_PROCESS") == "1": - if rank != 0: - raise RuntimeError( - "only LLMAPI rank 0 may run the infrastructure pytest controller" - ) - _run_with_existing_llmapi_launcher(world_size) - else: - _run_distributed_rank(rank, local_rank, world_size) - return - - if not torch.cuda.is_available(): - raise RuntimeError("CUDA is required for this infrastructure dry-run stage") - device_count = torch.cuda.device_count() - if device_count < 1: - raise RuntimeError("no CUDA devices are visible to the infrastructure dry run") - if device_count == 1: - _run_cuda_matmul(0) - return - - torch.multiprocessing.spawn( - _worker_import_module()._local_rank_worker, - args=(device_count, _reserve_local_port(), _DISTRIBUTED_TIMEOUT_SECONDS), - nprocs=device_count, - join=True, - ) diff --git a/tests/integration/defs/test_infra_dry_run_benchmark.py b/tests/integration/defs/test_infra_dry_run_benchmark.py new file mode 100644 index 000000000000..e9698841fad7 --- /dev/null +++ b/tests/integration/defs/test_infra_dry_run_benchmark.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Small, model-free benchmark for the infrastructure dry-run test context.""" + +from __future__ import annotations + +import os + +import torch + +_MATRIX_SIZE = 32 + + +def _validate_matmul(device: torch.device, dtype: torch.dtype) -> None: + left = torch.full((_MATRIX_SIZE, _MATRIX_SIZE), 0.25, dtype=dtype, device=device) + right = torch.full((_MATRIX_SIZE, _MATRIX_SIZE), 0.5, dtype=dtype, device=device) + output = torch.matmul(left, right) + expected = torch.full_like(output, _MATRIX_SIZE * 0.25 * 0.5) + assert output.device.type == device.type + assert output.dtype == dtype + assert torch.isfinite(output).all().item() + assert torch.equal(output, expected) + + +def _run_cpu() -> None: + _validate_matmul(torch.device("cpu"), torch.float32) + + +def _run_cuda() -> None: + assert torch.cuda.is_available(), "CUDA is required for this infrastructure dry-run stage" + device_count = torch.cuda.device_count() + assert device_count > 0, "no CUDA devices are visible to the infrastructure dry run" + for device_index in range(device_count): + device = torch.device("cuda", device_index) + torch.cuda.set_device(device) + _validate_matmul(device, torch.float16) + torch.cuda.synchronize(device) + + +def test_infra_dry_run_benchmark() -> None: + """Exercise the CPU or every CUDA device visible to the pytest runner.""" + if os.environ.get("stageName", "").startswith("CPU-"): + _run_cpu() + else: + _run_cuda() diff --git a/tests/integration/test_lists/test-db/infra_dry_run.yml b/tests/integration/test_lists/test-db/infra_dry_run.yml index f51d5a907fb0..ab73d456bb79 100644 --- a/tests/integration/test_lists/test-db/infra_dry_run.yml +++ b/tests/integration/test_lists/test-db/infra_dry_run.yml @@ -6,4 +6,4 @@ infra_dry_run: gte: 0 lte: 1024 tests: - - infra_dry_run_benchmark.py::test_infra_dry_run_benchmark + - test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index e912f7dcfd39..223ac1e84686 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -14,165 +14,125 @@ # limitations under the License. import os -import re import subprocess import tempfile import unittest from pathlib import Path -REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent -GROOVY = (REPO_ROOT / "jenkins" / "L0_Test.groovy").read_text() -PARENT_GROOVY = (REPO_ROOT / "jenkins" / "L0_MergeRequest.groovy").read_text() +REPO_ROOT = Path(__file__).resolve().parents[3] +L0_TEST = (REPO_ROOT / "jenkins" / "L0_Test.groovy").read_text() +L0_PARENT = (REPO_ROOT / "jenkins" / "L0_MergeRequest.groovy").read_text() SLURM_RUN = (REPO_ROOT / "jenkins" / "scripts" / "slurm_run.sh").read_text() -LLMAPI_LAUNCHER = (REPO_ROOT / "tensorrt_llm" / "llmapi" / "trtllm-llmapi-launch").read_text() -EXECUTOR_UTILS = (REPO_ROOT / "tensorrt_llm" / "executor" / "utils.py").read_text() -MPI_SESSION = (REPO_ROOT / "tensorrt_llm" / "llmapi" / "mpi_session.py").read_text() SLURM_INSTALL_PATH = REPO_ROOT / "jenkins" / "scripts" / "slurm_install.sh" -BENCHMARK_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "infra_dry_run_benchmark.py" -BENCHMARK = BENCHMARK_PATH.read_text() +CHECK_TEST_LIST = (REPO_ROOT / "scripts" / "check_test_list.py").read_text() +BENCHMARK_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "test_infra_dry_run_benchmark.py" DRY_RUN_DB_PATH = ( REPO_ROOT / "tests" / "integration" / "test_lists" / "test-db" / "infra_dry_run.yml" ) -CHECK_TEST_LIST = (REPO_ROOT / "scripts" / "check_test_list.py").read_text() def _function_body(source, name, next_name): start = source.index(f"def {name}") - end = source.index(f"def {next_name}", start + len(f"def {name}")) - return source[start:end] - - -def _map_keys(source, assignment_index): - start = source.index("[", assignment_index) - line_start = source.rindex("\n", 0, assignment_index) + 1 - indentation = source[line_start:assignment_index] - end = source.index(f"\n{indentation}]", start) - return set(re.findall(r"""['"]([^'"]+)['"]\s*:""", source[start:end])) + return source[start : source.index(f"def {next_name}", start + len(name))] class InfraDryRunPipelineTest(unittest.TestCase): - def test_dry_run_is_a_test_db_selected_positional_pytest_module(self): - process = _function_body(GROOVY, "processShardTestList", "isValidSlurmJobId") - platform = _function_body( - GROOVY, "runLLMTestlistOnPlatformImpl", "runLLMTestlistOnPlatform" - ) + def test_dedicated_context_selects_one_standard_pytest_case(self): + database = DRY_RUN_DB_PATH.read_text() self.assertTrue(BENCHMARK_PATH.is_file()) - self.assertFalse(BENCHMARK_PATH.name.startswith("test_")) - self.assertEqual(DRY_RUN_DB_PATH.read_text().splitlines()[1], "infra_dry_run:") + self.assertTrue(BENCHMARK_PATH.name.startswith("test_")) + self.assertEqual(database.count("::test_"), 1) + self.assertIn("infra_dry_run:", database) + self.assertIn("test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark", database) + self.assertNotIn("infra_dry_run_benchmark.py", CHECK_TEST_LIST) + verify_l0 = _function_body(CHECK_TEST_LIST, "verify_l0_test_lists", "verify_qa_test_lists") + self.assertIn("pytest --test-list={test_list}", verify_l0) + + def test_platform_runner_uses_standard_pytest_results_and_reporting(self): + body = _function_body(L0_TEST, "runLLMTestlistOnPlatformImpl", "runLLMTestlistOnPlatform") self.assertIn( - "infra_dry_run_benchmark.py::test_infra_dry_run_benchmark", - DRY_RUN_DB_PATH.read_text(), - ) - l0_validation = _function_body( - CHECK_TEST_LIST, "verify_l0_test_lists", "verify_qa_test_lists" + "effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList", body ) - self.assertIn('pytest -o \\"python_files={_L0_PYTEST_FILE_PATTERNS}\\"', l0_validation) - self.assertIn('positionalTest=""', process) - self.assertIn("if (positionalTest)", process) - self.assertIn("testListCmd += [positionalTest]", process) + self.assertIn("effectiveSplitId = infraDryRun ? 1 : splitId", body) + self.assertIn("effectiveSplits = infraDryRun ? 1 : splits", body) + self.assertIn("effectivePerfMode = infraDryRun ? false : perfMode", body) + self.assertIn("getPytestBaseCommandLine(", body) + self.assertIn("--test-list=${preprocessedLists.regular}", body) + self.assertIn("rerunFailedTests(", body) + self.assertIn("runIsolatedTests(", body) + self.assertIn("generateRerunReport(", body) + self.assertIn('testEnvironment += ["stageName=${stageName}"]', body) + self.assertNotIn("test_infra_dry_run_benchmark.py", body) + self.assertNotIn("positionalTest", L0_TEST) + + def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): + prepared = _function_body(L0_TEST, "runInfraDryRunInPreparedWorkspace", "runLLMDocBuild") + docs = _function_body(L0_TEST, "runLLMDocBuild", "launchTestListCheck") + for call in ("renderTestDB(", "processShardTestList(", "getPytestBaseCommandLine("): + self.assertIn(call, prepared) + self.assertIn("--test-list=${preprocessedLists.regular}", prepared) + self.assertIn('withEnv(["stageName=${stageName}"])', prepared) + self.assertNotIn("test_infra_dry_run_benchmark.py", prepared) + self.assertLess(docs.index("if (isInfraDryRun())"), docs.index("make html")) + + def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self): + body = _function_body(L0_TEST, "runLLMTestlistWithSbatch", "runLLMTestlistOnSlurm") self.assertIn( - "effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList", platform + "effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList", body ) - self.assertIn("extraArgs += [benchmarkPath]", platform) - self.assertIn("--test-list=${preprocessedLists.regular}", platform) - self.assertIn("rerunFailedTests(", platform) - self.assertIn("runIsolatedTests(", platform) - self.assertNotIn("getInfraDryRunDirectCommand", GROOVY) - self.assertIn("create_mpi_comm_session", BENCHMARK) - self.assertIn("session.submit_sync(", BENCHMARK) - self.assertIn("_pickleable_llmapi_rank_task(), timeout_seconds", BENCHMARK) - self.assertNotIn("torchrun", BENCHMARK) - self.assertNotIn("subprocess", BENCHMARK) - self.assertFalse( - (REPO_ROOT / "jenkins" / "scripts" / "infra_dry_run_benchmark.py").exists() + self.assertIn("String[] taskArgs = getNodeArgs(", body) + self.assertIn('pytestUtil = "$llmSrcNode/tensorrt_llm/llmapi/trtllm-llmapi-launch"', body) + self.assertIn("if (!isInfraDryRun() && (disaggMultiNodeMode || aggMultiNodeMode))", body) + self.assertNotIn("test_infra_dry_run_benchmark.py", body) + self.assertNotIn("MASTER_ADDR", body) + self.assertNotIn("MASTER_PORT", body) + dispatch = _function_body(L0_TEST, "runLLMTestlistOnSlurm", "INFRA_DRY_RUN") + self.assertIn("if (isInfraDryRun() || nodeCount > 1 || runWithSbatch)", dispatch) + self.assertIn( + 'if [[ "${infraDryRun:-false}" == "true" || "$stageName" != *Disagg* ]]', + SLURM_RUN, ) + for rank_variable in ("RANK=", "LOCAL_RANK=", "WORLD_SIZE=", "MASTER_ADDR"): + self.assertNotIn(rank_variable, SLURM_RUN) - def test_docs_dry_run_bypasses_normal_doc_build_and_keeps_results(self): - prepared = _function_body(GROOVY, "runInfraDryRunInPreparedWorkspace", "runLLMDocBuild") - body = _function_body(GROOVY, "runLLMDocBuild", "launchTestListCheck") - dry_guard = body.index("if (isInfraDryRun())") - benchmark = body.index("runInfraDryRunInPreparedWorkspace(", dry_guard) - early_return = body.index("return", benchmark) - sphinx = body.index("make html") - self.assertLess(dry_guard, benchmark) - self.assertLess(benchmark, early_return) - self.assertLess(early_return, sphinx) - self.assertIn("renderTestDB(", prepared) - self.assertIn("processShardTestList(", prepared) - self.assertIn("getPytestBaseCommandLine(", prepared) - self.assertIn("withCredentials([", prepared) - self.assertIn("benchmarkPath", prepared) - - doc_jobs = GROOVY[ - GROOVY.index("docBuildConfigs = [") : GROOVY.index("// Python version and OS") + def test_parent_uses_parameter_only_and_explicit_non_fail_fast_helper(self): + setup = L0_PARENT[ + L0_PARENT.index("boolean infraDryRun =") : L0_PARENT.index("String reuseBuild =") ] - self.assertIn('runLLMDocBuild(pipeline, VANILLA_CONFIG, "A10-Build_Docs")', doc_jobs) - self.assertIn("{}, !isInfraDryRun(), attemptTag", doc_jobs) - - def test_package_sanity_uses_the_shared_platform_pytest_path(self): - package_jobs = GROOVY[ - GROOVY.index("sanityCheckJobs =") : GROOVY.index( - "multiGpuJobs =", GROOVY.index("sanityCheckJobs =") + helper = _function_body(L0_PARENT, "launchInfraDryRunTestJob", "launchStages") + launch = _function_body(L0_PARENT, "launchJob", "launchInfraDryRunTestJob") + self.assertIn("params.InfraDryRun?.toString()?.toBoolean()", setup) + self.assertIn("(INFRA_DRY_RUN): infraDryRun", setup) + self.assertNotIn("JOB_NAME", setup.splitlines()[0]) + self.assertIn('"L0_Test-${arch}-Single-GPU"', helper) + self.assertNotIn('"L0_Test-${arch}-Multi-GPU"', helper) + self.assertIn(", false, false, globalVars,", helper) + self.assertIn("'testPhase2StageName': ''", helper) + self.assertIn("additionalParameters.containsKey('testPhase2StageName')", launch) + self.assertIn("parallelJobs.failFast = enableFailFast", L0_PARENT) + + def test_normal_gating_and_result_collection_remain_in_place(self): + stages_start = L0_PARENT.index("def launchStages") + stages = L0_PARENT[stages_start : L0_PARENT.index("\npipeline {", stages_start)] + for arch in ("x86_64", "SBSA"): + normal_single = stages.index(f'testStageName = "[Test-{arch}-Single-GPU] Remote Run"') + approval = stages.index( + f'currentBuild.description?.contains("Require {arch} Multi-GPU Testing")', + normal_single, ) - ] - self.assertIn("runLLMTestlistOnPlatform(", package_jobs) - self.assertIn("toStageName(values[1], key)", package_jobs) - self.assertNotIn('"CPU-', package_jobs) + normal_multi = stages.index(f'launchJob(pipeline, "L0_Test-{arch}-Multi-GPU"', approval) + self.assertLess(normal_single, approval) + self.assertLess(approval, normal_multi) - def test_slurm_uses_standard_resources_and_pytest_command(self): - body = GROOVY[ - GROOVY.index("def runLLMTestlistWithSbatch") : GROOVY.index("def runLLMTestlistOnSlurm") - ] - self.assertIn("String[] taskArgs = getNodeArgs(", body) - self.assertNotIn("getInfraDryRunNodeArgs", GROOVY) + upload = _function_body(L0_TEST, "uploadResults", "runIsolatedTests") + self.assertNotIn("isInfraDryRun", upload) self.assertIn( - "effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList", body + 'junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml")', L0_TEST ) - self.assertIn("effectiveSplitId = infraDryRun ? 1 : splitId", body) - self.assertIn("effectiveSplits = infraDryRun ? 1 : splits", body) - self.assertIn("effectivePerfMode = infraDryRun ? false : perfMode", body) - self.assertIn("infra_dry_run_waives.txt", body) - self.assertIn("${INFRA_DRY_RUN_BENCHMARK}", body) - self.assertIn('pytestUtil = "$llmSrcNode/tensorrt_llm/llmapi/trtllm-llmapi-launch"', body) - self.assertIn("if(nodeCount > 1) {", body) - - def test_slurm_maps_ranks_and_uses_stable_rendezvous(self): - for assignment in ( - 'RANK="$SLURM_PROCID"', - 'LOCAL_RANK="$SLURM_LOCALID"', - 'WORLD_SIZE="$SLURM_NTASKS"', - 'MASTER_ADDR="${MASTER_ADDR:?MASTER_ADDR must be set by the Slurm launch script}"', - 'MASTER_PORT="${MASTER_PORT:?MASTER_PORT must be set by the Slurm launch script}"', - ): - self.assertIn(assignment, SLURM_RUN) - self.assertIn('scontrol show hostnames "\\$SLURM_JOB_NODELIST"', GROOVY) - self.assertIn("20000 + SLURM_JOB_ID % 20000", GROOVY) - self.assertIn("--container-env=MASTER_ADDR", GROOVY) - self.assertIn("--container-env=MASTER_PORT", GROOVY) - self.assertLess( - SLURM_RUN.index('if [[ "${infraDryRun:-false}" == "true"'), - SLURM_RUN.index("eval $pytestCommand"), - ) - self.assertNotIn("infra_dry_run_benchmark.py", SLURM_RUN) - self.assertNotIn("exit $?", SLURM_RUN) - self.assertIn("export TLLM_SPAWN_PROXY_PROCESS=1", LLMAPI_LAUNCHER) - self.assertIn('if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]', LLMAPI_LAUNCHER) - self.assertIn("python3 -m tensorrt_llm.llmapi.mgmn_worker_node", LLMAPI_LAUNCHER) - self.assertIn("unset RANK LOCAL_RANK WORLD_SIZE", SLURM_RUN) - - def test_llmapi_session_contract_matches_the_benchmark_adapter(self): - create_session = _function_body(EXECUTOR_UTILS, "create_mpi_comm_session", "has_event_loop") - remote_session = MPI_SESSION[ - MPI_SESSION.index("class RemoteMpiCommSessionClient") : MPI_SESSION.index( - "class RemoteMpiCommSessionServer" - ) - ] - self.assertIn("n_workers: int", create_session) - self.assertIn("RemoteMpiCommSessionClient(", create_session) - self.assertIn("def submit_sync(self, task, *args, **kwargs) -> List[T]", remote_session) - self.assertIn("return res", remote_session) - self.assertIn("pickle.dumps(obj)", (REPO_ROOT / "tensorrt_llm/executor/ipc.py").read_text()) - self.assertIn("_pickleable_llmapi_rank_task()", BENCHMARK) + always_start = L0_PARENT.index(" always {") + always_block = L0_PARENT[always_start : L0_PARENT.index(" stages {", always_start)] + self.assertIn("collectTestResults(this, testFilter, globalVars)", always_block) + self.assertNotIn("testFilter[INFRA_DRY_RUN]", always_block) def test_slurm_artifact_download_replaces_existing_archive(self): with tempfile.TemporaryDirectory() as temp_dir: @@ -185,25 +145,14 @@ def test_slurm_artifact_download_replaces_existing_archive(self): script = r""" source "$SLURM_INSTALL_PATH" retry_command() { - if [[ "$1" == "--timeout" ]]; then - shift 2 - fi + if [[ "$1" == "--timeout" ]]; then shift 2; fi "$@" } wget() { local output_path="" while (( "$#" )); do - if [[ "$1" == "-O" ]]; then - output_path="$2" - shift 2 - else - shift - fi + if [[ "$1" == "-O" ]]; then output_path="$2"; shift 2; else shift; fi done - if [[ -z "$output_path" ]]; then - output_path="$resourcePathNode/$tarName" - [[ ! -e "$output_path" ]] || output_path="${output_path}.1" - fi printf 'fresh\n' > "$output_path" printf '%s\n' "$output_path" > "$WGET_RECORD_PATH" } @@ -238,11 +187,7 @@ def test_slurm_artifact_download_replaces_existing_archive(self): "TAR_RECORD_PATH": str(tar_record_path), } subprocess.run( - ["bash", "-c", script], - check=True, - capture_output=True, - text=True, - env=env, + ["bash", "-c", script], check=True, capture_output=True, text=True, env=env ) expected_tmp = f"{archive_path}.tmp.123.0" @@ -252,223 +197,6 @@ def test_slurm_artifact_download_replaces_existing_archive(self): self.assertFalse(Path(f"{archive_path}.1").exists()) self.assertFalse(Path(expected_tmp).exists()) - def test_dry_pytest_failure_propagates_without_rerun_or_isolation(self): - body = _function_body( - GROOVY, - "runLLMTestlistOnPlatformImpl", - "runLLMTestlistOnPlatform", - ) - command_area = body.index('withEnv(["LD_LIBRARY_PATH=') - branch_start = body.index("if (infraDryRun) {", command_area) - dry_branch = body[branch_start : body.index("try {", branch_start)] - self.assertIn('${pytestCommand.join(" ")}', dry_branch) - self.assertNotIn("rerunFailedTests", dry_branch) - self.assertNotIn("runIsolatedTests", dry_branch) - self.assertNotIn("catch", dry_branch) - - def test_standard_junit_is_used_and_cbts_is_disabled(self): - self.assertIn( - 'junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml")', - GROOVY, - ) - self.assertNotIn("results-infra_dry_run", GROOVY) - self.assertNotIn("infra_dry_run*.json", GROOVY) - cbts_body = _function_body(GROOVY, "isCbtsStage", "scpFromRemoteCmd") - self.assertIn("if (isInfraDryRun())", cbts_body) - self.assertIn("return false", cbts_body) - - def test_empty_single_gpu_filter_fails_only_for_dry_run(self): - single_branch_start = GROOVY.index("if (env.JOB_NAME ==~ /.*Single-GPU.*/)") - single_branch_end = GROOVY.index( - "} else if (env.JOB_NAME ==~ /.*Multi-GPU.*/)", - single_branch_start, - ) - single_branch = GROOVY[single_branch_start:single_branch_end] - dry_guard = single_branch.index("else if (isInfraDryRun())") - dry_error = single_branch.index( - 'error "Skip single-GPU testing. No test to run for infrastructure dry run."' - ) - normal_skip = single_branch.index('echo "Skip single-GPU testing. No test to run."') - self.assertLess(dry_guard, dry_error) - self.assertLess(dry_error, normal_skip) - - -class InfraDryRunParentPipelineTest(unittest.TestCase): - def test_parameter_is_propagated_to_the_helper_filter(self): - filter_setup = PARENT_GROOVY[ - PARENT_GROOVY.index("boolean infraDryRun =") : PARENT_GROOVY.index( - "String reuseBuild =" - ) - ] - self.assertIn("params.InfraDryRun?.toString()?.toBoolean()", filter_setup) - self.assertIn("(INFRA_DRY_RUN): infraDryRun", filter_setup) - - def test_dry_run_uses_one_combined_helper_without_inner_parallel(self): - body = _function_body( - PARENT_GROOVY, - "launchInfraDryRunTestJob", - "launchStages", - ) - launch_job = _function_body( - PARENT_GROOVY, - "launchJob", - "launchInfraDryRunTestJob", - ) - self.assertIn('"L0_Test-${arch}-Single-GPU"', body) - self.assertNotIn('"L0_Test-${arch}-Multi-GPU"', body) - self.assertIn(", false, false, globalVars,", body) - self.assertIn("'testFilter': testFilterJson", body) - self.assertIn("'testPhase2StageName': ''", body) - self.assertNotIn("pipeline.parallel", body) - self.assertIn( - "if (!additionalParameters.containsKey('testPhase2StageName') && " - "env.testPhase2StageName)", - launch_job, - ) - - selection = GROOVY[GROOVY.index("singleGpuJobs = parallelJobs") :] - phase2_guard = selection.index("if (testPhase2StageName)") - single_start = selection.index("if (env.JOB_NAME ==~ /.*Single-GPU.*/)") - single_end = selection.index("} else if (env.JOB_NAME ==~ /.*Multi-GPU.*/)") - self.assertLess(phase2_guard, selection.index("singleGpuJobs = parallelJobs.findAll")) - self.assertIn("dgxJobs = [:]", selection[:phase2_guard]) - self.assertIn( - "parallel singleGpuJobs", - selection[single_start:single_end], - ) - - def test_image_parameters_match_normal_jobs(self): - start = PARENT_GROOVY.index("def launchStages") - launch_stages = PARENT_GROOVY[start : PARENT_GROOVY.index("\npipeline {", start)] - expected_keys = { - "x86_64": { - "dockerImage", - "wheelDockerImagePy310", - "wheelDockerImagePy312", - }, - "SBSA": {"dockerImage", "wheelDockerImage"}, - } - for arch, expected in expected_keys.items(): - dry_call = launch_stages.index(f'launchInfraDryRunTestJob(pipeline, "{arch}"') - dry_map = launch_stages.rindex("def imageParameters = [", 0, dry_call) - normal_stage = launch_stages.index( - f'testStageName = "[Test-{arch}-Single-GPU] Remote Run"', - dry_call, - ) - normal_map = launch_stages.index( - "def additionalParameters = [", - normal_stage, - ) - self.assertEqual(_map_keys(launch_stages, dry_map), expected) - self.assertEqual( - _map_keys(launch_stages, normal_map) - {"testFilter"}, - expected, - ) - - def test_dry_run_branch_precedes_normal_single_gpu_gating(self): - start = PARENT_GROOVY.index("def launchStages") - launch_stages = PARENT_GROOVY[start : PARENT_GROOVY.index("\npipeline {", start)] - for arch in ("x86_64", "SBSA"): - dry_run_call = launch_stages.index(f'launchInfraDryRunTestJob(pipeline, "{arch}"') - build_call = launch_stages.rindex( - f'launchJob(pipeline, "/LLM/helpers/Build-{arch}"', - 0, - dry_run_call, - ) - normal_single = launch_stages.index( - f'testStageName = "[Test-{arch}-Single-GPU] Remote Run"', - dry_run_call, - ) - marker = launch_stages.index( - f'currentBuild.description?.contains("Require {arch} Multi-GPU Testing")', - normal_single, - ) - normal_multi = launch_stages.index( - f'launchJob(pipeline, "L0_Test-{arch}-Multi-GPU"', - marker, - ) - self.assertLess(build_call, dry_run_call) - self.assertLess(dry_run_call, normal_single) - self.assertLess(normal_single, marker) - self.assertLess(marker, normal_multi) - self.assertIn( - "parallelJobs.failFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast", - launch_stages, - ) - - def test_product_reporting_is_excluded_but_test_results_are_collected(self): - setup = _function_body( - PARENT_GROOVY, - "setupPipelineEnvironment", - "mergeWaiveList", - ) - self.assertLess( - setup.index("if (testFilter[INFRA_DRY_RUN])"), - setup.index("getCbtsResult("), - ) - always_start = PARENT_GROOVY.index(" always {") - always_block = PARENT_GROOVY[ - always_start : PARENT_GROOVY.index(" stages {", always_start) - ] - self.assertIn( - "if (!isReleaseCheckMode && !GEN_POST_MERGE_BUILDS_ONLY) {", - always_block, - ) - self.assertIn("collectTestResults(this, testFilter, globalVars)", always_block) - self.assertNotIn("testFilter[INFRA_DRY_RUN]", always_block) - self.assertNotIn("L0_Stability", PARENT_GROOVY) - - def test_dry_run_skips_changed_file_analysis(self): - setup = _function_body( - PARENT_GROOVY, - "setupPipelineEnvironment", - "mergeWaiveList", - ) - first_guard = setup.index("if (testFilter[INFRA_DRY_RUN])") - second_guard = setup.index("if (testFilter[INFRA_DRY_RUN])", first_guard + 1) - changed_file_block = setup[first_guard:second_guard] - normal_path = changed_file_block.index("} else {") - self.assertIn("Changed-file analysis is skipped", changed_file_block[:normal_path]) - self.assertIn("(MULTI_GPU_FILE_CHANGED)] = false", changed_file_block[:normal_path]) - self.assertIn('(ONLY_ONE_GROUP_CHANGED)] = ""', changed_file_block[:normal_path]) - self.assertIn("(AUTO_TRIGGER_TAG_LIST)] = []", changed_file_block[:normal_path]) - for call in ( - "getMultiGpuFileChanged(", - "getOnlyOneGroupChanged(", - "getAutoTriggerTagList(", - ): - self.assertGreater(changed_file_block.index(call), normal_path) - - def test_dry_run_skips_waive_merge_and_release_check(self): - preparation = _function_body(PARENT_GROOVY, "preparation", "launchReleaseCheck") - waive_stage = preparation[preparation.index('stage("Merge Test Waive List")') :] - waive_guard = waive_stage.index("if (testFilter[INFRA_DRY_RUN])") - waive_skip = waive_stage.index("Skipping Merge Test Waive List") - waive_normal = waive_stage.index("mergeWaiveList(") - self.assertLess(waive_guard, waive_skip) - self.assertLess(waive_skip, waive_normal) - - launch_stages_start = PARENT_GROOVY.index("def launchStages") - launch_stages = PARENT_GROOVY[ - launch_stages_start : PARENT_GROOVY.index("\npipeline {", launch_stages_start) - ] - release_branch = launch_stages[ - launch_stages.index('"Release-Check":') : launch_stages.index('"x86_64-Linux":') - ] - self.assertLess( - release_branch.index("if (testFilter[INFRA_DRY_RUN])"), - release_branch.index("launchReleaseCheck("), - ) - - release_mode = PARENT_GROOVY.index("if (isReleaseCheckMode)") - release_only = PARENT_GROOVY[ - release_mode : PARENT_GROOVY.index("launchStages(this", release_mode) - ] - self.assertLess( - release_only.index("if (testFilter[INFRA_DRY_RUN])"), - release_only.index("launchReleaseCheck("), - ) - if __name__ == "__main__": unittest.main() diff --git a/tests/unittest/tools/test_infra_dry_run_pytest.py b/tests/unittest/tools/test_infra_dry_run_pytest.py index faf1ae75e5e4..b4056a7ef278 100644 --- a/tests/unittest/tools/test_infra_dry_run_pytest.py +++ b/tests/unittest/tools/test_infra_dry_run_pytest.py @@ -15,110 +15,138 @@ import importlib.util import os -import pickle -import signal import subprocess import sys import tempfile import textwrap -import threading import types import unittest from pathlib import Path -from types import SimpleNamespace from unittest import mock -REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent -BENCHMARK_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "infra_dry_run_benchmark.py" +REPO_ROOT = Path(__file__).resolve().parents[3] +BENCHMARK_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "test_infra_dry_run_benchmark.py" -_IMPORT_TORCH = types.ModuleType("torch") -with mock.patch.dict(sys.modules, {"torch": _IMPORT_TORCH}): - # Pytest's default import mode treats defs/ as a package; exercise the - # package-qualified controller name rather than the worker's top-level name. - SPEC = importlib.util.spec_from_file_location("defs.infra_dry_run_benchmark", BENCHMARK_PATH) - BENCHMARK = importlib.util.module_from_spec(SPEC) - assert SPEC.loader is not None - SPEC.loader.exec_module(BENCHMARK) +_TORCH_IMPORT_STUB = types.ModuleType("torch") +with mock.patch.dict(sys.modules, {"torch": _TORCH_IMPORT_STUB}): + _SPEC = importlib.util.spec_from_file_location("test_infra_dry_run_benchmark", BENCHMARK_PATH) + assert _SPEC is not None and _SPEC.loader is not None + BENCHMARK = importlib.util.module_from_spec(_SPEC) + _SPEC.loader.exec_module(BENCHMARK) class _Scalar: def __init__(self, value): - self.value = value + self._value = value def item(self): - return self.value + return self._value class _Tensor: - def __init__(self, values, *, device="cpu", dtype="float32"): - self.values = values - self.device = SimpleNamespace(type=device) + def __init__(self, value, *, dtype, device): + self.value = value self.dtype = dtype + self.device = device def all(self): return _Scalar(True) - def cpu(self): - return self - def tolist(self): - return list(self.values) +class _Cuda: + def __init__(self, available=True, count=2): + self.available = available + self.count = count + self.selected = [] + self.synchronized = [] - def item(self): - return self.values + def is_available(self): + return self.available + + def device_count(self): + return self.count + + def set_device(self, device): + self.selected.append(device.index) + def synchronize(self, device): + self.synchronized.append(device.index) -class _CpuTorch: + +class _Torch: + float16 = "float16" float32 = "float32" - def __init__(self): - self.seed = None - self.matmul_calls = 0 + def __init__(self, *, cuda_available=True, cuda_count=2): + self.cuda = _Cuda(cuda_available, cuda_count) + self.devices = [] - def manual_seed(self, seed): - self.seed = seed + def device(self, device_type, index=None): + device = types.SimpleNamespace(type=device_type, index=index) + self.devices.append(device) + return device def full(self, _shape, value, *, dtype, device): - return _Tensor(value, device=device, dtype=dtype) + return _Tensor(value, dtype=dtype, device=device) def matmul(self, _left, _right): - self.matmul_calls += 1 - return _Tensor(4.0) + device = self.devices[-1] + dtype = self.float32 if device.type == "cpu" else self.float16 + return _Tensor(4.0, dtype=dtype, device=device) - def full_like(self, _tensor, value): - return _Tensor(value) + def full_like(self, tensor, value): + return _Tensor(value, dtype=tensor.dtype, device=tensor.device) def isfinite(self, tensor): return tensor def equal(self, left, right): - return left.values == right.values + return left.value == right.value + + +class InfraDryRunBenchmarkTest(unittest.TestCase): + def test_cpu_path_uses_fp32_cpu_matmul(self): + torch_stub = _Torch() + with mock.patch.object(BENCHMARK, "torch", torch_stub): + BENCHMARK._run_cpu() + self.assertEqual( + [(device.type, device.index) for device in torch_stub.devices], [("cpu", None)] + ) + def test_cuda_path_exercises_every_visible_device(self): + torch_stub = _Torch(cuda_count=3) + with mock.patch.object(BENCHMARK, "torch", torch_stub): + BENCHMARK._run_cuda() + self.assertEqual(torch_stub.cuda.selected, [0, 1, 2]) + self.assertEqual(torch_stub.cuda.synchronized, [0, 1, 2]) -class InfraDryRunPytestTest(unittest.TestCase): - def test_explicit_module_and_test_list_execute_but_normal_collection_ignores_it(self): + def test_cuda_path_does_not_fall_back_to_cpu(self): + with mock.patch.object(BENCHMARK, "torch", _Torch(cuda_available=False)): + with self.assertRaisesRegex(AssertionError, "CUDA is required"): + BENCHMARK._run_cuda() + + def test_standard_pytest_collection_selects_only_the_requested_context(self): with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) - benchmark = root / BENCHMARK_PATH.name - benchmark.write_text(BENCHMARK_PATH.read_text()) + (root / BENCHMARK_PATH.name).write_text(BENCHMARK_PATH.read_text()) + (root / "test_product.py").write_text("def test_product(): pass\n") (root / "torch.py").write_text( textwrap.dedent( """ float32 = "float32" - + class Device: + def __init__(self, kind, index=None): + self.type, self.index = kind, index + class Scalar: + def item(self): return True class Tensor: - def __init__(self, value, dtype="float32", device="cpu"): - self.value = value - self.dtype = dtype - self.device = type("Device", (), {"type": device})() - def all(self): return self - def item(self): return self.value - - def manual_seed(_seed): pass - def full(_shape, value, *, dtype, device): - return Tensor(value, dtype, device) - def matmul(_left, _right): return Tensor(4.0) - def full_like(_tensor, value): return Tensor(value) + def __init__(self, value, dtype, device): + self.value, self.dtype, self.device = value, dtype, device + def all(self): return Scalar() + def device(kind, index=None): return Device(kind, index) + def full(_shape, value, *, dtype, device): return Tensor(value, dtype, device) + def matmul(left, _right): return Tensor(4.0, left.dtype, left.device) + def full_like(tensor, value): return Tensor(value, tensor.dtype, tensor.device) def isfinite(tensor): return tensor def equal(left, right): return left.value == right.value """ @@ -129,19 +157,9 @@ def equal(left, right): return left.value == right.value """ def pytest_addoption(parser): parser.addoption("--test-list") - parser.addoption("--test-prefix") - def pytest_collection_modifyitems(config, items): - prefix = config.getoption("--test-prefix") - if prefix: - for item in items: - item._nodeid = f"{prefix}/{item.nodeid}" - test_list = config.getoption("--test-list") - if not test_list: - return wanted = { - f"{prefix}/{line.strip()}" if prefix else line.strip() - for line in open(test_list) + line.strip() for line in open(config.getoption("--test-list")) if line.strip() } selected = [item for item in items if item.nodeid in wanted] @@ -152,358 +170,26 @@ def pytest_collection_modifyitems(config, items): """ ) ) - test_list = root / "infra_dry_run.txt" - test_list.write_text("infra_dry_run_benchmark.py::test_infra_dry_run_benchmark\n") - (root / "test_normal.py").write_text("def test_normal(): pass\n") - validation_list = root / "all_l0.txt" - validation_list.write_text( - "infra_dry_run_benchmark.py::test_infra_dry_run_benchmark\n" - "test_normal.py::test_normal\n" - ) - env = {**os.environ, "stageName": "CPU-Validation"} - explicit = subprocess.run( - [ - sys.executable, - "-m", - "pytest", - "-q", - f"--test-list={test_list}", - "--test-prefix=CPU-Validation", - str(benchmark), - ], - cwd=root, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=False, - ) - validation = subprocess.run( - [ - sys.executable, - "-m", - "pytest", - "-q", - "--collect-only", - "-o", - "python_files=test_*.py *_test.py infra_dry_run_benchmark.py", - f"--test-list={validation_list}", - ], - cwd=root, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=False, - ) - normal = subprocess.run( - [sys.executable, "-m", "pytest", "--collect-only", "-q", str(root)], - cwd=root, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=False, - ) - - self.assertEqual(explicit.returncode, 0, explicit.stdout) - self.assertIn("1 passed", explicit.stdout) - self.assertEqual(validation.returncode, 0, validation.stdout) - self.assertIn("infra_dry_run_benchmark.py::test_infra_dry_run_benchmark", validation.stdout) - self.assertIn("test_normal.py::test_normal", validation.stdout) - self.assertIn("2 tests collected", validation.stdout) - self.assertEqual(normal.returncode, 0, normal.stdout) - self.assertIn("test_normal.py::test_normal", normal.stdout) - self.assertNotIn(BENCHMARK_PATH.name, normal.stdout) - - def test_bounded_wait_interrupts_and_restores_the_signal_handler(self): - previous_handler = signal.getsignal(signal.SIGALRM) - with self.assertRaisesRegex(TimeoutError, "exceeded 30 seconds"): - with BENCHMARK._bounded_wait(30): - signal.raise_signal(signal.SIGALRM) - self.assertIs(signal.getsignal(signal.SIGALRM), previous_handler) - - def test_bounded_wait_rejects_non_main_threads_and_existing_timers(self): - errors = [] - - def enter_wait(): - try: - with BENCHMARK._bounded_wait(30): - pass - except BaseException as error: - errors.append(error) - - thread = threading.Thread(target=enter_wait) - thread.start() - thread.join() - self.assertEqual(len(errors), 1) - self.assertRegex(str(errors[0]), "requires the main thread") - - with ( - mock.patch.object(BENCHMARK.signal, "getitimer", return_value=(1.0, 0.0)), - self.assertRaisesRegex(RuntimeError, "cannot replace an active ITIMER_REAL"), - ): - with BENCHMARK._bounded_wait(30): - pass - - def test_cpu_path_is_explicit_deterministic_fp32(self): - torch_module = _CpuTorch() - BENCHMARK._run_cpu(torch_module) - self.assertEqual(torch_module.seed, 0) - self.assertEqual(torch_module.matmul_calls, 1) - - def test_cpu_stage_routes_through_the_pytest_controller(self): - with ( - mock.patch.dict(os.environ, {"stageName": "CPU-Generic-x86-1"}, clear=True), - mock.patch.object(BENCHMARK, "_run_cpu") as run_cpu, - ): - BENCHMARK.test_infra_dry_run_benchmark() - run_cpu.assert_called_once_with() - - def test_gpu_stage_never_falls_back_to_cpu(self): - fake_torch = SimpleNamespace( - cuda=SimpleNamespace(is_available=lambda: False, device_count=lambda: 0) - ) - with ( - mock.patch.dict(os.environ, {"stageName": "A10-GPU"}, clear=True), - mock.patch.object(BENCHMARK, "torch", fake_torch), - mock.patch.object(BENCHMARK, "_run_cpu") as run_cpu, - self.assertRaisesRegex(RuntimeError, "CUDA is required"), - ): - BENCHMARK.test_infra_dry_run_benchmark() - run_cpu.assert_not_called() - - def test_rank_environment_must_be_complete_and_in_range(self): - self.assertIsNone(BENCHMARK._external_rank_context({})) - with self.assertRaisesRegex(RuntimeError, "missing"): - BENCHMARK._external_rank_context({"RANK": "0"}) - with self.assertRaisesRegex(RuntimeError, "outside WORLD_SIZE"): - BENCHMARK._external_rank_context({"RANK": "2", "LOCAL_RANK": "0", "WORLD_SIZE": "2"}) - self.assertEqual( - BENCHMARK._external_rank_context({"RANK": "1", "LOCAL_RANK": "1", "WORLD_SIZE": "2"}), - (1, 1, 2), - ) - - def test_rank_summary_validation_rejects_missing_or_inconsistent_ranks(self): - BENCHMARK._validate_rank_summaries([[0.0, 2.0, 10.0], [1.0, 2.0, 10.0]], 2) - with self.assertRaisesRegex(RuntimeError, "observed ranks"): - BENCHMARK._validate_rank_summaries([[0.0, 2.0, 10.0]], 2) - with self.assertRaisesRegex(RuntimeError, "world sizes"): - BENCHMARK._validate_rank_summaries([[0.0, 2.0, 10.0], [1.0, 3.0, 10.0]], 2) - with self.assertRaisesRegex(RuntimeError, "checksums"): - BENCHMARK._validate_rank_summaries([[0.0, 2.0, 10.0], [1.0, 2.0, 11.0]], 2) - - def test_distributed_failure_always_destroys_the_process_group(self): - class Distributed: - def __init__(self): - self.destroyed = False - - def is_available(self): - return True - - def is_nccl_available(self): - return True - - def is_initialized(self): - return True - - def destroy_process_group(self): - self.destroyed = True - - distributed = Distributed() - torch_module = SimpleNamespace(distributed=distributed) - with ( - mock.patch.object( - BENCHMARK, "_run_cuda_matmul", side_effect=RuntimeError("CUDA failed") - ), - self.assertRaisesRegex(RuntimeError, "CUDA failed"), - ): - BENCHMARK._run_distributed_rank(0, 0, 2, torch_module=torch_module) - self.assertTrue(distributed.destroyed) - - def test_multi_node_uses_the_existing_llmapi_session(self): - class Session: - def __init__(self): - self.shutdown_called = False - self.submission = None - - def submit_sync(self, task, timeout): - self.submission = (task, timeout) - return [[0.0, 2.0, 10.0], [1.0, 2.0, 10.0]] - - def shutdown(self): - self.shutdown_called = True - - session = Session() - with mock.patch.dict(sys.modules, {"torch": _IMPORT_TORCH}): - BENCHMARK._run_with_existing_llmapi_launcher( - 2, timeout_seconds=30, session_factory=lambda world_size: session - ) - task, timeout = session.submission - self.assertEqual(task.__module__, "infra_dry_run_benchmark") - self.assertEqual(Path(task.__code__.co_filename).resolve(), BENCHMARK_PATH) - self.assertEqual(timeout, 30) - self.assertTrue(session.shutdown_called) - - def test_llmapi_rank_task_pickle_is_importable_in_worker_directory(self): - with mock.patch.dict(sys.modules, {"torch": _IMPORT_TORCH}): - task = BENCHMARK._pickleable_llmapi_rank_task() - payload = pickle.dumps(task) - with tempfile.TemporaryDirectory() as temp_dir: - Path(temp_dir, "torch.py").write_text("# worker import stub\n") - env = os.environ.copy() - env["PYTHONPATH"] = os.pathsep.join( - value for value in (temp_dir, env.get("PYTHONPATH")) if value - ) - result = subprocess.run( - [ - sys.executable, - "-c", - "import pickle, sys; print(pickle.loads(sys.stdin.buffer.read()).__module__)", - ], - cwd=BENCHMARK_PATH.parent, - env=env, - input=payload, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr.decode()) - self.assertEqual(result.stdout.decode().strip(), "infra_dry_run_benchmark") - - def test_top_level_worker_module_is_importable_by_a_real_spawn_child(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_path = Path(temp_dir) - (temp_path / "torch.py").write_text("# import-only torch stub\n") - driver = temp_path / "spawn_driver.py" - driver.write_text( - textwrap.dedent( - f""" - import importlib.util - import multiprocessing - import sys - from pathlib import Path - benchmark_path = Path({str(BENCHMARK_PATH)!r}) - - def main(): - spec = importlib.util.spec_from_file_location( - "defs.infra_dry_run_benchmark", benchmark_path - ) - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - - module_dir = benchmark_path.parent.resolve() - sys.path[:] = [ - entry - for entry in sys.path - if Path(entry or ".").resolve() != module_dir - ] - worker_module = module._worker_import_module() - process = multiprocessing.get_context("spawn").Process( - target=worker_module._required_int, - args=({{"VALUE": "7"}}, "VALUE"), - ) - process.start() - process.join(30) - if process.is_alive(): - process.terminate() - process.join() - raise RuntimeError("spawn child did not finish") - raise SystemExit(process.exitcode) - - if __name__ == "__main__": - main() - """ + dry_list = root / "dry.txt" + dry_list.write_text(f"{BENCHMARK_PATH.name}::test_infra_dry_run_benchmark\n") + normal_list = root / "normal.txt" + normal_list.write_text("test_product.py::test_product\n") + env = {**os.environ, "stageName": "CPU-Generic-x86-1"} + for test_list, expected in ( + (dry_list, BENCHMARK_PATH.name), + (normal_list, "test_product.py"), + ): + result = subprocess.run( + [sys.executable, "-m", "pytest", f"--test-list={test_list}", "-vv"], + cwd=root, + env=env, + check=True, + capture_output=True, + text=True, ) - ) - env = {**os.environ, "PYTHONPATH": temp_dir} - result = subprocess.run( - [sys.executable, str(driver)], - cwd=temp_dir, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=False, - ) - - self.assertEqual(result.returncode, 0, result.stdout) - - def test_llmapi_worker_binds_rank_environment_to_the_mpi_communicator(self): - communicator = SimpleNamespace(Get_rank=lambda: 1, Get_size=lambda: 2) - mpi4py = SimpleNamespace(MPI=SimpleNamespace(COMM_WORLD=communicator)) - environ = {"RANK": "1", "LOCAL_RANK": "0", "WORLD_SIZE": "2"} - with ( - mock.patch.dict(os.environ, environ, clear=True), - mock.patch.dict(sys.modules, {"mpi4py": mpi4py}), - mock.patch.object( - BENCHMARK, "_run_distributed_rank", return_value=[1.0, 2.0, 10.0] - ) as run_rank, - ): - result = BENCHMARK._llmapi_rank_task(45) - self.assertEqual(result, [1.0, 2.0, 10.0]) - run_rank.assert_called_once_with(1, 0, 2, 45) - - def test_external_multi_node_rank_reuses_existing_launcher(self): - environ = { - "stageName": "GB300-MultiNode", - "RANK": "0", - "LOCAL_RANK": "0", - "WORLD_SIZE": "2", - "MASTER_ADDR": "host0", - "MASTER_PORT": "23456", - "TLLM_SPAWN_PROXY_PROCESS": "1", - } - with ( - mock.patch.dict(os.environ, environ, clear=True), - mock.patch.object(BENCHMARK, "_run_with_existing_llmapi_launcher") as run_launcher, - ): - BENCHMARK.test_infra_dry_run_benchmark() - run_launcher.assert_called_once_with(2) - - def test_nonzero_proxy_rank_cannot_be_the_pytest_controller(self): - environ = { - "stageName": "GB300-MultiNode", - "RANK": "1", - "LOCAL_RANK": "1", - "WORLD_SIZE": "2", - "MASTER_ADDR": "host0", - "MASTER_PORT": "23456", - "TLLM_SPAWN_PROXY_PROCESS": "1", - } - with ( - mock.patch.dict(os.environ, environ, clear=True), - mock.patch.object(BENCHMARK, "_run_with_existing_llmapi_launcher") as run_launcher, - self.assertRaisesRegex(RuntimeError, "only LLMAPI rank 0"), - ): - BENCHMARK.test_infra_dry_run_benchmark() - run_launcher.assert_not_called() - - def test_single_node_multi_gpu_spawns_one_worker_per_visible_gpu(self): - cuda = SimpleNamespace(is_available=lambda: True, device_count=lambda: 4) - multiprocessing = SimpleNamespace(spawn=mock.Mock()) - fake_torch = SimpleNamespace(cuda=cuda, multiprocessing=multiprocessing) - with ( - mock.patch.dict(os.environ, {"stageName": "H100-Multi-GPU"}, clear=True), - mock.patch.dict(sys.modules, {"torch": fake_torch}), - mock.patch.object(BENCHMARK, "torch", fake_torch), - mock.patch.object(BENCHMARK, "_reserve_local_port", return_value=23456), - ): - BENCHMARK.test_infra_dry_run_benchmark() - - multiprocessing.spawn.assert_called_once() - worker = multiprocessing.spawn.call_args.args[0] - self.assertEqual(worker.__module__, "infra_dry_run_benchmark") - self.assertEqual( - multiprocessing.spawn.call_args.kwargs, - { - "args": (4, 23456, BENCHMARK._DISTRIBUTED_TIMEOUT_SECONDS), - "nprocs": 4, - "join": True, - }, - ) + self.assertIn(expected, result.stdout) + self.assertIn("1 passed, 1 deselected", result.stdout) if __name__ == "__main__": From 70f06978d1c11cc449a131f8876b17cee0c94afd Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:52:23 -0700 Subject: [PATCH 13/34] ci: fix dry-run docs pytest options Remove the undefined ENABLE_S3_ECHO_STDOUT WorkflowScript property from the dry-run Docs pytest adapter. Keep the standard upload path intact and add structural regression coverage that rejects unbound conditional WorkflowScript properties. Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 6 ------ .../tools/test_infra_dry_run_pipeline.py | 20 +++++++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 95cced2a8e91..6c2a3b978022 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3759,12 +3759,6 @@ def runInfraDryRunInPreparedWorkspace(pipeline, String llmSrc, String stageName) "-s", "--s3-upload-path=${uploadPath}/${stageName}", ] - if (ENABLE_S3_ECHO_STDOUT) { - extraArgs += [ - "--s3-echo-stdout", - "--s3-capture-mode=timestamped", - ] - } } def pytestCommand = getPytestBaseCommandLine( llmSrc, diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 223ac1e84686..a1ddd74d3afd 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -14,6 +14,7 @@ # limitations under the License. import os +import re import subprocess import tempfile import unittest @@ -36,6 +37,19 @@ def _function_body(source, name, next_name): return source[start : source.index(f"def {next_name}", start + len(name))] +def _conditional_workflow_properties(function_body): + conditions = re.findall(r"\bif\s*\(([^)]*)\)", function_body) + return { + identifier + for condition in conditions + for identifier in re.findall(r"\b[A-Z][A-Z0-9_]+\b", condition) + } + + +def _top_level_workflow_properties(source): + return set(re.findall(r"(?m)^(?:def\s+)?([A-Z][A-Z0-9_]*)\s*=", source)) + + class InfraDryRunPipelineTest(unittest.TestCase): def test_dedicated_context_selects_one_standard_pytest_case(self): database = DRY_RUN_DB_PATH.read_text() @@ -74,6 +88,12 @@ def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): self.assertIn('withEnv(["stageName=${stageName}"])', prepared) self.assertNotIn("test_infra_dry_run_benchmark.py", prepared) self.assertLess(docs.index("if (isInfraDryRun())"), docs.index("make html")) + conditional_properties = _conditional_workflow_properties(prepared) + self.assertTrue(conditional_properties) + self.assertLessEqual( + conditional_properties, + _top_level_workflow_properties(L0_TEST), + ) def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self): body = _function_body(L0_TEST, "runLLMTestlistWithSbatch", "runLLMTestlistOnSlurm") From 2522e7ef4adc568ad40872caf1300a4ea72ae468 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:45:45 -0700 Subject: [PATCH 14/34] ci: preserve capture for dry-run docs upload Use the standard fd capture and deferred S3 upload options for the dry-run Docs pytest adapter. Add regression coverage that models pytest capture option precedence and rejects S3 upload arguments whose effective capture mode is not fd. Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 3 ++- .../tools/test_infra_dry_run_pipeline.py | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 6c2a3b978022..b97d16ce6d58 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3756,8 +3756,9 @@ def runInfraDryRunInPreparedWorkspace(pipeline, String llmSrc, String stageName) if (ENABLE_UPLOAD_TEST_RESULTS) { def uploadPath = UPLOAD_PATH.replaceFirst("sw-tensorrt-generic/llm-artifacts/LLM/", "") extraArgs += [ - "-s", + "--capture=fd", "--s3-upload-path=${uploadPath}/${stageName}", + "--s3-upload-mode=deferred", ] } def pytestCommand = getPytestBaseCommandLine( diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index a1ddd74d3afd..b7d054d3a6e6 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -50,6 +50,23 @@ def _top_level_workflow_properties(source): return set(re.findall(r"(?m)^(?:def\s+)?([A-Z][A-Z0-9_]*)\s*=", source)) +def _groovy_list_values_after(source, assignment): + assignment_start = source.index(assignment) + list_start = source.index("[", assignment_start) + list_end = source.index("]", list_start) + return re.findall(r'"([^"]+)"', source[list_start:list_end]) + + +def _pytest_capture_mode(args, initial_mode): + capture_mode = initial_mode + for arg in args: + if arg == "-s": + capture_mode = "no" + elif arg.startswith("--capture="): + capture_mode = arg.split("=", 1)[1] + return capture_mode + + class InfraDryRunPipelineTest(unittest.TestCase): def test_dedicated_context_selects_one_standard_pytest_case(self): database = DRY_RUN_DB_PATH.read_text() @@ -94,6 +111,9 @@ def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): conditional_properties, _top_level_workflow_properties(L0_TEST), ) + upload_args = _groovy_list_values_after(prepared, "extraArgs += [") + self.assertTrue(any(arg.startswith("--s3-upload-path=") for arg in upload_args)) + self.assertEqual(_pytest_capture_mode(upload_args, initial_mode="no"), "fd") def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self): body = _function_body(L0_TEST, "runLLMTestlistWithSbatch", "runLLMTestlistOnSlurm") From 74657285a4bfc4e054607a6978da1ec9130f72ce Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Fri, 14 Aug 2026 04:05:51 +0800 Subject: [PATCH 15/34] ci: keep agent-flow in infrastructure dry-run path Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 10 +++++++++- tests/unittest/tools/test_infra_dry_run_pipeline.py | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index b97d16ce6d58..213a643d443b 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3864,7 +3864,15 @@ def runLLMAgentFlowTest(pipeline, stageName) trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) trtllm_utils.llmExecStepWithRetry(pipeline, script: "git config --global --add safe.directory \"*\"") - def agentFlowRoot = "${LLM_ROOT}/agent-flow" + def llmSrc = sh(script: "realpath ${LLM_ROOT}", returnStdout: true).trim() + // Dry acceptance validates the shared benchmark/JUnit/upload path for every + // selected stage; it must not install or execute this product test suite. + if (isInfraDryRun()) { + runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName) + return + } + + def agentFlowRoot = "${llmSrc}/agent-flow" // Install agent-flow with its test extras (pytest, pytest-asyncio) and the // runtime deps from pyproject.toml (claude-agent-sdk, openai-codex, ...). diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index b7d054d3a6e6..bb4ab60ae473 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -115,6 +115,14 @@ def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): self.assertTrue(any(arg.startswith("--s3-upload-path=") for arg in upload_args)) self.assertEqual(_pytest_capture_mode(upload_args, initial_mode="no"), "fd") + def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(self): + body = _function_body(L0_TEST, "runLLMAgentFlowTest", "launchTestListCheck") + self.assertIn("if (isInfraDryRun())", body) + self.assertIn( + "runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)", body + ) + self.assertLess(body.index("if (isInfraDryRun())"), body.index("pip3 install -e")) + def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self): body = _function_body(L0_TEST, "runLLMTestlistWithSbatch", "runLLMTestlistOnSlurm") self.assertIn( From 1b8fd23f31dcc0ea6b8b3e9de088db21bcfe9e78 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Fri, 14 Aug 2026 06:38:08 +0800 Subject: [PATCH 16/34] ci: prepare agent-flow dry-run test environment Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 6 +++++- jenkins/L0_Test.groovy | 8 ++++++++ .../unittest/tools/test_infra_dry_run_pipeline.py | 14 +++++++++++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index b3a041f59b22..360321e34435 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -2291,7 +2291,11 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } }]} - parallelJobs.failFast = enableFailFast + // A dry acceptance run must finish both architecture tracks so one failure + // does not erase the remaining coverage. Preserve the existing fail-fast + // behavior for every normal pipeline. + def effectiveFailFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast + parallelJobs.failFast = effectiveFailFast pipeline.parallel parallelJobs } diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 213a643d443b..482ce0e00970 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3868,6 +3868,14 @@ def runLLMAgentFlowTest(pipeline, stageName) // Dry acceptance validates the shared benchmark/JUnit/upload path for every // selected stage; it must not install or execute this product test suite. if (isInfraDryRun()) { + // The build pod used by AgentFlow does not run the normal TRT-LLM test + // environment setup, so install only the pytest tooling consumed by the + // shared dry-run adapter. Keep the AgentFlow package and its product + // dependencies out of this path. + trtllm_utils.llmExecStepWithRetry( + pipeline, + script: "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout" + ) runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName) return } diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index bb4ab60ae473..6010b60d9550 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -118,9 +118,17 @@ def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(self): body = _function_body(L0_TEST, "runLLMAgentFlowTest", "launchTestListCheck") self.assertIn("if (isInfraDryRun())", body) + infra_pytest_install = ( + "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout" + ) + self.assertIn(infra_pytest_install, body) self.assertIn( "runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)", body ) + self.assertLess( + body.index(infra_pytest_install), + body.index("runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)"), + ) self.assertLess(body.index("if (isInfraDryRun())"), body.index("pip3 install -e")) def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self): @@ -157,7 +165,11 @@ def test_parent_uses_parameter_only_and_explicit_non_fail_fast_helper(self): self.assertIn(", false, false, globalVars,", helper) self.assertIn("'testPhase2StageName': ''", helper) self.assertIn("additionalParameters.containsKey('testPhase2StageName')", launch) - self.assertIn("parallelJobs.failFast = enableFailFast", L0_PARENT) + self.assertIn( + "def effectiveFailFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast", + L0_PARENT, + ) + self.assertIn("parallelJobs.failFast = effectiveFailFast", L0_PARENT) def test_normal_gating_and_result_collection_remain_in_place(self): stages_start = L0_PARENT.index("def launchStages") From 9ca2bf2cbd1f44b3f7e72dbbc0eaae68dc644a87 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Fri, 14 Aug 2026 11:08:23 +0800 Subject: [PATCH 17/34] ci: isolate dry-run conftest dependencies Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 4 +++- tests/integration/defs/conftest.py | 21 ++++++++++++++----- .../tools/test_infra_dry_run_pipeline.py | 18 +++++++++++++++- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 482ce0e00970..fb88062df62f 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3775,7 +3775,9 @@ def runInfraDryRunInPreparedWorkspace(pipeline, String llmSrc, String stageName) "--test-list=${preprocessedLists.regular}", ] - withEnv(["stageName=${stageName}"]) { + // The synthetic benchmark still uses the standard defs/conftest.py reporting + // hooks, but it must not require the TRT-LLM product wheel just to collect. + withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"]) { withCredentials([ string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), diff --git a/tests/integration/defs/conftest.py b/tests/integration/defs/conftest.py index 4edd7d09f975..663619a7e840 100644 --- a/tests/integration/defs/conftest.py +++ b/tests/integration/defs/conftest.py @@ -48,11 +48,6 @@ # is harmless. from test_common import session_prefetcher_hooks as _prefetch_hooks -from tensorrt_llm.bindings import ipc_nvls_supported -from tensorrt_llm.llmapi.mpi_session import get_mpi_world_size - -from .perf.gpu_clock_lock import GPUClockLock -from .perf.session_data_writer import SessionDataWriter from .test_list_parser import (TestCorrectionMode, apply_waives, get_test_name_corrections_v2, handle_corrections, modify_by_test_list, preprocess_test_list_lines) @@ -66,6 +61,22 @@ except ImportError: trt_environment = None +_INFRA_DRY_RUN = os.environ.get("TRTLLM_INFRA_DRY_RUN", "").lower() == "true" +if _INFRA_DRY_RUN: + + def ipc_nvls_supported(): + return False + + def get_mpi_world_size(): + return 1 + +else: + from tensorrt_llm.bindings import ipc_nvls_supported + from tensorrt_llm.llmapi.mpi_session import get_mpi_world_size + + from .perf.gpu_clock_lock import GPUClockLock + from .perf.session_data_writer import SessionDataWriter + # Logger logger = logging.getLogger(__name__) diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 6010b60d9550..69e0bca23571 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -27,6 +27,7 @@ SLURM_INSTALL_PATH = REPO_ROOT / "jenkins" / "scripts" / "slurm_install.sh" CHECK_TEST_LIST = (REPO_ROOT / "scripts" / "check_test_list.py").read_text() BENCHMARK_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "test_infra_dry_run_benchmark.py" +CONFTEST = (REPO_ROOT / "tests" / "integration" / "defs" / "conftest.py").read_text() DRY_RUN_DB_PATH = ( REPO_ROOT / "tests" / "integration" / "test_lists" / "test-db" / "infra_dry_run.yml" ) @@ -102,7 +103,9 @@ def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): for call in ("renderTestDB(", "processShardTestList(", "getPytestBaseCommandLine("): self.assertIn(call, prepared) self.assertIn("--test-list=${preprocessedLists.regular}", prepared) - self.assertIn('withEnv(["stageName=${stageName}"])', prepared) + self.assertIn( + 'withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"])', prepared + ) self.assertNotIn("test_infra_dry_run_benchmark.py", prepared) self.assertLess(docs.index("if (isInfraDryRun())"), docs.index("make html")) conditional_properties = _conditional_workflow_properties(prepared) @@ -131,6 +134,19 @@ def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(sel ) self.assertLess(body.index("if (isInfraDryRun())"), body.index("pip3 install -e")) + def test_dry_run_conftest_does_not_require_product_bindings(self): + dry_guard = '_INFRA_DRY_RUN = os.environ.get("TRTLLM_INFRA_DRY_RUN", "").lower() == "true"' + self.assertIn(dry_guard, CONFTEST) + guard_start = CONFTEST.index(dry_guard) + normal_import = CONFTEST.index( + "from tensorrt_llm.bindings import ipc_nvls_supported", guard_start + ) + fallback = CONFTEST[guard_start:normal_import] + self.assertIn("def ipc_nvls_supported():", fallback) + self.assertIn("def get_mpi_world_size():", fallback) + self.assertIn("else:", fallback) + self.assertNotIn("from .perf.gpu_clock_lock import GPUClockLock", fallback) + def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self): body = _function_body(L0_TEST, "runLLMTestlistWithSbatch", "runLLMTestlistOnSlurm") self.assertIn( From 9bda5f66d99557c1cab26cfb86d92be0afa5ff52 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:19:08 +0800 Subject: [PATCH 18/34] ci: install mako for AgentFlow dry run Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 8 ++++---- tests/unittest/tools/test_infra_dry_run_pipeline.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index fb88062df62f..e9248d6e594f 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3871,12 +3871,12 @@ def runLLMAgentFlowTest(pipeline, stageName) // selected stage; it must not install or execute this product test suite. if (isInfraDryRun()) { // The build pod used by AgentFlow does not run the normal TRT-LLM test - // environment setup, so install only the pytest tooling consumed by the - // shared dry-run adapter. Keep the AgentFlow package and its product - // dependencies out of this path. + // environment setup, so install only the pytest and test-list tooling + // consumed by the shared dry-run adapter. Keep the AgentFlow package + // and its product dependencies out of this path. trtllm_utils.llmExecStepWithRetry( pipeline, - script: "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout" + script: "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout mako" ) runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName) return diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 69e0bca23571..3eb7133ea519 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -122,7 +122,7 @@ def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(sel body = _function_body(L0_TEST, "runLLMAgentFlowTest", "launchTestListCheck") self.assertIn("if (isInfraDryRun())", body) infra_pytest_install = ( - "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout" + "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout mako" ) self.assertIn(infra_pytest_install, body) self.assertIn( From 499d77a2322d50be7af290e10eeb1c55c4054949 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Fri, 14 Aug 2026 00:24:49 -0700 Subject: [PATCH 19/34] ci: format dry-run pipeline test Apply the Ruff 0.9.4 formatting required by the latest main configuration after rebasing the focused dry-run regression test. Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- tests/unittest/tools/test_infra_dry_run_pipeline.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 3eb7133ea519..09eaa1352f87 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -103,9 +103,7 @@ def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): for call in ("renderTestDB(", "processShardTestList(", "getPytestBaseCommandLine("): self.assertIn(call, prepared) self.assertIn("--test-list=${preprocessedLists.regular}", prepared) - self.assertIn( - 'withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"])', prepared - ) + self.assertIn('withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"])', prepared) self.assertNotIn("test_infra_dry_run_benchmark.py", prepared) self.assertLess(docs.index("if (isInfraDryRun())"), docs.index("make html")) conditional_properties = _conditional_workflow_properties(prepared) @@ -125,9 +123,7 @@ def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(sel "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout mako" ) self.assertIn(infra_pytest_install, body) - self.assertIn( - "runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)", body - ) + self.assertIn("runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)", body) self.assertLess( body.index(infra_pytest_install), body.index("runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)"), From 2db0641cdc54e4e38fb5b2229e03ff439f543d3c Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:40:26 +0800 Subject: [PATCH 20/34] ci: keep dry-run import guard during collection Apply the infrastructure dry-run environment before test-list preprocessing so the collect-only pass does not require an installed TensorRT-LLM wheel. Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 81 ++++++++++--------- .../tools/test_infra_dry_run_pipeline.py | 7 ++ 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index e9248d6e594f..323bd2dbc7bb 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3735,49 +3735,50 @@ def runInfraDryRunInPreparedWorkspace(pipeline, String llmSrc, String stageName) def coverageConfigFile = "${llmSrc}/infra_dry_run.coveragerc" sh "rm -rf ${outputPath} && mkdir -p ${outputPath} && : > ${waivesFile} && : > ${coverageConfigFile}" - def testDBList = renderTestDB( - pipeline, - INFRA_DRY_RUN_TEST_CONTEXT, - llmSrc, - stageName, - ) - def preprocessedLists = processShardTestList( - llmSrc, - testDBList, - 1, - 1, - false, - ) - if (preprocessedLists.regularCount < 1) { - error "No infrastructure dry-run benchmark was selected for ${stageName}" - } + // Test-list preprocessing performs a pytest --collect-only pass before the + // final pytest invocation. Both passes load defs/conftest.py, so keep the + // dry-run import guard active for the entire prepared-workspace flow. + withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"]) { + def testDBList = renderTestDB( + pipeline, + INFRA_DRY_RUN_TEST_CONTEXT, + llmSrc, + stageName, + ) + def preprocessedLists = processShardTestList( + llmSrc, + testDBList, + 1, + 1, + false, + ) + if (preprocessedLists.regularCount < 1) { + error "No infrastructure dry-run benchmark was selected for ${stageName}" + } - def extraArgs = [] - if (ENABLE_UPLOAD_TEST_RESULTS) { - def uploadPath = UPLOAD_PATH.replaceFirst("sw-tensorrt-generic/llm-artifacts/LLM/", "") - extraArgs += [ - "--capture=fd", - "--s3-upload-path=${uploadPath}/${stageName}", - "--s3-upload-mode=deferred", + def extraArgs = [] + if (ENABLE_UPLOAD_TEST_RESULTS) { + def uploadPath = UPLOAD_PATH.replaceFirst("sw-tensorrt-generic/llm-artifacts/LLM/", "") + extraArgs += [ + "--capture=fd", + "--s3-upload-path=${uploadPath}/${stageName}", + "--s3-upload-mode=deferred", + ] + } + def pytestCommand = getPytestBaseCommandLine( + llmSrc, + stageName, + waivesFile, + false, + outputPath, + coverageConfigFile, + "", + extraArgs, + ) + pytestCommand += [ + "--test-list=${preprocessedLists.regular}", ] - } - def pytestCommand = getPytestBaseCommandLine( - llmSrc, - stageName, - waivesFile, - false, - outputPath, - coverageConfigFile, - "", - extraArgs, - ) - pytestCommand += [ - "--test-list=${preprocessedLists.regular}", - ] - // The synthetic benchmark still uses the standard defs/conftest.py reporting - // hooks, but it must not require the TRT-LLM product wheel just to collect. - withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"]) { withCredentials([ string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 09eaa1352f87..b55a8420d927 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -130,6 +130,13 @@ def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(sel ) self.assertLess(body.index("if (isInfraDryRun())"), body.index("pip3 install -e")) + def test_prepared_workspace_sets_dry_environment_before_collection(self): + body = _function_body(L0_TEST, "runInfraDryRunInPreparedWorkspace", "runLLMDocBuild") + dry_environment = 'withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"])' + self.assertEqual(body.count(dry_environment), 1) + self.assertLess(body.index(dry_environment), body.index("processShardTestList(")) + self.assertLess(body.index(dry_environment), body.index("${pytestCommand.join")) + def test_dry_run_conftest_does_not_require_product_bindings(self): dry_guard = '_INFRA_DRY_RUN = os.environ.get("TRTLLM_INFRA_DRY_RUN", "").lower() == "true"' self.assertIn(dry_guard, CONFTEST) From a6842a2ce625d80d29b711831a479bf3c8c0ac29 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:29:55 +0800 Subject: [PATCH 21/34] ci: install unused fixtures plugin for dry run Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 2 +- tests/unittest/tools/test_infra_dry_run_pipeline.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 323bd2dbc7bb..9ff2acfa40e9 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3877,7 +3877,7 @@ def runLLMAgentFlowTest(pipeline, stageName) // and its product dependencies out of this path. trtllm_utils.llmExecStepWithRetry( pipeline, - script: "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout mako" + script: "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout pytest-unused-fixtures mako" ) runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName) return diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index b55a8420d927..08c18c26e584 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -120,7 +120,8 @@ def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(sel body = _function_body(L0_TEST, "runLLMAgentFlowTest", "launchTestListCheck") self.assertIn("if (isInfraDryRun())", body) infra_pytest_install = ( - "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout mako" + "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout " + "pytest-unused-fixtures mako" ) self.assertIn(infra_pytest_install, body) self.assertIn("runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)", body) From 1eba03ec78d1d3fa5948a7106855d5b315644809 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:32:21 +0800 Subject: [PATCH 22/34] ci: use CPU stage identity for dry docs Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 2 +- tests/unittest/tools/test_infra_dry_run_pipeline.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 9ff2acfa40e9..30c4a01f2aa2 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6405,7 +6405,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) docBuildConfigs = [ "CPU-Build_Docs": [docBuildSpec, { sh "rm -rf **/*.xml *.tar.gz" - runLLMDocBuild(pipeline, VANILLA_CONFIG, "A10-Build_Docs") + runLLMDocBuild(pipeline, VANILLA_CONFIG, "CPU-Build_Docs") }], ] diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 08c18c26e584..5248353df35a 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -112,6 +112,12 @@ def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): conditional_properties, _top_level_workflow_properties(L0_TEST), ) + self.assertIn( + 'runLLMDocBuild(pipeline, VANILLA_CONFIG, "CPU-Build_Docs")', L0_TEST + ) + self.assertNotIn( + 'runLLMDocBuild(pipeline, VANILLA_CONFIG, "A10-Build_Docs")', L0_TEST + ) upload_args = _groovy_list_values_after(prepared, "extraArgs += [") self.assertTrue(any(arg.startswith("--s3-upload-path=") for arg in upload_args)) self.assertEqual(_pytest_capture_mode(upload_args, initial_mode="no"), "fd") From 54a778b5a09e0b681fd4c53c86bc967b5d884a1a Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Sat, 15 Aug 2026 04:33:01 +0800 Subject: [PATCH 23/34] Limit dry-run pytest collection to synthetic targets Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 27 ++++++++- .../tools/test_infra_dry_run_pipeline.py | 26 ++++++++ .../tools/test_infra_dry_run_pytest.py | 60 +++++++++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 30c4a01f2aa2..8efaabe7c5f7 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -569,6 +569,24 @@ def runIsolatedTests(preprocessedLists, testCmdLine, llmSrc, stageName) { return rerunFailed // Return the updated value } +def getInfraDryRunPytestTargets(testListPath) { + if (!isInfraDryRun()) { + return [] + } + + // --test-list filters items only after pytest has imported every file under + // the current directory. The CPU-only dry runners intentionally do not + // install the product test environment, so limit collection to the rendered + // synthetic nodeids as positional arguments as well. + def targets = readFile(file: testListPath).readLines() + .collect { it.trim().split(/\s+/, 2)[0] } + .findAll { it.contains("::") } + if (!targets) { + error "No pytest targets found in infrastructure dry-run list ${testListPath}" + } + return targets +} + def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, durationsPath="") { // Preprocess testDBList to extract ISOLATION markers echo "Preprocessing testDBList to extract ISOLATION markers..." @@ -641,6 +659,7 @@ def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, du if (durationsPath) { testListCmd += ["--durations-path ${durationsPath}"] } + testListCmd += getInfraDryRunPytestTargets(cleanedTestDBList) try { // First execute the pytest command and check if it succeeds @@ -1927,7 +1946,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "--s3-upload-mode=deferred", ] } - def pytestCommand = getPytestBaseCommandLine( + def pytestCommandParts = getPytestBaseCommandLine( llmSrcNode, stageName, waivesListPathNode, @@ -1936,7 +1955,9 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "$jobWorkspace/.coveragerc", pytestUtil, extraArgs, - ).join(" ") + ) + pytestCommandParts += getInfraDryRunPytestTargets(testListPathLocal) + def pytestCommand = pytestCommandParts.join(" ") // Generate Job Launch Script def container = LLM_DOCKER_IMAGE @@ -3778,6 +3799,7 @@ def runInfraDryRunInPreparedWorkspace(pipeline, String llmSrc, String stageName) pytestCommand += [ "--test-list=${preprocessedLists.regular}", ] + pytestCommand += getInfraDryRunPytestTargets(preprocessedLists.regular) withCredentials([ string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), @@ -5030,6 +5052,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO // Only add --test-list if there are regular tests to run if (preprocessedLists.regularCount > 0) { pytestCommand += ["--test-list=${preprocessedLists.regular}"] + pytestCommand += getInfraDryRunPytestTargets(preprocessedLists.regular) } def containerPIP_LLM_LIB_PATH = sh(script: "pip3 show tensorrt_llm | grep \"Location\" | awk -F\":\" '{ gsub(/ /, \"\", \$2); print \$2\"/tensorrt_llm/libs\"}'", returnStdout: true).replaceAll("\\s","") diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 5248353df35a..ba45c5a421b3 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -90,6 +90,10 @@ def test_platform_runner_uses_standard_pytest_results_and_reporting(self): self.assertIn("effectivePerfMode = infraDryRun ? false : perfMode", body) self.assertIn("getPytestBaseCommandLine(", body) self.assertIn("--test-list=${preprocessedLists.regular}", body) + self.assertIn( + "pytestCommand += getInfraDryRunPytestTargets(preprocessedLists.regular)", + body, + ) self.assertIn("rerunFailedTests(", body) self.assertIn("runIsolatedTests(", body) self.assertIn("generateRerunReport(", body) @@ -103,6 +107,10 @@ def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): for call in ("renderTestDB(", "processShardTestList(", "getPytestBaseCommandLine("): self.assertIn(call, prepared) self.assertIn("--test-list=${preprocessedLists.regular}", prepared) + self.assertIn( + "pytestCommand += getInfraDryRunPytestTargets(preprocessedLists.regular)", + prepared, + ) self.assertIn('withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"])', prepared) self.assertNotIn("test_infra_dry_run_benchmark.py", prepared) self.assertLess(docs.index("if (isInfraDryRun())"), docs.index("make html")) @@ -144,6 +152,20 @@ def test_prepared_workspace_sets_dry_environment_before_collection(self): self.assertLess(body.index(dry_environment), body.index("processShardTestList(")) self.assertLess(body.index(dry_environment), body.index("${pytestCommand.join")) + def test_dry_run_limits_collection_to_rendered_nodeids(self): + targets = _function_body( + L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList" + ) + preprocessing = _function_body( + L0_TEST, "processShardTestList", "isValidSlurmJobId" + ) + self.assertIn("if (!isInfraDryRun())", targets) + self.assertIn('.findAll { it.contains("::") }', targets) + self.assertIn( + "testListCmd += getInfraDryRunPytestTargets(cleanedTestDBList)", + preprocessing, + ) + def test_dry_run_conftest_does_not_require_product_bindings(self): dry_guard = '_INFRA_DRY_RUN = os.environ.get("TRTLLM_INFRA_DRY_RUN", "").lower() == "true"' self.assertIn(dry_guard, CONFTEST) @@ -163,6 +185,10 @@ def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self): "effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList", body ) self.assertIn("String[] taskArgs = getNodeArgs(", body) + self.assertIn( + "pytestCommandParts += getInfraDryRunPytestTargets(testListPathLocal)", + body, + ) self.assertIn('pytestUtil = "$llmSrcNode/tensorrt_llm/llmapi/trtllm-llmapi-launch"', body) self.assertIn("if (!isInfraDryRun() && (disaggMultiNodeMode || aggMultiNodeMode))", body) self.assertNotIn("test_infra_dry_run_benchmark.py", body) diff --git a/tests/unittest/tools/test_infra_dry_run_pytest.py b/tests/unittest/tools/test_infra_dry_run_pytest.py index b4056a7ef278..cc4da0f5181b 100644 --- a/tests/unittest/tools/test_infra_dry_run_pytest.py +++ b/tests/unittest/tools/test_infra_dry_run_pytest.py @@ -191,6 +191,66 @@ def pytest_collection_modifyitems(config, items): self.assertIn(expected, result.stdout) self.assertIn("1 passed, 1 deselected", result.stdout) + def test_positional_nodeid_does_not_import_unrelated_product_test(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / BENCHMARK_PATH.name).write_text(BENCHMARK_PATH.read_text()) + (root / "test_unrelated_product.py").write_text( + 'raise RuntimeError("unrelated product test imported")\n' + ) + (root / "torch.py").write_text( + textwrap.dedent( + """ + float32 = "float32" + class Device: + def __init__(self, kind, index=None): + self.type, self.index = kind, index + class Scalar: + def item(self): return True + class Tensor: + def __init__(self, value, dtype, device): + self.value, self.dtype, self.device = value, dtype, device + def all(self): return Scalar() + def device(kind, index=None): return Device(kind, index) + def full(_shape, value, *, dtype, device): return Tensor(value, dtype, device) + def matmul(left, _right): return Tensor(4.0, left.dtype, left.device) + def full_like(tensor, value): return Tensor(value, tensor.dtype, tensor.device) + def isfinite(tensor): return tensor + def equal(left, right): return left.value == right.value + """ + ) + ) + (root / "conftest.py").write_text( + textwrap.dedent( + """ + def pytest_addoption(parser): + parser.addoption("--test-list") + """ + ) + ) + + target = f"{BENCHMARK_PATH.name}::test_infra_dry_run_benchmark" + dry_list = root / "dry.txt" + dry_list.write_text(f"{target}\n") + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + f"--test-list={dry_list}", + target, + "-q", + ], + cwd=root, + env={**os.environ, "stageName": "CPU-Generic-x86-1"}, + check=True, + capture_output=True, + text=True, + ) + self.assertIn(target, result.stdout) + self.assertNotIn("test_unrelated_product.py", result.stdout) + if __name__ == "__main__": unittest.main() From ffba2ede579a3e1a138cb76d3f70a3aa76389d65 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Sat, 15 Aug 2026 09:12:21 +0800 Subject: [PATCH 24/34] Add dry-run S3 upload dependency for AgentFlow Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 8 ++++---- tests/unittest/tools/test_infra_dry_run_pipeline.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 8efaabe7c5f7..df0dd21b3225 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3894,12 +3894,12 @@ def runLLMAgentFlowTest(pipeline, stageName) // selected stage; it must not install or execute this product test suite. if (isInfraDryRun()) { // The build pod used by AgentFlow does not run the normal TRT-LLM test - // environment setup, so install only the pytest and test-list tooling - // consumed by the shared dry-run adapter. Keep the AgentFlow package - // and its product dependencies out of this path. + // environment setup, so install only the pytest, test-list, and upload + // tooling consumed by the shared dry-run adapter. Keep the AgentFlow + // package and its product dependencies out of this path. trtllm_utils.llmExecStepWithRetry( pipeline, - script: "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout pytest-unused-fixtures mako" + script: "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout pytest-unused-fixtures mako boto3" ) runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName) return diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index ba45c5a421b3..e5480c6d16c6 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -135,7 +135,7 @@ def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(sel self.assertIn("if (isInfraDryRun())", body) infra_pytest_install = ( "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout " - "pytest-unused-fixtures mako" + "pytest-unused-fixtures mako boto3" ) self.assertIn(infra_pytest_install, body) self.assertIn("runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)", body) From 97226d0864c93589d33edb6df5f95d85f7a144a9 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:40:28 +0800 Subject: [PATCH 25/34] ci: preserve normal paths around infrastructure dry run Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 12 ++++----- .../tools/test_infra_dry_run_pipeline.py | 27 ++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index df0dd21b3225..5acbe73546c0 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2131,7 +2131,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG export llmSrcNode=$llmSrcNode export stageName=$stageName export perfMode=$effectivePerfMode - export infraDryRun=$infraDryRun + ${infraDryRun ? "export infraDryRun=true" : ""} export resourcePathNode=$resourcePathNode export pytestCommand="$pytestCommand" export coverageConfigFile="$coverageConfigFile" @@ -3814,7 +3814,7 @@ def runInfraDryRunInPreparedWorkspace(pipeline, String llmSrc, String stageName) } } -def runLLMDocBuild(pipeline, config, stageName) +def runLLMDocBuild(pipeline, config) { // Step 1: cloning source code sh "pwd && ls -alh" @@ -3840,7 +3840,7 @@ def runLLMDocBuild(pipeline, config, stageName) trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmPath} && pip3 install --force-reinstall --no-deps TensorRT-LLM/tensorrt_llm-*.whl") if (isInfraDryRun()) { - runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName) + runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, "CPU-Build_Docs") return } @@ -3889,10 +3889,10 @@ def runLLMAgentFlowTest(pipeline, stageName) trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) trtllm_utils.llmExecStepWithRetry(pipeline, script: "git config --global --add safe.directory \"*\"") - def llmSrc = sh(script: "realpath ${LLM_ROOT}", returnStdout: true).trim() // Dry acceptance validates the shared benchmark/JUnit/upload path for every // selected stage; it must not install or execute this product test suite. if (isInfraDryRun()) { + def llmSrc = sh(script: "realpath ${LLM_ROOT}", returnStdout: true).trim() // The build pod used by AgentFlow does not run the normal TRT-LLM test // environment setup, so install only the pytest, test-list, and upload // tooling consumed by the shared dry-run adapter. Keep the AgentFlow @@ -3905,7 +3905,7 @@ def runLLMAgentFlowTest(pipeline, stageName) return } - def agentFlowRoot = "${llmSrc}/agent-flow" + def agentFlowRoot = "${LLM_ROOT}/agent-flow" // Install agent-flow with its test extras (pytest, pytest-asyncio) and the // runtime deps from pyproject.toml (claude-agent-sdk, openai-codex, ...). @@ -6428,7 +6428,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) docBuildConfigs = [ "CPU-Build_Docs": [docBuildSpec, { sh "rm -rf **/*.xml *.tar.gz" - runLLMDocBuild(pipeline, VANILLA_CONFIG, "CPU-Build_Docs") + runLLMDocBuild(pipeline, config=VANILLA_CONFIG) }], ] diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index e5480c6d16c6..4348534291f7 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -120,12 +120,9 @@ def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): conditional_properties, _top_level_workflow_properties(L0_TEST), ) - self.assertIn( - 'runLLMDocBuild(pipeline, VANILLA_CONFIG, "CPU-Build_Docs")', L0_TEST - ) - self.assertNotIn( - 'runLLMDocBuild(pipeline, VANILLA_CONFIG, "A10-Build_Docs")', L0_TEST - ) + self.assertIn("def runLLMDocBuild(pipeline, config)", L0_TEST) + self.assertIn("runLLMDocBuild(pipeline, config=VANILLA_CONFIG)", L0_TEST) + self.assertIn('runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, "CPU-Build_Docs")', docs) upload_args = _groovy_list_values_after(prepared, "extraArgs += [") self.assertTrue(any(arg.startswith("--s3-upload-path=") for arg in upload_args)) self.assertEqual(_pytest_capture_mode(upload_args, initial_mode="no"), "fd") @@ -144,6 +141,14 @@ def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(sel body.index("runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)"), ) self.assertLess(body.index("if (isInfraDryRun())"), body.index("pip3 install -e")) + dry_guard = body.index("if (isInfraDryRun())") + realpath = body.index("realpath ${LLM_ROOT}") + dry_runner = body.index("runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)") + normal_root = body.index('def agentFlowRoot = "${LLM_ROOT}/agent-flow"') + self.assertLess(dry_guard, realpath) + self.assertLess(realpath, dry_runner) + self.assertLess(body.index("\n return", dry_runner), normal_root) + self.assertIn('def agentFlowRoot = "${LLM_ROOT}/agent-flow"', body) def test_prepared_workspace_sets_dry_environment_before_collection(self): body = _function_body(L0_TEST, "runInfraDryRunInPreparedWorkspace", "runLLMDocBuild") @@ -153,12 +158,8 @@ def test_prepared_workspace_sets_dry_environment_before_collection(self): self.assertLess(body.index(dry_environment), body.index("${pytestCommand.join")) def test_dry_run_limits_collection_to_rendered_nodeids(self): - targets = _function_body( - L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList" - ) - preprocessing = _function_body( - L0_TEST, "processShardTestList", "isValidSlurmJobId" - ) + targets = _function_body(L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList") + preprocessing = _function_body(L0_TEST, "processShardTestList", "isValidSlurmJobId") self.assertIn("if (!isInfraDryRun())", targets) self.assertIn('.findAll { it.contains("::") }', targets) self.assertIn( @@ -189,6 +190,8 @@ def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self): "pytestCommandParts += getInfraDryRunPytestTargets(testListPathLocal)", body, ) + self.assertIn('${infraDryRun ? "export infraDryRun=true" : ""}', body) + self.assertNotIn("export infraDryRun=$infraDryRun", body) self.assertIn('pytestUtil = "$llmSrcNode/tensorrt_llm/llmapi/trtllm-llmapi-launch"', body) self.assertIn("if (!isInfraDryRun() && (disaggMultiNodeMode || aggMultiNodeMode))", body) self.assertNotIn("test_infra_dry_run_benchmark.py", body) From 65a64894ff535f676027aba9d8f3fdb75b5181cc Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:14:11 +0800 Subject: [PATCH 26/34] ci: validate dry-run pytest target Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 6 ++++-- .../unittest/tools/test_infra_dry_run_pipeline.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 5acbe73546c0..025633dc8260 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -581,8 +581,10 @@ def getInfraDryRunPytestTargets(testListPath) { def targets = readFile(file: testListPath).readLines() .collect { it.trim().split(/\s+/, 2)[0] } .findAll { it.contains("::") } - if (!targets) { - error "No pytest targets found in infrastructure dry-run list ${testListPath}" + def expectedTarget = + "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" + if (targets != [expectedTarget]) { + error "Unexpected pytest targets in infrastructure dry-run list ${testListPath}: ${targets}" } return targets } diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 4348534291f7..db3bf60c524e 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -160,13 +160,28 @@ def test_prepared_workspace_sets_dry_environment_before_collection(self): def test_dry_run_limits_collection_to_rendered_nodeids(self): targets = _function_body(L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList") preprocessing = _function_body(L0_TEST, "processShardTestList", "isValidSlurmJobId") + expected_target = ( + "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" + ) self.assertIn("if (!isInfraDryRun())", targets) self.assertIn('.findAll { it.contains("::") }', targets) + self.assertIn(f'"{expected_target}"', targets) + self.assertIn("if (targets != [expectedTarget])", targets) self.assertIn( "testListCmd += getInfraDryRunPytestTargets(cleanedTestDBList)", preprocessing, ) + def test_dry_run_rejects_shell_metacharacters_in_rendered_target(self): + targets = _function_body(L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList") + expected_target = ( + "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" + ) + metacharacter_target = f"{expected_target};touch${{IFS}}/tmp/infra-dry-run" + parsed_targets = [metacharacter_target.split(maxsplit=1)[0]] + self.assertNotEqual(parsed_targets, [expected_target]) + self.assertIn("if (targets != [expectedTarget])", targets) + def test_dry_run_conftest_does_not_require_product_bindings(self): dry_guard = '_INFRA_DRY_RUN = os.environ.get("TRTLLM_INFRA_DRY_RUN", "").lower() == "true"' self.assertIn(dry_guard, CONFTEST) From 99dde8c0e9b032a4a91a9c50e2a74743d1d3d4b9 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:39:02 +0800 Subject: [PATCH 27/34] ci: annotate dry-run pipeline test helpers Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- .../tools/test_infra_dry_run_pipeline.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index db3bf60c524e..20d02689c900 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -33,12 +33,12 @@ ) -def _function_body(source, name, next_name): +def _function_body(source: str, name: str, next_name: str) -> str: start = source.index(f"def {name}") return source[start : source.index(f"def {next_name}", start + len(name))] -def _conditional_workflow_properties(function_body): +def _conditional_workflow_properties(function_body: str) -> set[str]: conditions = re.findall(r"\bif\s*\(([^)]*)\)", function_body) return { identifier @@ -47,18 +47,18 @@ def _conditional_workflow_properties(function_body): } -def _top_level_workflow_properties(source): +def _top_level_workflow_properties(source: str) -> set[str]: return set(re.findall(r"(?m)^(?:def\s+)?([A-Z][A-Z0-9_]*)\s*=", source)) -def _groovy_list_values_after(source, assignment): +def _groovy_list_values_after(source: str, assignment: str) -> list[str]: assignment_start = source.index(assignment) list_start = source.index("[", assignment_start) list_end = source.index("]", list_start) return re.findall(r'"([^"]+)"', source[list_start:list_end]) -def _pytest_capture_mode(args, initial_mode): +def _pytest_capture_mode(args: list[str], initial_mode: str) -> str: capture_mode = initial_mode for arg in args: if arg == "-s": @@ -160,9 +160,7 @@ def test_prepared_workspace_sets_dry_environment_before_collection(self): def test_dry_run_limits_collection_to_rendered_nodeids(self): targets = _function_body(L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList") preprocessing = _function_body(L0_TEST, "processShardTestList", "isValidSlurmJobId") - expected_target = ( - "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" - ) + expected_target = "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" self.assertIn("if (!isInfraDryRun())", targets) self.assertIn('.findAll { it.contains("::") }', targets) self.assertIn(f'"{expected_target}"', targets) @@ -174,9 +172,7 @@ def test_dry_run_limits_collection_to_rendered_nodeids(self): def test_dry_run_rejects_shell_metacharacters_in_rendered_target(self): targets = _function_body(L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList") - expected_target = ( - "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" - ) + expected_target = "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" metacharacter_target = f"{expected_target};touch${{IFS}}/tmp/infra-dry-run" parsed_targets = [metacharacter_target.split(maxsplit=1)[0]] self.assertNotEqual(parsed_targets, [expected_target]) From 25484b0cf7a36d30e77176e252fa5e1aba80a978 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Wed, 19 Aug 2026 01:41:07 +0800 Subject: [PATCH 28/34] ci: address dry-run review feedback Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 13 +++ jenkins/L0_Test.groovy | 3 + .../tools/test_infra_dry_run_pipeline.py | 90 +------------------ 3 files changed, 20 insertions(+), 86 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 360321e34435..3539c21cff40 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -154,12 +154,21 @@ def CBTS_COVERAGE = "cbts_coverage" def DISABLE_CBTS = "disable_cbts" @Field def INFRA_DRY_RUN = "infra_dry_run" +// Source-level dry-run invariants are covered by +// tests/unittest/tools/test_infra_dry_run_pipeline.py. Keep those checks scoped +// to behavior owned by this mode when refactoring the surrounding pipeline. // Kill switch for CBTS per-test coverage; official post-merge pipeline only, single-GPU stages only in Phase 1. @Field def ENABLE_CBTS_COVERAGE = true @Field def OSS_COMPLIANCE_FILE_CHANGED = "oss_compliance_file_changed" +// InfraDryRun is an opt-in parameter supplied by the externally configured +// L0_Stability job. It is intentionally absent from normal job definitions and +// therefore defaults to false when not provided. Operators use an exact commit +// SHA, InfraDryRun=true, and no explicit stage_list; the downstream phase name +// is forced empty in launchInfraDryRunTestJob so all synthetic GPU-count stages +// stay in the one helper run. boolean infraDryRun = params.InfraDryRun?.toString()?.toBoolean() ?: false def testFilter = [ @@ -1663,6 +1672,8 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor ] } + // An explicit empty override is meaningful for the infrastructure dry run: + // it keeps L0_Test from splitting \d+_GPUs stages into a separate helper. if (!additionalParameters.containsKey('testPhase2StageName') && env.testPhase2StageName) { parameters += [ 'testPhase2StageName': env.testPhase2StageName, @@ -1713,6 +1724,8 @@ def launchInfraDryRunTestJob(pipeline, arch, testFilter, globalVars, platform, i String testFilterJson = writeJSON returnText: true, json: testFilter def additionalParameters = [ 'testFilter': testFilterJson, + // Keep this explicitly empty so the Single-GPU helper executes both + // single- and multi-GPU synthetic stages in one dry-run invocation. 'testPhase2StageName': '', ] + imageParameters stage("[Test-${arch}-Single-GPU] Remote Run") { diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 025633dc8260..070c4dbf938b 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -104,6 +104,9 @@ def LINUX_AARCH64_CONFIG = "linux_aarch64" @Field def INFRA_DRY_RUN_TEST_CONTEXT = "infra_dry_run" +// Source-level dry-run invariants are covered by +// tests/unittest/tools/test_infra_dry_run_pipeline.py. Keep those checks scoped +// to behavior owned by this mode when refactoring the surrounding pipeline. @Field def BUILD_CONFIGS = [ diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 20d02689c900..a35b260dad56 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -13,10 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import re -import subprocess -import tempfile import unittest from pathlib import Path @@ -24,7 +21,6 @@ L0_TEST = (REPO_ROOT / "jenkins" / "L0_Test.groovy").read_text() L0_PARENT = (REPO_ROOT / "jenkins" / "L0_MergeRequest.groovy").read_text() SLURM_RUN = (REPO_ROOT / "jenkins" / "scripts" / "slurm_run.sh").read_text() -SLURM_INSTALL_PATH = REPO_ROOT / "jenkins" / "scripts" / "slurm_install.sh" CHECK_TEST_LIST = (REPO_ROOT / "scripts" / "check_test_list.py").read_text() BENCHMARK_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "test_infra_dry_run_benchmark.py" CONFTEST = (REPO_ROOT / "tests" / "integration" / "defs" / "conftest.py").read_text() @@ -69,6 +65,9 @@ def _pytest_capture_mode(args: list[str], initial_mode: str) -> str: class InfraDryRunPipelineTest(unittest.TestCase): + # Keep source-level assertions scoped to behavior introduced by the dry-run + # feature so unrelated Jenkins refactors do not break this regression suite. + def test_dedicated_context_selects_one_standard_pytest_case(self): database = DRY_RUN_DB_PATH.read_text() self.assertTrue(BENCHMARK_PATH.is_file()) @@ -203,19 +202,14 @@ def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self): ) self.assertIn('${infraDryRun ? "export infraDryRun=true" : ""}', body) self.assertNotIn("export infraDryRun=$infraDryRun", body) - self.assertIn('pytestUtil = "$llmSrcNode/tensorrt_llm/llmapi/trtllm-llmapi-launch"', body) self.assertIn("if (!isInfraDryRun() && (disaggMultiNodeMode || aggMultiNodeMode))", body) self.assertNotIn("test_infra_dry_run_benchmark.py", body) - self.assertNotIn("MASTER_ADDR", body) - self.assertNotIn("MASTER_PORT", body) dispatch = _function_body(L0_TEST, "runLLMTestlistOnSlurm", "INFRA_DRY_RUN") self.assertIn("if (isInfraDryRun() || nodeCount > 1 || runWithSbatch)", dispatch) self.assertIn( 'if [[ "${infraDryRun:-false}" == "true" || "$stageName" != *Disagg* ]]', SLURM_RUN, ) - for rank_variable in ("RANK=", "LOCAL_RANK=", "WORLD_SIZE=", "MASTER_ADDR"): - self.assertNotIn(rank_variable, SLURM_RUN) def test_parent_uses_parameter_only_and_explicit_non_fail_fast_helper(self): setup = L0_PARENT[ @@ -228,7 +222,6 @@ def test_parent_uses_parameter_only_and_explicit_non_fail_fast_helper(self): self.assertNotIn("JOB_NAME", setup.splitlines()[0]) self.assertIn('"L0_Test-${arch}-Single-GPU"', helper) self.assertNotIn('"L0_Test-${arch}-Multi-GPU"', helper) - self.assertIn(", false, false, globalVars,", helper) self.assertIn("'testPhase2StageName': ''", helper) self.assertIn("additionalParameters.containsKey('testPhase2StageName')", launch) self.assertIn( @@ -237,19 +230,7 @@ def test_parent_uses_parameter_only_and_explicit_non_fail_fast_helper(self): ) self.assertIn("parallelJobs.failFast = effectiveFailFast", L0_PARENT) - def test_normal_gating_and_result_collection_remain_in_place(self): - stages_start = L0_PARENT.index("def launchStages") - stages = L0_PARENT[stages_start : L0_PARENT.index("\npipeline {", stages_start)] - for arch in ("x86_64", "SBSA"): - normal_single = stages.index(f'testStageName = "[Test-{arch}-Single-GPU] Remote Run"') - approval = stages.index( - f'currentBuild.description?.contains("Require {arch} Multi-GPU Testing")', - normal_single, - ) - normal_multi = stages.index(f'launchJob(pipeline, "L0_Test-{arch}-Multi-GPU"', approval) - self.assertLess(normal_single, approval) - self.assertLess(approval, normal_multi) - + def test_standard_result_collection_remains_in_place(self): upload = _function_body(L0_TEST, "uploadResults", "runIsolatedTests") self.assertNotIn("isInfraDryRun", upload) self.assertIn( @@ -260,69 +241,6 @@ def test_normal_gating_and_result_collection_remain_in_place(self): self.assertIn("collectTestResults(this, testFilter, globalVars)", always_block) self.assertNotIn("testFilter[INFRA_DRY_RUN]", always_block) - def test_slurm_artifact_download_replaces_existing_archive(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_path = Path(temp_dir) - archive_path = temp_path / "TensorRT-LLM.tar.gz" - wget_record_path = temp_path / "wget-output-path" - tar_record_path = temp_path / "tar-input-path" - archive_path.write_text("stale\n") - - script = r""" -source "$SLURM_INSTALL_PATH" -retry_command() { - if [[ "$1" == "--timeout" ]]; then shift 2; fi - "$@" -} -wget() { - local output_path="" - while (( "$#" )); do - if [[ "$1" == "-O" ]]; then output_path="$2"; shift 2; else shift; fi - done - printf 'fresh\n' > "$output_path" - printf '%s\n' "$output_path" > "$WGET_RECORD_PATH" -} -tar() { - [[ "$1" == "-zxf" ]] - [[ "$2" == "$EXPECTED_ARCHIVE_PATH" ]] - grep -qx fresh "$2" - mkdir -p "$resourcePathNode/TensorRT-LLM/src" - printf '%s\n' "$2" > "$TAR_RECORD_PATH" -} -apt-get() { :; } -nvidia-smi() { :; } -pip3() { :; } -python3() { :; } -export -f pip3 wget -slurm_install_setup -""" - env = { - **os.environ, - "SLURM_INSTALL_PATH": str(SLURM_INSTALL_PATH), - "resourcePathNode": temp_dir, - "tarName": archive_path.name, - "llmTarfile": "https://artifacts.example/TensorRT-LLM.tar.gz", - "SLURM_LOCALID": "0", - "SLURM_JOB_ID": "123", - "SLURM_NODEID": "0", - "pytestCommand": "pytest", - "stageName": "test-stage", - "HOST_NODE_NAME": "test-host", - "EXPECTED_ARCHIVE_PATH": str(archive_path), - "WGET_RECORD_PATH": str(wget_record_path), - "TAR_RECORD_PATH": str(tar_record_path), - } - subprocess.run( - ["bash", "-c", script], check=True, capture_output=True, text=True, env=env - ) - - expected_tmp = f"{archive_path}.tmp.123.0" - self.assertEqual(wget_record_path.read_text(), f"{expected_tmp}\n") - self.assertEqual(archive_path.read_text(), "fresh\n") - self.assertEqual(tar_record_path.read_text(), f"{archive_path}\n") - self.assertFalse(Path(f"{archive_path}.1").exists()) - self.assertFalse(Path(expected_tmp).exists()) - if __name__ == "__main__": unittest.main() From dcbcc045387ac5fd06b58e23c7a7f268fecb6e49 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:20:22 +0800 Subject: [PATCH 29/34] tests: annotate infra dry-run pipeline cases Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- .../tools/test_infra_dry_run_pipeline.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index a35b260dad56..13d69e89c561 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -68,7 +68,7 @@ class InfraDryRunPipelineTest(unittest.TestCase): # Keep source-level assertions scoped to behavior introduced by the dry-run # feature so unrelated Jenkins refactors do not break this regression suite. - def test_dedicated_context_selects_one_standard_pytest_case(self): + def test_dedicated_context_selects_one_standard_pytest_case(self) -> None: database = DRY_RUN_DB_PATH.read_text() self.assertTrue(BENCHMARK_PATH.is_file()) self.assertTrue(BENCHMARK_PATH.name.startswith("test_")) @@ -79,7 +79,7 @@ def test_dedicated_context_selects_one_standard_pytest_case(self): verify_l0 = _function_body(CHECK_TEST_LIST, "verify_l0_test_lists", "verify_qa_test_lists") self.assertIn("pytest --test-list={test_list}", verify_l0) - def test_platform_runner_uses_standard_pytest_results_and_reporting(self): + def test_platform_runner_uses_standard_pytest_results_and_reporting(self) -> None: body = _function_body(L0_TEST, "runLLMTestlistOnPlatformImpl", "runLLMTestlistOnPlatform") self.assertIn( "effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList", body @@ -100,7 +100,7 @@ def test_platform_runner_uses_standard_pytest_results_and_reporting(self): self.assertNotIn("test_infra_dry_run_benchmark.py", body) self.assertNotIn("positionalTest", L0_TEST) - def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): + def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self) -> None: prepared = _function_body(L0_TEST, "runInfraDryRunInPreparedWorkspace", "runLLMDocBuild") docs = _function_body(L0_TEST, "runLLMDocBuild", "launchTestListCheck") for call in ("renderTestDB(", "processShardTestList(", "getPytestBaseCommandLine("): @@ -126,7 +126,7 @@ def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self): self.assertTrue(any(arg.startswith("--s3-upload-path=") for arg in upload_args)) self.assertEqual(_pytest_capture_mode(upload_args, initial_mode="no"), "fd") - def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(self): + def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(self) -> None: body = _function_body(L0_TEST, "runLLMAgentFlowTest", "launchTestListCheck") self.assertIn("if (isInfraDryRun())", body) infra_pytest_install = ( @@ -149,14 +149,14 @@ def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(sel self.assertLess(body.index("\n return", dry_runner), normal_root) self.assertIn('def agentFlowRoot = "${LLM_ROOT}/agent-flow"', body) - def test_prepared_workspace_sets_dry_environment_before_collection(self): + def test_prepared_workspace_sets_dry_environment_before_collection(self) -> None: body = _function_body(L0_TEST, "runInfraDryRunInPreparedWorkspace", "runLLMDocBuild") dry_environment = 'withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"])' self.assertEqual(body.count(dry_environment), 1) self.assertLess(body.index(dry_environment), body.index("processShardTestList(")) self.assertLess(body.index(dry_environment), body.index("${pytestCommand.join")) - def test_dry_run_limits_collection_to_rendered_nodeids(self): + def test_dry_run_limits_collection_to_rendered_nodeids(self) -> None: targets = _function_body(L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList") preprocessing = _function_body(L0_TEST, "processShardTestList", "isValidSlurmJobId") expected_target = "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" @@ -169,7 +169,7 @@ def test_dry_run_limits_collection_to_rendered_nodeids(self): preprocessing, ) - def test_dry_run_rejects_shell_metacharacters_in_rendered_target(self): + def test_dry_run_rejects_shell_metacharacters_in_rendered_target(self) -> None: targets = _function_body(L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList") expected_target = "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" metacharacter_target = f"{expected_target};touch${{IFS}}/tmp/infra-dry-run" @@ -177,7 +177,7 @@ def test_dry_run_rejects_shell_metacharacters_in_rendered_target(self): self.assertNotEqual(parsed_targets, [expected_target]) self.assertIn("if (targets != [expectedTarget])", targets) - def test_dry_run_conftest_does_not_require_product_bindings(self): + def test_dry_run_conftest_does_not_require_product_bindings(self) -> None: dry_guard = '_INFRA_DRY_RUN = os.environ.get("TRTLLM_INFRA_DRY_RUN", "").lower() == "true"' self.assertIn(dry_guard, CONFTEST) guard_start = CONFTEST.index(dry_guard) @@ -190,7 +190,7 @@ def test_dry_run_conftest_does_not_require_product_bindings(self): self.assertIn("else:", fallback) self.assertNotIn("from .perf.gpu_clock_lock import GPUClockLock", fallback) - def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self): + def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self) -> None: body = _function_body(L0_TEST, "runLLMTestlistWithSbatch", "runLLMTestlistOnSlurm") self.assertIn( "effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList", body @@ -211,7 +211,7 @@ def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self): SLURM_RUN, ) - def test_parent_uses_parameter_only_and_explicit_non_fail_fast_helper(self): + def test_parent_uses_parameter_only_and_explicit_non_fail_fast_helper(self) -> None: setup = L0_PARENT[ L0_PARENT.index("boolean infraDryRun =") : L0_PARENT.index("String reuseBuild =") ] @@ -230,7 +230,7 @@ def test_parent_uses_parameter_only_and_explicit_non_fail_fast_helper(self): ) self.assertIn("parallelJobs.failFast = effectiveFailFast", L0_PARENT) - def test_standard_result_collection_remains_in_place(self): + def test_standard_result_collection_remains_in_place(self) -> None: upload = _function_body(L0_TEST, "uploadResults", "runIsolatedTests") self.assertNotIn("isInfraDryRun", upload) self.assertIn( From e56175315ef2c8afaf8bba16163e9a5575b8674c Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:41:00 +0800 Subject: [PATCH 30/34] ci: merge duplicate dry-run setup branches Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 3539c21cff40..5f002cefaa0e 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -376,14 +376,11 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars) testFilter[(MULTI_GPU_FILE_CHANGED)] = false testFilter[(ONLY_ONE_GROUP_CHANGED)] = "" testFilter[(AUTO_TRIGGER_TAG_LIST)] = [] + pipeline.echo("CBTS is skipped for the infrastructure dry run.") } else { testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) - } - if (testFilter[INFRA_DRY_RUN]) { - pipeline.echo("CBTS is skipped for the infrastructure dry run.") - } else { testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars) // Decide CBTS coverage eligibility here so L0_Test only consumes the propagated flag. testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE && (env.JOB_NAME ==~ /.*PostMerge.*/) From 70266942f72b968d26e287ea897ffb312046bd04 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:19:09 +0800 Subject: [PATCH 31/34] ci: preserve Slurm agent dry-run coverage Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 2 +- tests/unittest/tools/test_infra_dry_run_pipeline.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 070c4dbf938b..d408a478b3a7 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2736,7 +2736,7 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p backoffMs: 60L * 1000L, ] - if (isInfraDryRun() || nodeCount > 1 || runWithSbatch) { + if (nodeCount > 1 || runWithSbatch) { runLLMTestlistWithSbatch(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, nodeCount, skipInstallWheel, cpver, postTag, useClusterDurations, attemptPlacementContext, slurmRetryContext) } else { runLLMTestlistWithAgent(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, skipInstallWheel, cpver, postTag, useClusterDurations, attemptPlacementContext, slurmRetryContext) diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 13d69e89c561..87a24497f221 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -205,7 +205,11 @@ def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self) -> None: self.assertIn("if (!isInfraDryRun() && (disaggMultiNodeMode || aggMultiNodeMode))", body) self.assertNotIn("test_infra_dry_run_benchmark.py", body) dispatch = _function_body(L0_TEST, "runLLMTestlistOnSlurm", "INFRA_DRY_RUN") - self.assertIn("if (isInfraDryRun() || nodeCount > 1 || runWithSbatch)", dispatch) + self.assertIn("if (nodeCount > 1 || runWithSbatch)", dispatch) + self.assertNotIn("isInfraDryRun() || nodeCount", dispatch) + agent = _function_body(L0_TEST, "runLLMTestlistWithAgent", "executeLLMTestOnSlurm") + self.assertIn("runInDockerOnNodeMultiStage", agent) + self.assertIn("runInEnrootOnNode", agent) self.assertIn( 'if [[ "${infraDryRun:-false}" == "true" || "$stageName" != *Disagg* ]]', SLURM_RUN, From 963a54a13a679465c6c25c521329946cc8ef7045 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:34:10 +0800 Subject: [PATCH 32/34] ci: simplify infrastructure dry-run stage coverage Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 23 +- jenkins/L0_Test.groovy | 154 +++---------- tests/integration/defs/conftest.py | 21 +- .../tools/test_infra_dry_run_pipeline.py | 214 ++---------------- .../tools/test_infra_dry_run_pytest.py | 78 +++---- 5 files changed, 98 insertions(+), 392 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 5f002cefaa0e..80d33886f60e 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -154,23 +154,12 @@ def CBTS_COVERAGE = "cbts_coverage" def DISABLE_CBTS = "disable_cbts" @Field def INFRA_DRY_RUN = "infra_dry_run" -// Source-level dry-run invariants are covered by -// tests/unittest/tools/test_infra_dry_run_pipeline.py. Keep those checks scoped -// to behavior owned by this mode when refactoring the surrounding pipeline. // Kill switch for CBTS per-test coverage; official post-merge pipeline only, single-GPU stages only in Phase 1. @Field def ENABLE_CBTS_COVERAGE = true @Field def OSS_COMPLIANCE_FILE_CHANGED = "oss_compliance_file_changed" -// InfraDryRun is an opt-in parameter supplied by the externally configured -// L0_Stability job. It is intentionally absent from normal job definitions and -// therefore defaults to false when not provided. Operators use an exact commit -// SHA, InfraDryRun=true, and no explicit stage_list; the downstream phase name -// is forced empty in launchInfraDryRunTestJob so all synthetic GPU-count stages -// stay in the one helper run. -boolean infraDryRun = params.InfraDryRun?.toString()?.toBoolean() ?: false - def testFilter = [ (REUSE_TEST): gitlabParamsFromBot.get(REUSE_TEST, null), (REUSE_STAGE_LIST): trimForStageList(gitlabParamsFromBot.get(REUSE_STAGE_LIST, null)?.tokenize(',')), @@ -191,7 +180,7 @@ def testFilter = [ (CBTS_RESULT): null, (CBTS_COVERAGE): false, (DISABLE_CBTS): gitlabParamsFromBot.get((DISABLE_CBTS), false), - (INFRA_DRY_RUN): infraDryRun, + (INFRA_DRY_RUN): (params.InfraDryRun?.toString()?.toBoolean() ?: false), ] String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) @@ -1669,8 +1658,7 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor ] } - // An explicit empty override is meaningful for the infrastructure dry run: - // it keeps L0_Test from splitting \d+_GPUs stages into a separate helper. + // Preserve an explicit empty phase override from the dry-run helper. if (!additionalParameters.containsKey('testPhase2StageName') && env.testPhase2StageName) { parameters += [ 'testPhase2StageName': env.testPhase2StageName, @@ -1721,8 +1709,7 @@ def launchInfraDryRunTestJob(pipeline, arch, testFilter, globalVars, platform, i String testFilterJson = writeJSON returnText: true, json: testFilter def additionalParameters = [ 'testFilter': testFilterJson, - // Keep this explicitly empty so the Single-GPU helper executes both - // single- and multi-GPU synthetic stages in one dry-run invocation. + // Keep all synthetic GPU-count stages in this helper. 'testPhase2StageName': '', ] + imageParameters stage("[Test-${arch}-Single-GPU] Remote Run") { @@ -2301,9 +2288,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } }]} - // A dry acceptance run must finish both architecture tracks so one failure - // does not erase the remaining coverage. Preserve the existing fail-fast - // behavior for every normal pipeline. + // Preserve both architecture tracks during dry acceptance. def effectiveFailFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast parallelJobs.failFast = effectiveFailFast pipeline.parallel parallelJobs diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index d408a478b3a7..80bcd35bc45c 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -104,9 +104,6 @@ def LINUX_AARCH64_CONFIG = "linux_aarch64" @Field def INFRA_DRY_RUN_TEST_CONTEXT = "infra_dry_run" -// Source-level dry-run invariants are covered by -// tests/unittest/tools/test_infra_dry_run_pipeline.py. Keep those checks scoped -// to behavior owned by this mode when refactoring the surrounding pipeline. @Field def BUILD_CONFIGS = [ @@ -577,10 +574,8 @@ def getInfraDryRunPytestTargets(testListPath) { return [] } - // --test-list filters items only after pytest has imported every file under - // the current directory. The CPU-only dry runners intentionally do not - // install the product test environment, so limit collection to the rendered - // synthetic nodeids as positional arguments as well. + // --test-list filters after collection, so also pass the exact rendered + // nodeid positionally to avoid importing unrelated product tests. def targets = readFile(file: testListPath).readLines() .collect { it.trim().split(/\s+/, 2)[0] } .findAll { it.contains("::") } @@ -1729,10 +1724,12 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG def disaggMultiNodeMode = stageName.contains("Disagg-PerfSanity") def aggMultiNodeMode = !disaggMultiNodeMode && nodeCount > 1 && stageName.contains("PerfSanity") def infraDryRun = isInfraDryRun() - def effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList - def effectiveSplitId = infraDryRun ? 1 : splitId - def effectiveSplits = infraDryRun ? 1 : splits - def effectivePerfMode = infraDryRun ? false : perfMode + if (infraDryRun) { + testList = INFRA_DRY_RUN_TEST_CONTEXT + splitId = 1 + splits = 1 + perfMode = false + } Utils.exec(pipeline, script: "env | sort && pwd && ls -alh") @@ -1761,7 +1758,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG def scriptInstallPathNode = "${jobWorkspace}/${jobUID}-slurm_install.sh" def scriptBashUtilsLocalPath = "${llmSrcLocal}/jenkins/scripts/bash_utils.sh" def scriptBashUtilsPathNode = "${jobWorkspace}/${jobUID}-bash_utils.sh" - def testListPathNode = "${jobWorkspace}/${effectiveTestList}.txt" + def testListPathNode = "${jobWorkspace}/${testList}.txt" def waivesListPathNode = "${jobWorkspace}/waives.txt" def waivesListPathLocal = infraDryRun ? "${llmPath}/infra_dry_run_waives.txt" @@ -1823,7 +1820,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG // if the line cannot be split by "=", just ignore that line. def makoOptsJson = transformMakoArgsToJson(["Mako options:"] + makoArgs) String clusterNameForDurations = useClusterDurations ? partition.clusterName.replaceAll('[^a-zA-Z0-9]', '_') : null - def testListPathLocal = renderTestDB(pipeline, effectiveTestList, llmSrcLocal, stageName, makoOptsJson, clusterNameForDurations) + def testListPathLocal = renderTestDB(pipeline, testList, llmSrcLocal, stageName, makoOptsJson, clusterNameForDurations) // Copy the test list atomically. A retry that reuses a still-active job // re-copies over ${testListPathNode} while that job may be reading it via // --test-list; scp truncates-then-streams, so a concurrent read could see a @@ -1940,8 +1937,8 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG def extraArgs = [ "--test-list=$testListPathNode", "--splitting-algorithm least_duration", - "--splits $effectiveSplits", - "--group $effectiveSplitId", + "--splits $splits", + "--group $splitId", *clusterDurationsArgsNode, ] if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { @@ -1955,7 +1952,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG llmSrcNode, stageName, waivesListPathNode, - effectivePerfMode, + perfMode, jobWorkspace, "$jobWorkspace/.coveragerc", pytestUtil, @@ -2135,7 +2132,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG export llmTarfile=$llmTarfile export llmSrcNode=$llmSrcNode export stageName=$stageName - export perfMode=$effectivePerfMode + export perfMode=$perfMode ${infraDryRun ? "export infraDryRun=true" : ""} export resourcePathNode=$resourcePathNode export pytestCommand="$pytestCommand" @@ -3754,71 +3751,6 @@ def echoNodeAndGpuInfo(pipeline, stageName) pipeline.echo "HOST_NODE_NAME = ${hostNodeName} ; GPU_UUIDS = ${gpuUuids} ; STAGE_NAME = ${stageName}" } -def runInfraDryRunInPreparedWorkspace(pipeline, String llmSrc, String stageName) -{ - def outputPath = "${WORKSPACE}/${stageName}" - def waivesFile = "${llmSrc}/infra_dry_run_waives.txt" - def coverageConfigFile = "${llmSrc}/infra_dry_run.coveragerc" - - sh "rm -rf ${outputPath} && mkdir -p ${outputPath} && : > ${waivesFile} && : > ${coverageConfigFile}" - // Test-list preprocessing performs a pytest --collect-only pass before the - // final pytest invocation. Both passes load defs/conftest.py, so keep the - // dry-run import guard active for the entire prepared-workspace flow. - withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"]) { - def testDBList = renderTestDB( - pipeline, - INFRA_DRY_RUN_TEST_CONTEXT, - llmSrc, - stageName, - ) - def preprocessedLists = processShardTestList( - llmSrc, - testDBList, - 1, - 1, - false, - ) - if (preprocessedLists.regularCount < 1) { - error "No infrastructure dry-run benchmark was selected for ${stageName}" - } - - def extraArgs = [] - if (ENABLE_UPLOAD_TEST_RESULTS) { - def uploadPath = UPLOAD_PATH.replaceFirst("sw-tensorrt-generic/llm-artifacts/LLM/", "") - extraArgs += [ - "--capture=fd", - "--s3-upload-path=${uploadPath}/${stageName}", - "--s3-upload-mode=deferred", - ] - } - def pytestCommand = getPytestBaseCommandLine( - llmSrc, - stageName, - waivesFile, - false, - outputPath, - coverageConfigFile, - "", - extraArgs, - ) - pytestCommand += [ - "--test-list=${preprocessedLists.regular}", - ] - pytestCommand += getInfraDryRunPytestTargets(preprocessedLists.regular) - - withCredentials([ - string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), - string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), - string(credentialsId: 'llm_evaltool_repo_url', variable: 'EVALTOOL_REPO_URL') - ]) { - sh """ - cd ${llmSrc}/tests/integration/defs && \ - ${pytestCommand.join(" ")} - """ - } - } -} - def runLLMDocBuild(pipeline, config) { // Step 1: cloning source code @@ -3844,11 +3776,6 @@ def runLLMDocBuild(pipeline, config) trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmSrc} && pip3 install -r requirements-dev.txt") trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmPath} && pip3 install --force-reinstall --no-deps TensorRT-LLM/tensorrt_llm-*.whl") - if (isInfraDryRun()) { - runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, "CPU-Build_Docs") - return - } - // Step 3: build doc trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y doxygen python3-pip graphviz") @@ -3894,22 +3821,6 @@ def runLLMAgentFlowTest(pipeline, stageName) trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) trtllm_utils.llmExecStepWithRetry(pipeline, script: "git config --global --add safe.directory \"*\"") - // Dry acceptance validates the shared benchmark/JUnit/upload path for every - // selected stage; it must not install or execute this product test suite. - if (isInfraDryRun()) { - def llmSrc = sh(script: "realpath ${LLM_ROOT}", returnStdout: true).trim() - // The build pod used by AgentFlow does not run the normal TRT-LLM test - // environment setup, so install only the pytest, test-list, and upload - // tooling consumed by the shared dry-run adapter. Keep the AgentFlow - // package and its product dependencies out of this path. - trtllm_utils.llmExecStepWithRetry( - pipeline, - script: "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout pytest-unused-fixtures mako boto3" - ) - runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName) - return - } - def agentFlowRoot = "${LLM_ROOT}/agent-flow" // Install agent-flow with its test extras (pytest, pytest-asyncio) and the @@ -3917,6 +3828,18 @@ def runLLMAgentFlowTest(pipeline, stageName) // These resolve from the container's default PyPI mirror. trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${agentFlowRoot} && pip3 install -e \".[test]\"") + if (isInfraDryRun()) { + // Keep the normal environment and reporting path, but replace product tests. + sh """ + rm -rf "${agentFlowRoot}/tests" && \ + mkdir -p "${agentFlowRoot}/tests" && \ + printf '%s\\n' \ + 'def test_infra_dry_run_placeholder():' \ + ' pass' \ + > "${agentFlowRoot}/tests/test_infra_dry_run_placeholder.py" + """ + } + sh "mkdir -p ${WORKSPACE}/${stageName}" // test_workflow_entrypoint_modules_run_without_import_warnings is deselected @@ -4955,10 +4878,12 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def noIsolateTests = false def rerunFailed = false def infraDryRun = isInfraDryRun() - def effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList - def effectiveSplitId = infraDryRun ? 1 : splitId - def effectiveSplits = infraDryRun ? 1 : splits - def effectivePerfMode = infraDryRun ? false : perfMode + if (infraDryRun) { + testList = INFRA_DRY_RUN_TEST_CONTEXT + splitId = 1 + splits = 1 + perfMode = false + } // When useClusterDurations is set, use a per-cluster durations file keyed on // partition.clusterName (e.g. "oci-hsg", "dlcluster"). This lets each cluster @@ -4975,7 +4900,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO clusterDurationsArgs = ["--durations-path ${clusterDurationsPath}"] } - def testDBList = renderTestDB(pipeline, effectiveTestList, llmSrc, stageName, null, clusterNameForDurations) + def testDBList = renderTestDB(pipeline, testList, llmSrc, stageName, null, clusterNameForDurations) def waivesFilePath = infraDryRun ? "${llmSrc}/infra_dry_run_waives.txt" : "${llmSrc}/tests/integration/test_lists/waives.txt" @@ -4993,14 +4918,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } // Process shard test list and create separate files for regular and isolate tests - def preprocessedLists = processShardTestList( - llmSrc, - testDBList, - effectiveSplitId, - effectiveSplits, - effectivePerfMode, - clusterDurationsPath, - ) + def preprocessedLists = processShardTestList(llmSrc, testDBList, splitId, splits, perfMode, clusterDurationsPath) // Test Coverage def TRTLLM_WHL_PATH = sh(returnStdout: true, script: "pip3 show tensorrt_llm | grep Location | cut -d ' ' -f 2").replaceAll("\\s","") @@ -5045,7 +4963,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO llmSrc, stageName, waivesFilePath, - effectivePerfMode, + perfMode, "${WORKSPACE}/${stageName}", coverageConfigFile, "", // pytestUtil @@ -5167,7 +5085,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO error "Some tests terminated unexpectedly, please check the test report." } - if (effectivePerfMode) { + if (perfMode) { // Only PyTorch perf stages remain; the TensorRT perf baseline was removed. basePerfFilename = "base_perf_pytorch.csv" basePerfPath = "${llmSrc}/tests/integration/defs/perf/${basePerfFilename}" diff --git a/tests/integration/defs/conftest.py b/tests/integration/defs/conftest.py index 663619a7e840..4edd7d09f975 100644 --- a/tests/integration/defs/conftest.py +++ b/tests/integration/defs/conftest.py @@ -48,6 +48,11 @@ # is harmless. from test_common import session_prefetcher_hooks as _prefetch_hooks +from tensorrt_llm.bindings import ipc_nvls_supported +from tensorrt_llm.llmapi.mpi_session import get_mpi_world_size + +from .perf.gpu_clock_lock import GPUClockLock +from .perf.session_data_writer import SessionDataWriter from .test_list_parser import (TestCorrectionMode, apply_waives, get_test_name_corrections_v2, handle_corrections, modify_by_test_list, preprocess_test_list_lines) @@ -61,22 +66,6 @@ except ImportError: trt_environment = None -_INFRA_DRY_RUN = os.environ.get("TRTLLM_INFRA_DRY_RUN", "").lower() == "true" -if _INFRA_DRY_RUN: - - def ipc_nvls_supported(): - return False - - def get_mpi_world_size(): - return 1 - -else: - from tensorrt_llm.bindings import ipc_nvls_supported - from tensorrt_llm.llmapi.mpi_session import get_mpi_world_size - - from .perf.gpu_clock_lock import GPUClockLock - from .perf.session_data_writer import SessionDataWriter - # Logger logger = logging.getLogger(__name__) diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 87a24497f221..21cd2b1f6972 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re import unittest from pathlib import Path @@ -21,12 +20,9 @@ L0_TEST = (REPO_ROOT / "jenkins" / "L0_Test.groovy").read_text() L0_PARENT = (REPO_ROOT / "jenkins" / "L0_MergeRequest.groovy").read_text() SLURM_RUN = (REPO_ROOT / "jenkins" / "scripts" / "slurm_run.sh").read_text() -CHECK_TEST_LIST = (REPO_ROOT / "scripts" / "check_test_list.py").read_text() -BENCHMARK_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "test_infra_dry_run_benchmark.py" -CONFTEST = (REPO_ROOT / "tests" / "integration" / "defs" / "conftest.py").read_text() -DRY_RUN_DB_PATH = ( +DRY_RUN_DB = ( REPO_ROOT / "tests" / "integration" / "test_lists" / "test-db" / "infra_dry_run.yml" -) +).read_text() def _function_body(source: str, name: str, next_name: str) -> str: @@ -34,216 +30,52 @@ def _function_body(source: str, name: str, next_name: str) -> str: return source[start : source.index(f"def {next_name}", start + len(name))] -def _conditional_workflow_properties(function_body: str) -> set[str]: - conditions = re.findall(r"\bif\s*\(([^)]*)\)", function_body) - return { - identifier - for condition in conditions - for identifier in re.findall(r"\b[A-Z][A-Z0-9_]+\b", condition) - } - - -def _top_level_workflow_properties(source: str) -> set[str]: - return set(re.findall(r"(?m)^(?:def\s+)?([A-Z][A-Z0-9_]*)\s*=", source)) - - -def _groovy_list_values_after(source: str, assignment: str) -> list[str]: - assignment_start = source.index(assignment) - list_start = source.index("[", assignment_start) - list_end = source.index("]", list_start) - return re.findall(r'"([^"]+)"', source[list_start:list_end]) - - -def _pytest_capture_mode(args: list[str], initial_mode: str) -> str: - capture_mode = initial_mode - for arg in args: - if arg == "-s": - capture_mode = "no" - elif arg.startswith("--capture="): - capture_mode = arg.split("=", 1)[1] - return capture_mode - - class InfraDryRunPipelineTest(unittest.TestCase): - # Keep source-level assertions scoped to behavior introduced by the dry-run - # feature so unrelated Jenkins refactors do not break this regression suite. - - def test_dedicated_context_selects_one_standard_pytest_case(self) -> None: - database = DRY_RUN_DB_PATH.read_text() - self.assertTrue(BENCHMARK_PATH.is_file()) - self.assertTrue(BENCHMARK_PATH.name.startswith("test_")) - self.assertEqual(database.count("::test_"), 1) - self.assertIn("infra_dry_run:", database) - self.assertIn("test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark", database) - self.assertNotIn("infra_dry_run_benchmark.py", CHECK_TEST_LIST) - verify_l0 = _function_body(CHECK_TEST_LIST, "verify_l0_test_lists", "verify_qa_test_lists") - self.assertIn("pytest --test-list={test_list}", verify_l0) - - def test_platform_runner_uses_standard_pytest_results_and_reporting(self) -> None: - body = _function_body(L0_TEST, "runLLMTestlistOnPlatformImpl", "runLLMTestlistOnPlatform") - self.assertIn( - "effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList", body - ) - self.assertIn("effectiveSplitId = infraDryRun ? 1 : splitId", body) - self.assertIn("effectiveSplits = infraDryRun ? 1 : splits", body) - self.assertIn("effectivePerfMode = infraDryRun ? false : perfMode", body) - self.assertIn("getPytestBaseCommandLine(", body) - self.assertIn("--test-list=${preprocessedLists.regular}", body) - self.assertIn( - "pytestCommand += getInfraDryRunPytestTargets(preprocessedLists.regular)", - body, - ) - self.assertIn("rerunFailedTests(", body) - self.assertIn("runIsolatedTests(", body) - self.assertIn("generateRerunReport(", body) - self.assertIn('testEnvironment += ["stageName=${stageName}"]', body) - self.assertNotIn("test_infra_dry_run_benchmark.py", body) - self.assertNotIn("positionalTest", L0_TEST) - - def test_docs_use_prepared_standard_pytest_workspace_only_for_dry_run(self) -> None: - prepared = _function_body(L0_TEST, "runInfraDryRunInPreparedWorkspace", "runLLMDocBuild") - docs = _function_body(L0_TEST, "runLLMDocBuild", "launchTestListCheck") - for call in ("renderTestDB(", "processShardTestList(", "getPytestBaseCommandLine("): - self.assertIn(call, prepared) - self.assertIn("--test-list=${preprocessedLists.regular}", prepared) - self.assertIn( - "pytestCommand += getInfraDryRunPytestTargets(preprocessedLists.regular)", - prepared, - ) - self.assertIn('withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"])', prepared) - self.assertNotIn("test_infra_dry_run_benchmark.py", prepared) - self.assertLess(docs.index("if (isInfraDryRun())"), docs.index("make html")) - conditional_properties = _conditional_workflow_properties(prepared) - self.assertTrue(conditional_properties) - self.assertLessEqual( - conditional_properties, - _top_level_workflow_properties(L0_TEST), - ) - self.assertIn("def runLLMDocBuild(pipeline, config)", L0_TEST) - self.assertIn("runLLMDocBuild(pipeline, config=VANILLA_CONFIG)", L0_TEST) - self.assertIn('runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, "CPU-Build_Docs")', docs) - upload_args = _groovy_list_values_after(prepared, "extraArgs += [") - self.assertTrue(any(arg.startswith("--s3-upload-path=") for arg in upload_args)) - self.assertEqual(_pytest_capture_mode(upload_args, initial_mode="no"), "fd") - - def test_agent_flow_uses_prepared_standard_pytest_workspace_only_for_dry_run(self) -> None: - body = _function_body(L0_TEST, "runLLMAgentFlowTest", "launchTestListCheck") - self.assertIn("if (isInfraDryRun())", body) - infra_pytest_install = ( - "pip3 install 'pytest<9.1' pytest-csv pytest-split pytest-timeout " - "pytest-unused-fixtures mako boto3" - ) - self.assertIn(infra_pytest_install, body) - self.assertIn("runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)", body) - self.assertLess( - body.index(infra_pytest_install), - body.index("runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)"), - ) - self.assertLess(body.index("if (isInfraDryRun())"), body.index("pip3 install -e")) - dry_guard = body.index("if (isInfraDryRun())") - realpath = body.index("realpath ${LLM_ROOT}") - dry_runner = body.index("runInfraDryRunInPreparedWorkspace(pipeline, llmSrc, stageName)") - normal_root = body.index('def agentFlowRoot = "${LLM_ROOT}/agent-flow"') - self.assertLess(dry_guard, realpath) - self.assertLess(realpath, dry_runner) - self.assertLess(body.index("\n return", dry_runner), normal_root) - self.assertIn('def agentFlowRoot = "${LLM_ROOT}/agent-flow"', body) - - def test_prepared_workspace_sets_dry_environment_before_collection(self) -> None: - body = _function_body(L0_TEST, "runInfraDryRunInPreparedWorkspace", "runLLMDocBuild") - dry_environment = 'withEnv(["stageName=${stageName}", "TRTLLM_INFRA_DRY_RUN=true"])' - self.assertEqual(body.count(dry_environment), 1) - self.assertLess(body.index(dry_environment), body.index("processShardTestList(")) - self.assertLess(body.index(dry_environment), body.index("${pytestCommand.join")) - - def test_dry_run_limits_collection_to_rendered_nodeids(self) -> None: + def test_dry_run_allows_only_the_synthetic_pytest_target(self) -> None: + expected = "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" targets = _function_body(L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList") - preprocessing = _function_body(L0_TEST, "processShardTestList", "isValidSlurmJobId") - expected_target = "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" - self.assertIn("if (!isInfraDryRun())", targets) - self.assertIn('.findAll { it.contains("::") }', targets) - self.assertIn(f'"{expected_target}"', targets) - self.assertIn("if (targets != [expectedTarget])", targets) - self.assertIn( - "testListCmd += getInfraDryRunPytestTargets(cleanedTestDBList)", - preprocessing, - ) - def test_dry_run_rejects_shell_metacharacters_in_rendered_target(self) -> None: - targets = _function_body(L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList") - expected_target = "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" - metacharacter_target = f"{expected_target};touch${{IFS}}/tmp/infra-dry-run" - parsed_targets = [metacharacter_target.split(maxsplit=1)[0]] - self.assertNotEqual(parsed_targets, [expected_target]) + self.assertEqual(DRY_RUN_DB.count("::test_"), 1) + self.assertIn(expected, DRY_RUN_DB) + self.assertIn(f'expectedTarget =\n "{expected}"', targets) self.assertIn("if (targets != [expectedTarget])", targets) + self.assertIn("return targets", targets) - def test_dry_run_conftest_does_not_require_product_bindings(self) -> None: - dry_guard = '_INFRA_DRY_RUN = os.environ.get("TRTLLM_INFRA_DRY_RUN", "").lower() == "true"' - self.assertIn(dry_guard, CONFTEST) - guard_start = CONFTEST.index(dry_guard) - normal_import = CONFTEST.index( - "from tensorrt_llm.bindings import ipc_nvls_supported", guard_start - ) - fallback = CONFTEST[guard_start:normal_import] - self.assertIn("def ipc_nvls_supported():", fallback) - self.assertIn("def get_mpi_world_size():", fallback) - self.assertIn("else:", fallback) - self.assertNotIn("from .perf.gpu_clock_lock import GPUClockLock", fallback) - - def test_slurm_keeps_only_dry_gates_needed_by_the_standard_runner(self) -> None: - body = _function_body(L0_TEST, "runLLMTestlistWithSbatch", "runLLMTestlistOnSlurm") - self.assertIn( - "effectiveTestList = infraDryRun ? INFRA_DRY_RUN_TEST_CONTEXT : testList", body - ) - self.assertIn("String[] taskArgs = getNodeArgs(", body) - self.assertIn( - "pytestCommandParts += getInfraDryRunPytestTargets(testListPathLocal)", - body, - ) - self.assertIn('${infraDryRun ? "export infraDryRun=true" : ""}', body) - self.assertNotIn("export infraDryRun=$infraDryRun", body) - self.assertIn("if (!isInfraDryRun() && (disaggMultiNodeMode || aggMultiNodeMode))", body) - self.assertNotIn("test_infra_dry_run_benchmark.py", body) + def test_slurm_dry_run_preserves_agent_and_sbatch_dispatch(self) -> None: dispatch = _function_body(L0_TEST, "runLLMTestlistOnSlurm", "INFRA_DRY_RUN") + agent = _function_body(L0_TEST, "runLLMTestlistWithAgent", "executeLLMTestOnSlurm") + sbatch = _function_body(L0_TEST, "runLLMTestlistWithSbatch", "runLLMTestlistOnSlurm") + self.assertIn("if (nodeCount > 1 || runWithSbatch)", dispatch) self.assertNotIn("isInfraDryRun() || nodeCount", dispatch) - agent = _function_body(L0_TEST, "runLLMTestlistWithAgent", "executeLLMTestOnSlurm") self.assertIn("runInDockerOnNodeMultiStage", agent) self.assertIn("runInEnrootOnNode", agent) + self.assertIn("testList = INFRA_DRY_RUN_TEST_CONTEXT", sbatch) + self.assertIn("pytestCommandParts += getInfraDryRunPytestTargets", sbatch) self.assertIn( 'if [[ "${infraDryRun:-false}" == "true" || "$stageName" != *Disagg* ]]', SLURM_RUN, ) - def test_parent_uses_parameter_only_and_explicit_non_fail_fast_helper(self) -> None: - setup = L0_PARENT[ - L0_PARENT.index("boolean infraDryRun =") : L0_PARENT.index("String reuseBuild =") - ] + def test_parent_dry_run_is_opt_in_non_fail_fast_and_collects_results(self) -> None: helper = _function_body(L0_PARENT, "launchInfraDryRunTestJob", "launchStages") - launch = _function_body(L0_PARENT, "launchJob", "launchInfraDryRunTestJob") - self.assertIn("params.InfraDryRun?.toString()?.toBoolean()", setup) - self.assertIn("(INFRA_DRY_RUN): infraDryRun", setup) - self.assertNotIn("JOB_NAME", setup.splitlines()[0]) + + self.assertIn( + "(INFRA_DRY_RUN): (params.InfraDryRun?.toString()?.toBoolean() ?: false)", + L0_PARENT, + ) self.assertIn('"L0_Test-${arch}-Single-GPU"', helper) - self.assertNotIn('"L0_Test-${arch}-Multi-GPU"', helper) self.assertIn("'testPhase2StageName': ''", helper) - self.assertIn("additionalParameters.containsKey('testPhase2StageName')", launch) self.assertIn( "def effectiveFailFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast", L0_PARENT, ) self.assertIn("parallelJobs.failFast = effectiveFailFast", L0_PARENT) - - def test_standard_result_collection_remains_in_place(self) -> None: - upload = _function_body(L0_TEST, "uploadResults", "runIsolatedTests") - self.assertNotIn("isInfraDryRun", upload) + self.assertIn("collectTestResults(this, testFilter, globalVars)", L0_PARENT) self.assertIn( - 'junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml")', L0_TEST + 'junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml")', + L0_TEST, ) - always_start = L0_PARENT.index(" always {") - always_block = L0_PARENT[always_start : L0_PARENT.index(" stages {", always_start)] - self.assertIn("collectTestResults(this, testFilter, globalVars)", always_block) - self.assertNotIn("testFilter[INFRA_DRY_RUN]", always_block) if __name__ == "__main__": diff --git a/tests/unittest/tools/test_infra_dry_run_pytest.py b/tests/unittest/tools/test_infra_dry_run_pytest.py index cc4da0f5181b..e8dd886c4db7 100644 --- a/tests/unittest/tools/test_infra_dry_run_pytest.py +++ b/tests/unittest/tools/test_infra_dry_run_pytest.py @@ -34,6 +34,32 @@ BENCHMARK = importlib.util.module_from_spec(_SPEC) _SPEC.loader.exec_module(BENCHMARK) +_FAKE_TORCH_SOURCE = textwrap.dedent( + """ + float32 = "float32" + class Device: + def __init__(self, kind, index=None): + self.type, self.index = kind, index + class Scalar: + def item(self): return True + class Tensor: + def __init__(self, value, dtype, device): + self.value, self.dtype, self.device = value, dtype, device + def all(self): return Scalar() + def device(kind, index=None): return Device(kind, index) + def full(_shape, value, *, dtype, device): return Tensor(value, dtype, device) + def matmul(left, _right): return Tensor(4.0, left.dtype, left.device) + def full_like(tensor, value): return Tensor(value, tensor.dtype, tensor.device) + def isfinite(tensor): return tensor + def equal(left, right): return left.value == right.value + """ +) + + +def _write_benchmark_sandbox(root: Path) -> None: + (root / BENCHMARK_PATH.name).write_text(BENCHMARK_PATH.read_text()) + (root / "torch.py").write_text(_FAKE_TORCH_SOURCE) + class _Scalar: def __init__(self, value): @@ -128,30 +154,8 @@ def test_cuda_path_does_not_fall_back_to_cpu(self): def test_standard_pytest_collection_selects_only_the_requested_context(self): with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) - (root / BENCHMARK_PATH.name).write_text(BENCHMARK_PATH.read_text()) + _write_benchmark_sandbox(root) (root / "test_product.py").write_text("def test_product(): pass\n") - (root / "torch.py").write_text( - textwrap.dedent( - """ - float32 = "float32" - class Device: - def __init__(self, kind, index=None): - self.type, self.index = kind, index - class Scalar: - def item(self): return True - class Tensor: - def __init__(self, value, dtype, device): - self.value, self.dtype, self.device = value, dtype, device - def all(self): return Scalar() - def device(kind, index=None): return Device(kind, index) - def full(_shape, value, *, dtype, device): return Tensor(value, dtype, device) - def matmul(left, _right): return Tensor(4.0, left.dtype, left.device) - def full_like(tensor, value): return Tensor(value, tensor.dtype, tensor.device) - def isfinite(tensor): return tensor - def equal(left, right): return left.value == right.value - """ - ) - ) (root / "conftest.py").write_text( textwrap.dedent( """ @@ -184,42 +188,20 @@ def pytest_collection_modifyitems(config, items): [sys.executable, "-m", "pytest", f"--test-list={test_list}", "-vv"], cwd=root, env=env, - check=True, capture_output=True, text=True, ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertIn(expected, result.stdout) self.assertIn("1 passed, 1 deselected", result.stdout) def test_positional_nodeid_does_not_import_unrelated_product_test(self): with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) - (root / BENCHMARK_PATH.name).write_text(BENCHMARK_PATH.read_text()) + _write_benchmark_sandbox(root) (root / "test_unrelated_product.py").write_text( 'raise RuntimeError("unrelated product test imported")\n' ) - (root / "torch.py").write_text( - textwrap.dedent( - """ - float32 = "float32" - class Device: - def __init__(self, kind, index=None): - self.type, self.index = kind, index - class Scalar: - def item(self): return True - class Tensor: - def __init__(self, value, dtype, device): - self.value, self.dtype, self.device = value, dtype, device - def all(self): return Scalar() - def device(kind, index=None): return Device(kind, index) - def full(_shape, value, *, dtype, device): return Tensor(value, dtype, device) - def matmul(left, _right): return Tensor(4.0, left.dtype, left.device) - def full_like(tensor, value): return Tensor(value, tensor.dtype, tensor.device) - def isfinite(tensor): return tensor - def equal(left, right): return left.value == right.value - """ - ) - ) (root / "conftest.py").write_text( textwrap.dedent( """ @@ -244,10 +226,10 @@ def pytest_addoption(parser): ], cwd=root, env={**os.environ, "stageName": "CPU-Generic-x86-1"}, - check=True, capture_output=True, text=True, ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertIn(target, result.stdout) self.assertNotIn("test_unrelated_product.py", result.stdout) From 95a2fdccd77d1803d0da3806cd90faa3f06b43f5 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:21:16 +0800 Subject: [PATCH 33/34] ci: skip docs junit when no results are expected Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_Test.groovy | 2 +- tests/unittest/tools/test_infra_dry_run_pipeline.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 80bcd35bc45c..64522b27e634 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6366,7 +6366,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) // pod-launch attempt; isFinalAttempt suppresses synthetic stage-fail XML // and junit() on intermediate retryable infra failures. stage("[${key}] Run") { - cacheErrorAndUploadResult("${key}", values[1], {}, !isInfraDryRun(), attemptTag, isFinalAttempt, retryContext) + cacheErrorAndUploadResult("${key}", values[1], {}, true, attemptTag, isFinalAttempt, retryContext) } }]]} diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 21cd2b1f6972..9c62e35814c5 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -72,8 +72,11 @@ def test_parent_dry_run_is_opt_in_non_fail_fast_and_collects_results(self) -> No ) self.assertIn("parallelJobs.failFast = effectiveFailFast", L0_PARENT) self.assertIn("collectTestResults(this, testFilter, globalVars)", L0_PARENT) + + def test_docs_skip_junit_after_a_successful_build(self) -> None: self.assertIn( - 'junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml")', + 'cacheErrorAndUploadResult("${key}", values[1], {}, true, attemptTag, ' + "isFinalAttempt, retryContext)", L0_TEST, ) From 0c242d769e78731f792868fa1f677db13aba6604 Mon Sep 17 00:00:00 2001 From: Abby Wei <18545893+mzweilz@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:56:06 +0800 Subject: [PATCH 34/34] ci: skip MR diff lookups during dry run Signed-off-by: Abby Wei <18545893+mzweilz@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 6 ++++-- .../unittest/tools/test_infra_dry_run_pipeline.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 80d33886f60e..59fe9f0f3000 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -732,7 +732,8 @@ def requireMultiGpuApprovalLabel(pipeline, globalVars, String arch) { def getMergeRequestChangedFileList(pipeline, globalVars) { def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) - if (env.alternativeTRT || + if ((params.InfraDryRun?.toString()?.toBoolean() ?: false) || + env.alternativeTRT || isOfficialPostMergeJob || runMode == "nightly_release") { pipeline.echo("Force set changed file list to empty list.") @@ -768,7 +769,8 @@ def getMergeRequestOneFileChanges(pipeline, globalVars, filePath) { // Note: This function intentionally propagates exceptions to the caller. // If there is an error to get the changed file diff, skip merging the waive list. def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) - if (env.alternativeTRT || + if ((params.InfraDryRun?.toString()?.toBoolean() ?: false) || + env.alternativeTRT || isOfficialPostMergeJob || runMode == "nightly_release") { pipeline.echo("Force set changed file diff to empty string.") diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py index 9c62e35814c5..5687293e6520 100644 --- a/tests/unittest/tools/test_infra_dry_run_pipeline.py +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -73,6 +73,21 @@ def test_parent_dry_run_is_opt_in_non_fail_fast_and_collects_results(self) -> No self.assertIn("parallelJobs.failFast = effectiveFailFast", L0_PARENT) self.assertIn("collectTestResults(this, testFilter, globalVars)", L0_PARENT) + def test_dry_run_skips_merge_request_diff_lookups(self) -> None: + infra_dry_run_check = "(params.InfraDryRun?.toString()?.toBoolean() ?: false)" + changed_files = _function_body( + L0_PARENT, "getMergeRequestChangedFileList", "getMergeRequestOneFileChanges" + ) + one_file_diff = _function_body( + L0_PARENT, "getMergeRequestOneFileChanges", "getAutoTriggerTagList" + ) + + for body, empty_result in ((changed_files, "return []"), (one_file_diff, 'return ""')): + with self.subTest(empty_result=empty_result): + self.assertIn(f"if ({infra_dry_run_check} ||", body) + self.assertLess(body.index(infra_dry_run_check), body.index("def githubPrApiUrl")) + self.assertLess(body.index(empty_result), body.index("def githubPrApiUrl")) + def test_docs_skip_junit_after_a_successful_build(self) -> None: self.assertIn( 'cacheErrorAndUploadResult("${key}", values[1], {}, true, attemptTag, '