From 0e8c677ac3873406f23fdc8b268b2844b2305320 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Mon, 3 Aug 2026 16:01:45 -0700 Subject: [PATCH 01/13] feat(openengine): add stub gRPC protocol Signed-off-by: Connor Carpenter --- .github/CODEOWNERS | 1 + docker/Dockerfile.multi | 2 +- requirements-openengine.txt | 12 ++++ setup.py | 2 + tensorrt_llm/commands/serve.py | 65 ++++++++++++++------ tensorrt_llm/grpc/openengine/README.md | 46 ++++++++++++++ tensorrt_llm/grpc/openengine/__init__.py | 8 +++ tensorrt_llm/grpc/openengine/server.py | 78 ++++++++++++++++++++++++ tensorrt_llm/grpc/smg/server.py | 9 +++ 9 files changed, 204 insertions(+), 19 deletions(-) create mode 100644 requirements-openengine.txt create mode 100644 tensorrt_llm/grpc/openengine/README.md create mode 100644 tensorrt_llm/grpc/openengine/__init__.py create mode 100644 tensorrt_llm/grpc/openengine/server.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 514ea98aa8a7..e22732225961 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -482,6 +482,7 @@ /pyproject.toml @NVIDIA/trt-llm-oss-compliance /requirements-dev.txt @NVIDIA/trt-llm-oss-compliance /requirements-grpc-smg.txt @NVIDIA/trt-llm-oss-compliance +/requirements-openengine.txt @NVIDIA/trt-llm-oss-compliance /requirements.txt @NVIDIA/trt-llm-oss-compliance /setup.py @NVIDIA/trt-llm-oss-compliance /tests/unittest/api_stability/ @NVIDIA/trt-llm-noncommitted-api-review-committee diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index 14798a5a354a..ed0748e5b5a0 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -124,7 +124,7 @@ COPY scripts scripts COPY tensorrt_llm tensorrt_llm COPY triton_kernels triton_kernels COPY 3rdparty 3rdparty -COPY setup.py requirements.txt requirements-dev.txt requirements-grpc-smg.txt constraints.txt LICENSE README.md ./ +COPY setup.py requirements.txt requirements-dev.txt requirements-grpc-smg.txt requirements-openengine.txt constraints.txt LICENSE README.md ./ ENV CCACHE_DIR=/root/.cache/ccache # Build the TRT-LLM wheel diff --git a/requirements-openengine.txt b/requirements-openengine.txt new file mode 100644 index 000000000000..3ac09eceb170 --- /dev/null +++ b/requirements-openengine.txt @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Source: https://github.com/ai-dynamo/openengine/releases/tag/v0.1.0 +# BSR module: https://buf.build/openengine/openengine +# Immutable BSR commit: 768a93c7b44e40f28c692ad0b471a8f2 +--extra-index-url https://buf.build/gen/python +openengine-openengine-grpc-python==1.67.1.2.20260730172104+768a93c7b44e +openengine-openengine-protocolbuffers-python==31.1.0.2.20260730172104+768a93c7b44e +openengine-openengine-protocolbuffers-pyi==31.1.0.2.20260730172104+768a93c7b44e +grpcio>=1.67.1,<2 +protobuf>=6.31.1,<7 diff --git a/setup.py b/setup.py index d1d4dc643d98..205735a4daae 100644 --- a/setup.py +++ b/setup.py @@ -147,6 +147,7 @@ def has_ext_modules(self): devel_deps, _ = parse_requirements( Path("requirements-dev-windows.txt" if on_windows else "requirements-dev.txt")) +openengine_deps, _ = parse_requirements(Path("requirements-openengine.txt")) mx_deps = ["modelexpress>=0.4.1,<0.6.0"] # Gateway protocol adapters are opt-in extras: the default installation must # not carry any gateway protobuf package. Each gateway owns a dedicated @@ -563,6 +564,7 @@ def get_build_state_options(): scripts=['tensorrt_llm/llmapi/trtllm-llmapi-launch'], extras_require={ "devel": devel_deps + grpc_smg_deps, + "openengine": openengine_deps, "mx": mx_deps, "grpc-smg": grpc_smg_deps, }, diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 7becb894ce86..d07abcee09df 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -1205,10 +1205,13 @@ def launch_visual_gen_server( "--grpc", is_flag=True, default=False, - help="Run gRPC server instead of OpenAI HTTP server. " - "gRPC server accepts pre-tokenized requests and returns raw token IDs. " - "Requires the tensorrt_llm[grpc-smg] extra.", + help="Run the selected gRPC protocol instead of the OpenAI HTTP server.", status="prototype") +@stability_option("--grpc-protocol", + type=click.Choice(["smg", "openengine"]), + default="smg", + help="Protocol used when --grpc is enabled.", + status="prototype") @stability_option( "--served_model_name", type=str, @@ -1276,9 +1279,9 @@ def serve(model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str], agent_types: Optional[str], video_pruning_rate: Optional[float], telemetry: bool, custom_module_dirs: list[Path], chat_template: Optional[str], allow_request_chat_template: bool, - middleware: tuple[str, ...], grpc: bool, enable_visual_gen: bool, - served_model_name: Optional[str], visual_gen_args: Optional[str], - report_addr: Optional[str]): + middleware: tuple[str, ...], grpc: bool, grpc_protocol: str, + enable_visual_gen: bool, served_model_name: Optional[str], + visual_gen_args: Optional[str], report_addr: Optional[str]) -> None: """Running an OpenAI API compatible server MODEL: model name | HF checkpoint path | TensorRT engine path @@ -1292,6 +1295,9 @@ def serve(model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str], "to the 'pytorch' backend. See " "https://github.com/NVIDIA/TensorRT-LLM/issues/15638 for details.") + if not grpc and grpc_protocol != "smg": + raise click.UsageError("--grpc-protocol requires --grpc") + if moe_cluster_parallel_size is not None: logger.warning( "--moe_cluster_parallel_size / --cluster_size is deprecated and " @@ -1439,6 +1445,10 @@ def _serve_llm(): media_io_kwargs=parsed_media_io_kwargs) if grpc: + if num_serve_frontends != 1: + raise click.UsageError( + "--num_serve_frontends must be 1 when --grpc is enabled.") + # gRPC mode: launch gRPC server instead of OpenAI HTTP server # Check for unsupported arguments that are silently ignored in gRPC mode unsupported_args = { @@ -1466,18 +1476,34 @@ def _serve_llm(): f"Argument '{name}' is not supported when running in gRPC mode. " f"The gRPC server is designed for use with external routers that handle " f"these features (e.g., tool parsing, chat templates).") - if find_spec("smg_grpc_proto") is None: - raise ValueError( - "gRPC serving with the SMG protocol requires the optional " - "'smg-grpc-proto' package. Install it with: " - 'pip install "tensorrt_llm[grpc-smg]"') - - from tensorrt_llm.grpc.smg.server import launch_smg_server - - launch_smg_server(host, - port, - llm_args, - served_model_name=served_model_name) + if grpc_protocol == "smg": + if find_spec("smg_grpc_proto") is None: + raise ValueError( + "gRPC serving with the SMG protocol requires the optional " + "'smg-grpc-proto' package. Install it with: " + 'pip install "tensorrt_llm[grpc-smg]"') + + from tensorrt_llm.grpc.smg.server import launch_smg_server + + launch_smg_server(host, + port, + llm_args, + served_model_name=served_model_name) + else: + try: + from tensorrt_llm.grpc.openengine.server import \ + launch_server as launch_grpc_server + except ModuleNotFoundError as error: + if error.name == "grpc" or ( + error.name and error.name.startswith("openengine")): + raise click.ClickException( + "OpenEngine support requires the optional Python " + "bindings. Install them with `python -m pip install " + "--extra-index-url https://buf.build/gen/python " + "\"tensorrt_llm[openengine]\"`.") from error + raise + + launch_grpc_server(host, port) else: # Default: launch OpenAI HTTP server launch_server( @@ -1519,6 +1545,9 @@ def _serve_visual_gen(): "--report_addr is only supported for the OpenAI HTTP server, not " f"the {'gRPC' if grpc else 'VisualGen'} server.") if is_visual_gen: + if grpc: + raise click.UsageError( + "--grpc is not supported by the VisualGen server") _serve_visual_gen() else: _serve_llm() diff --git a/tensorrt_llm/grpc/openengine/README.md b/tensorrt_llm/grpc/openengine/README.md new file mode 100644 index 000000000000..da8d1f915bd5 --- /dev/null +++ b/tensorrt_llm/grpc/openengine/README.md @@ -0,0 +1,46 @@ + + + +# TensorRT-LLM OpenEngine stub server + +`trtllm-serve` can expose an experimental OpenEngine gRPC server instead of its normal OpenAI HTTP server. SMG remains the default gRPC protocol. + +Install the optional Python bindings from the Buf Schema Registry: + +```bash +python -m pip install \ + --extra-index-url https://buf.build/gen/python \ + "tensorrt_llm[openengine]" +``` + +Then select OpenEngine when starting the gRPC server: + +```bash +trtllm-serve \ + --grpc \ + --grpc-protocol openengine \ + --host 0.0.0.0 \ + --port 50051 +``` + +Existing `--grpc` invocations continue to select SMG. OpenEngine and VisualGen cannot be enabled together. + +This initial integration is a protocol stub. Every OpenEngine RPC returns gRPC status `UNIMPLEMENTED`; no request reaches the TensorRT-LLM engine. OpenEngine and SMG are independent protocol integrations. This integration does not make a replacement or convergence decision between them. + +## Dependency provenance + +The schema source is the Apache-2.0-licensed [`ai-dynamo/openengine`](https://github.com/ai-dynamo/openengine) repository at signed Git tag [`v0.1.0`](https://github.com/ai-dynamo/openengine/releases/tag/v0.1.0). That release maps to the public [`buf.build/openengine/openengine`](https://buf.build/openengine/openengine) module at immutable BSR commit `768a93c7b44e40f28c692ad0b471a8f2`. + +The BSR generated the pinned wheels from that module commit: + +| Package | Generator | Version | SHA-256 | +| --- | --- | --- | --- | +| `openengine-openengine-grpc-python` | [`grpc/python`](https://buf.build/grpc/python) | `1.67.1.2.20260730172104+768a93c7b44e` | `1485aed9799c4eb9367d1a261ca5cc5319f1e9b8d950ac98a26f3cb3641b8cf6` | +| `openengine-openengine-protocolbuffers-python` | [`protocolbuffers/python`](https://buf.build/protocolbuffers/python) | `31.1.0.2.20260730172104+768a93c7b44e` | `6eae12c3d8d06147fccf608da9772d6391139031fabdafdb7cf4c71a19c1f25e` | +| `openengine-openengine-protocolbuffers-pyi` | [`protocolbuffers/pyi`](https://buf.build/protocolbuffers/pyi) | `31.1.0.2.20260730172104+768a93c7b44e` | `8b0a054dbdaaa67459b3fa4786f13d8f6f4d30cf30be325f5416dbd97aba46a6` | + +Buf documents the package naming and version format in its [Python-generated SDK guide](https://buf.build/docs/bsr/generated-sdks/python/). The final version segment is the BSR commit prefix. The exact requirements are pinned in `requirements-openengine.txt`. + +## Maintenance boundary + +The OpenEngine contributor community owns this adapter, its tests, protocol version updates, and integration bugs. TensorRT-LLM internal APIs do not provide compatibility guarantees to protocol adapters. Adapter updates must follow core runtime changes and must not block normal TensorRT-LLM development or releases. diff --git a/tensorrt_llm/grpc/openengine/__init__.py b/tensorrt_llm/grpc/openengine/__init__.py new file mode 100644 index 000000000000..d1049d02ea13 --- /dev/null +++ b/tensorrt_llm/grpc/openengine/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""OpenEngine gRPC integration for TensorRT-LLM.""" + +from .server import OpenEngineServer, launch_server + +__all__ = ["OpenEngineServer", "launch_server"] diff --git a/tensorrt_llm/grpc/openengine/server.py b/tensorrt_llm/grpc/openengine/server.py new file mode 100644 index 000000000000..e5713ead57f3 --- /dev/null +++ b/tensorrt_llm/grpc/openengine/server.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""OpenEngine gRPC server lifecycle for TensorRT-LLM.""" + +import grpc +import uvloop +from openengine.v1 import openengine_pb2_grpc + +from tensorrt_llm.logger import logger + +__all__ = ["OpenEngineServer", "launch_server"] + + +def _format_bind_address(host: str, port: int) -> str: + """Format a host and port as a gRPC bind address.""" + if ":" in host and not (host.startswith("[") and host.endswith("]")): + host = f"[{host}]" + return f"{host}:{port}" + + +class OpenEngineServer: + """OpenEngine gRPC server with intentionally unimplemented RPCs. + + Args: + host: Interface on which the server listens. + port: Port on which the server listens. Use zero to select a free port. + """ + + def __init__(self, host: str, port: int) -> None: + self.host = host + self.port = port + self._server = grpc.aio.server() + openengine_pb2_grpc.add_InferenceServicer_to_server( + openengine_pb2_grpc.InferenceServicer(), self._server + ) + openengine_pb2_grpc.add_ControlServicer_to_server( + openengine_pb2_grpc.ControlServicer(), self._server + ) + self._bind_address = _format_bind_address(host, port) + bound_port = self._server.add_insecure_port(self._bind_address) + if bound_port == 0: + raise RuntimeError(f"Failed to bind OpenEngine server to {self._bind_address}") + if port == 0: + self.port = bound_port + + async def start(self) -> None: + """Start accepting OpenEngine requests.""" + await self._server.start() + address = _format_bind_address(self.host, self.port) + logger.info(f"OpenEngine stub server started on {address}") + + async def stop(self, grace: float = 5.0) -> None: + """Stop accepting OpenEngine requests. + + Args: + grace: Maximum time in seconds to allow active RPCs to finish. + """ + await self._server.stop(grace=grace) + logger.info("OpenEngine stub server stopped") + + async def wait_for_termination(self) -> None: + """Wait until the OpenEngine server terminates.""" + await self._server.wait_for_termination() + + +def launch_server(host: str, port: int) -> None: + """Launch the dedicated OpenEngine gRPC server.""" + + async def serve() -> None: + server = OpenEngineServer(host=host, port=port) + try: + await server.start() + await server.wait_for_termination() + finally: + await server.stop() + + uvloop.run(serve()) diff --git a/tensorrt_llm/grpc/smg/server.py b/tensorrt_llm/grpc/smg/server.py index 608baa47428f..fc86a1ede202 100644 --- a/tensorrt_llm/grpc/smg/server.py +++ b/tensorrt_llm/grpc/smg/server.py @@ -48,6 +48,15 @@ def launch_smg_server( served_model_name: Model name returned by discovery RPCs. Defaults to the model path. """ + if not PROTOS_AVAILABLE: + raise click.ClickException( + "SMG gRPC support requires smg-grpc-proto. Install it with " + "`python -m pip install smg-grpc-proto`." + ) + + from .request_manager import GrpcRequestManager + from .servicer import TrtllmServiceServicer + try: from grpc_reflection.v1alpha import reflection except ModuleNotFoundError as e: From ba31a7351f41f64a7f4996c56439e17e72ad5c32 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Mon, 3 Aug 2026 16:50:02 -0700 Subject: [PATCH 02/13] fix(grpc): address protocol launch feedback Signed-off-by: Connor Carpenter --- tensorrt_llm/grpc/openengine/server.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/grpc/openengine/server.py b/tensorrt_llm/grpc/openengine/server.py index e5713ead57f3..1b82ce86ad52 100644 --- a/tensorrt_llm/grpc/openengine/server.py +++ b/tensorrt_llm/grpc/openengine/server.py @@ -65,7 +65,12 @@ async def wait_for_termination(self) -> None: def launch_server(host: str, port: int) -> None: - """Launch the dedicated OpenEngine gRPC server.""" + """Launch the dedicated OpenEngine gRPC server. + + Args: + host: Interface on which the server listens. + port: Port on which the server listens. + """ async def serve() -> None: server = OpenEngineServer(host=host, port=port) From c8731fc238a23e910fd7d4360cf7198427acd172 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Wed, 5 Aug 2026 17:01:31 -0700 Subject: [PATCH 03/13] fix(grpc): harden server startup and shutdown Signed-off-by: Connor Carpenter --- tensorrt_llm/commands/serve.py | 16 +++++++--------- tensorrt_llm/grpc/openengine/server.py | 19 ++++++++++++++++++- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index d07abcee09df..cc03245a7c91 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -1493,15 +1493,13 @@ def _serve_llm(): try: from tensorrt_llm.grpc.openengine.server import \ launch_server as launch_grpc_server - except ModuleNotFoundError as error: - if error.name == "grpc" or ( - error.name and error.name.startswith("openengine")): - raise click.ClickException( - "OpenEngine support requires the optional Python " - "bindings. Install them with `python -m pip install " - "--extra-index-url https://buf.build/gen/python " - "\"tensorrt_llm[openengine]\"`.") from error - raise + except ImportError as error: + raise click.ClickException( + f"Failed to import OpenEngine support: {error}. " + "Install the optional Python bindings with `python -m " + "pip install --extra-index-url " + "https://buf.build/gen/python " + "\"tensorrt_llm[openengine]\"`.") from error launch_grpc_server(host, port) else: diff --git a/tensorrt_llm/grpc/openengine/server.py b/tensorrt_llm/grpc/openengine/server.py index 1b82ce86ad52..641ff7a90eb0 100644 --- a/tensorrt_llm/grpc/openengine/server.py +++ b/tensorrt_llm/grpc/openengine/server.py @@ -3,6 +3,9 @@ """OpenEngine gRPC server lifecycle for TensorRT-LLM.""" +import asyncio +import signal + import grpc import uvloop from openengine.v1 import openengine_pb2_grpc @@ -74,9 +77,23 @@ def launch_server(host: str, port: int) -> None: async def serve() -> None: server = OpenEngineServer(host=host, port=port) + loop = asyncio.get_running_loop() + stop_event = asyncio.Event() + + def signal_handler() -> None: + logger.info("Received shutdown signal") + stop_event.set() + + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, signal_handler) + try: + logger.warning( + "OpenEngine protocol support is a stub: no model is loaded and " + "all RPCs return UNIMPLEMENTED." + ) await server.start() - await server.wait_for_termination() + await stop_event.wait() finally: await server.stop() From 3e01ca07191a3a064092d6cd1ed2841a55ed22c5 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Thu, 13 Aug 2026 07:14:55 -0700 Subject: [PATCH 04/13] fix(grpc): preserve rebased CLI metadata Signed-off-by: Connor Carpenter --- tensorrt_llm/grpc/smg/server.py | 9 --------- .../api_stability/references/trtllm_serve_cli.yaml | 9 +++++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tensorrt_llm/grpc/smg/server.py b/tensorrt_llm/grpc/smg/server.py index fc86a1ede202..608baa47428f 100644 --- a/tensorrt_llm/grpc/smg/server.py +++ b/tensorrt_llm/grpc/smg/server.py @@ -48,15 +48,6 @@ def launch_smg_server( served_model_name: Model name returned by discovery RPCs. Defaults to the model path. """ - if not PROTOS_AVAILABLE: - raise click.ClickException( - "SMG gRPC support requires smg-grpc-proto. Install it with " - "`python -m pip install smg-grpc-proto`." - ) - - from .request_manager import GrpcRequestManager - from .servicer import TrtllmServiceServicer - try: from grpc_reflection.v1alpha import reflection except ModuleNotFoundError as e: diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml index 4d4cabd2777f..516c5a441057 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml @@ -193,6 +193,15 @@ commands: is_flag: true flags: - "--grpc" + grpc_protocol: + type: Choice(['smg', 'openengine']) + default: smg + status: prototype + required: false + multiple: false + is_flag: false + flags: + - "--grpc-protocol" host: type: str default: localhost From 2f2031351c9b51525af7d4a635fe8799c799db78 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Thu, 20 Aug 2026 06:40:32 -0700 Subject: [PATCH 05/13] fix(cli): align gRPC protocol stability metadata Signed-off-by: Connor Carpenter --- tests/unittest/api_stability/references/trtllm_serve_cli.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml index 516c5a441057..5970af6dbb94 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml @@ -194,7 +194,7 @@ commands: flags: - "--grpc" grpc_protocol: - type: Choice(['smg', 'openengine']) + type: Choice(['openengine', 'smg']) default: smg status: prototype required: false From 0258d6a3e90c28998c744562d660f7bbb4322f15 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Fri, 21 Aug 2026 06:37:16 -0700 Subject: [PATCH 06/13] ci: update Docker images for OpenEngine Signed-off-by: Connor Carpenter --- jenkins/current_image_tags.properties | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index 646d65ad937a..8492934ac6a6 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm -LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-skip-tritondevel-202607311529-16970 -LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-skip-tritondevel-202607311529-16970 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202607311529-16970 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202607311529-16970 -LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202607311529-16970 +LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-skip-tritondevel-202608210335-17084 +LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-skip-tritondevel-202608210335-17084 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202608210335-17084 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202608210335-17084 +LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202608210335-17084 From 8476d0b852ae0b206fbee59d87b4373b44bcfc30 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Tue, 25 Aug 2026 08:44:07 -0700 Subject: [PATCH 07/13] ci: refresh OpenEngine Docker images Signed-off-by: Connor Carpenter --- jenkins/current_image_tags.properties | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index 8492934ac6a6..65576a162492 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm -LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-skip-tritondevel-202608210335-17084 -LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-skip-tritondevel-202608210335-17084 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202608210335-17084 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202608210335-17084 -LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202608210335-17084 +LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-skip-tritondevel-202608250021-17084 +LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-skip-tritondevel-202608250021-17084 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202608250021-17084 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202608250021-17084 +LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202608250021-17084 From 6b8daf286dc581154718610c1269d9c0ec551601 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Wed, 26 Aug 2026 14:51:49 -0700 Subject: [PATCH 08/13] ci: skip empty test selections Signed-off-by: Connor Carpenter --- jenkins/L0_Test.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index a4bd89661adb..26bf4c196baf 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -614,7 +614,9 @@ def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, du def shardTestList = [] - if (perfMode) { + if (!cleanedTestLines) { + echo "No tests selected - skipping pytest collection" + } else if (perfMode) { // In perfMode, skip pytest collection as it may cause errors with automatically generated testcases // Instead, use all tests from the original testDBList echo "Performance mode enabled - skipping pytest collection, using all tests from testDBList" @@ -4859,7 +4861,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } if (noRegularTests && noIsolateTests) { - error "No tests were executed for stage ${stageName}, please check the test list and test-db rendering result." + echo "No tests were selected for stage ${stageName}; skipping" } } finally { if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { From 8e9b0812f32d6f2f2640c9ac82ab43669526dd67 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Wed, 26 Aug 2026 16:19:00 -0700 Subject: [PATCH 09/13] Revert "ci: skip empty test selections" This reverts commit 6b8daf286dc581154718610c1269d9c0ec551601. Signed-off-by: Connor Carpenter --- jenkins/L0_Test.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 26bf4c196baf..a4bd89661adb 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -614,9 +614,7 @@ def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, du def shardTestList = [] - if (!cleanedTestLines) { - echo "No tests selected - skipping pytest collection" - } else if (perfMode) { + if (perfMode) { // In perfMode, skip pytest collection as it may cause errors with automatically generated testcases // Instead, use all tests from the original testDBList echo "Performance mode enabled - skipping pytest collection, using all tests from testDBList" @@ -4861,7 +4859,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } if (noRegularTests && noIsolateTests) { - echo "No tests were selected for stage ${stageName}; skipping" + error "No tests were executed for stage ${stageName}, please check the test list and test-db rendering result." } } finally { if (ENABLE_UPLOAD_TEST_RESULTS && !testFilter[(DETAILED_LOG)]) { From 30993b6e91c8e7ca1315c4869b2ed3d27ef68949 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Wed, 26 Aug 2026 16:20:13 -0700 Subject: [PATCH 10/13] fix(ci): detect Linux distribution without distro Signed-off-by: Connor Carpenter --- tests/integration/defs/sysinfo/get_sysinfo.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/integration/defs/sysinfo/get_sysinfo.py b/tests/integration/defs/sysinfo/get_sysinfo.py index 02e1a68eaef5..695c56564cae 100644 --- a/tests/integration/defs/sysinfo/get_sysinfo.py +++ b/tests/integration/defs/sysinfo/get_sysinfo.py @@ -110,11 +110,15 @@ def is_power(): def get_linux_distribution(): try: - import distro - return (distro.id(), distro.version(), distro.codename()) - except: + release = platform.freedesktop_os_release() + return ( + release.get("ID", "na"), + release.get("VERSION_ID", "na"), + release.get("VERSION_CODENAME", "na"), + ) + except OSError: logger.warning( - "Unable to use distro module, defaulting operating system to ('na', 'na', 'na')" + "Unable to read the operating system release, defaulting to ('na', 'na', 'na')" ) return ("na", "na", "na") From 15efc5f72abd985907ca4e1a3c6a5d870be5c931 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Wed, 26 Aug 2026 17:37:00 -0700 Subject: [PATCH 11/13] revert(ci): restore distro-based Linux detection Signed-off-by: Connor Carpenter --- tests/integration/defs/sysinfo/get_sysinfo.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/integration/defs/sysinfo/get_sysinfo.py b/tests/integration/defs/sysinfo/get_sysinfo.py index 695c56564cae..02e1a68eaef5 100644 --- a/tests/integration/defs/sysinfo/get_sysinfo.py +++ b/tests/integration/defs/sysinfo/get_sysinfo.py @@ -110,15 +110,11 @@ def is_power(): def get_linux_distribution(): try: - release = platform.freedesktop_os_release() - return ( - release.get("ID", "na"), - release.get("VERSION_ID", "na"), - release.get("VERSION_CODENAME", "na"), - ) - except OSError: + import distro + return (distro.id(), distro.version(), distro.codename()) + except: logger.warning( - "Unable to read the operating system release, defaulting to ('na', 'na', 'na')" + "Unable to use distro module, defaulting operating system to ('na', 'na', 'na')" ) return ("na", "na", "na") From 8e768df297800494f3d790a675ac32573fb3ab0b Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Thu, 20 Aug 2026 14:26:09 +0800 Subject: [PATCH 12/13] [None][fix] Pin distro for CI sysinfo detection (#18004) Signed-off-by: Yanchao Lu --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9c452a1835fd..c44c56324057 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,6 +3,7 @@ # before pytest collection so already-imported Diffusers modules match the # package files. 0.39.0 matches the runtime floor in requirements.txt. diffusers==0.39.0 +distro==1.9.0 boto3 einops # Reference implementation for KDA kernel parity tests. From 0db4eeff455ba2f16e153ce84ec48254d40853c8 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Thu, 27 Aug 2026 10:43:53 -0700 Subject: [PATCH 13/13] ci: refresh OpenEngine Docker images Signed-off-by: Connor Carpenter --- jenkins/current_image_tags.properties | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index afc883f1468a..b883aec89d79 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm -LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-skip-tritondevel-202608191543-17891 -LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-skip-tritondevel-202608191543-17891 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202608191543-17891 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202608191543-17891 -LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202608191543-17891 +LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-skip-tritondevel-202608271702-17084 +LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-skip-tritondevel-202608271702-17084 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202608271702-17084 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202608271702-17084 +LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202608271702-17084