From 485eaaf0da259464d033e1542caf579bf9a28469 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Tue, 25 Aug 2026 12:14:43 -0700 Subject: [PATCH 1/3] remove runtime check --- .github/workflows/executorch-build-linux.yml | 7 ------- py/torch_tensorrt/_compile.py | 5 ----- .../dynamo/runtime/meta_ops/register_meta_ops.py | 8 +++++++- py/torch_tensorrt/executorch/_export.py | 9 +-------- setup.py | 13 +++++++++++-- 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/.github/workflows/executorch-build-linux.yml b/.github/workflows/executorch-build-linux.yml index 1dacb1625f..f89fcf4f8d 100644 --- a/.github/workflows/executorch-build-linux.yml +++ b/.github/workflows/executorch-build-linux.yml @@ -82,13 +82,6 @@ jobs: python -m pip install pyyaml "executorch==1.4.1" export TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION="$(python -c 'import importlib.metadata; print(importlib.metadata.version("torch-tensorrt"))')" - # The downloaded wheel has to carry the C++ runtime. A wheel built with - # PYTHON_ONLY=1 has no libtorchtrt.so, which leaves - # ENABLED_FEATURES.torch_tensorrt_runtime False, and the export near the - # end of this script then fails after the Bazel builds have already run. - # Check it here so the wrong wheel is reported as the wrong wheel. - python -c 'from torch_tensorrt._features import ENABLED_FEATURES as f; assert f.torch_tensorrt_runtime, f' - # Bazel truncates a failing action's output at 1 MB by default and then # prints nothing but the size, which hid the real error behind # stdout ... exceeds maximum size of diff --git a/py/torch_tensorrt/_compile.py b/py/torch_tensorrt/_compile.py index fa70e13c46..30235d6829 100644 --- a/py/torch_tensorrt/_compile.py +++ b/py/torch_tensorrt/_compile.py @@ -1396,11 +1396,6 @@ def _write_external_tensor_data(executorch_program: Any, file_path: str) -> None def _save_as_executorch(exp_program: Any, file_path: str, **kwargs: Any) -> None: """Save an engine-bearing ExportedProgram as an ExecuTorch program.""" - if not ENABLED_FEATURES.torch_tensorrt_runtime: - raise RuntimeError( - "output_format='executorch' requires the Torch-TensorRT runtime " - "(torch_tensorrt_runtime). Reinstall torch_tensorrt with the runtime extension." - ) try: from torch_tensorrt.executorch import export except ImportError: diff --git a/py/torch_tensorrt/dynamo/runtime/meta_ops/register_meta_ops.py b/py/torch_tensorrt/dynamo/runtime/meta_ops/register_meta_ops.py index 15ae608e52..708e92463c 100644 --- a/py/torch_tensorrt/dynamo/runtime/meta_ops/register_meta_ops.py +++ b/py/torch_tensorrt/dynamo/runtime/meta_ops/register_meta_ops.py @@ -3,6 +3,7 @@ from typing import Any, Dict, List import torch +from torch_tensorrt._features import ENABLED_FEATURES from torch_tensorrt.dynamo.runtime._TorchTensorRTModule import TorchTensorRTModule logger = logging.getLogger(__name__) @@ -225,7 +226,6 @@ def fake_tensorrt_execute_engine( ) -@torch._library.register_fake_class("tensorrt::Engine") class FakeTRTEngine: def __init__(self, engine_info: List[str]) -> None: self.version = engine_info[torch.ops.tensorrt.ABI_TARGET_IDX()] @@ -304,6 +304,12 @@ def __getstate__(self) -> Any: pass +if ENABLED_FEATURES.torch_tensorrt_runtime: + # Registration is a side effect; retaining the decorator's return value is + # unnecessary and mypy correctly rejects rebinding a class name here. + torch._library.register_fake_class("tensorrt::Engine")(FakeTRTEngine) + + @torch.library.custom_op( # type: ignore[misc] "tensorrt::no_op_placeholder_for_execute_engine", mutates_args=() ) diff --git a/py/torch_tensorrt/executorch/_export.py b/py/torch_tensorrt/executorch/_export.py index e793e48751..835705da27 100644 --- a/py/torch_tensorrt/executorch/_export.py +++ b/py/torch_tensorrt/executorch/_export.py @@ -273,8 +273,8 @@ def _apply_weight_streaming_budget( """ from executorch.exir.backend.compile_spec_schema import CompileSpec from torch_tensorrt.executorch.partitioner import ( - normalize_weight_streaming_budget_per_engine, WEIGHT_STREAMING_BUDGET_COMPILE_SPEC_KEY, + normalize_weight_streaming_budget_per_engine, ) for name, specs in method_compile_specs.items(): @@ -438,18 +438,11 @@ def export( executorch.exir.EdgeProgramManager: The Edge program, ready for inspection, further transformation, or ``to_executorch()``. """ - from torch_tensorrt._features import ENABLED_FEATURES - if platform.system() != "Linux": raise ValueError( f"The executorch format is only supported on Linux, {platform.system()} " "is not a supported platform for this format" ) - if not ENABLED_FEATURES.torch_tensorrt_runtime: - raise RuntimeError( - "ExecuTorch export requires the Torch-TensorRT runtime " - "(torch_tensorrt_runtime). Reinstall torch_tensorrt with the runtime extension." - ) if inputs is not None and arg_inputs is not None: raise ValueError("inputs and arg_inputs are mutually exclusive.") arguments = inputs if inputs is not None else arg_inputs diff --git a/setup.py b/setup.py index 1c994f6f39..1ca036e59b 100644 --- a/setup.py +++ b/setup.py @@ -688,6 +688,16 @@ def run(self): package_data = {} +# ExecuTorch export uses the Python fake/meta kernels to represent TensorRT +# engine calls while lowering. They do not depend on the native Torch-TensorRT +# runtime, so Python-only wheels must ship them too. +packages += ["torch_tensorrt.dynamo.runtime.meta_ops"] +package_dir.update( + { + "torch_tensorrt.dynamo.runtime.meta_ops": "py/torch_tensorrt/dynamo/runtime/meta_ops", + } +) + if not (PY_ONLY or NO_TS): tensorrt_x86_64_external_dir = ( lambda: subprocess.check_output( @@ -918,12 +928,11 @@ def run(self): ) ] - packages += ["torch_tensorrt.ts", "torch_tensorrt.dynamo.runtime.meta_ops"] + packages += ["torch_tensorrt.ts"] package_dir.update( { "torch_tensorrt.ts": "py/torch_tensorrt/ts", - "torch_tensorrt.dynamo.runtime.meta_ops": "py/torch_tensorrt/dynamo/runtime/meta_ops", } ) From 3acc17e55dc933860eca2603184bc6d91ac1302d Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Tue, 25 Aug 2026 12:48:48 -0700 Subject: [PATCH 2/3] fix it --- py/torch_tensorrt/dynamo/runtime/_TRTEngine.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/py/torch_tensorrt/dynamo/runtime/_TRTEngine.py b/py/torch_tensorrt/dynamo/runtime/_TRTEngine.py index 5f2473f587..a8dbd1552d 100644 --- a/py/torch_tensorrt/dynamo/runtime/_TRTEngine.py +++ b/py/torch_tensorrt/dynamo/runtime/_TRTEngine.py @@ -74,7 +74,7 @@ def _current_serialized_platform() -> str: """Return the current platform using the engine-metadata representation.""" platform = Platform.current_platform() return ( - platform._to_serialized_rt_platform() + cast(str, platform._to_serialized_rt_platform()) if ENABLED_FEATURES.torch_tensorrt_runtime else str(platform) ) @@ -368,6 +368,18 @@ def __str__(self) -> str: def __repr__(self) -> str: return self.__str__() + def serialize_metadata_only(self) -> List[Any]: + """Return the serialized layout without base64-encoding the engine bytes. + + This mirrors the C++ runtime accessor used by ExecuTorch export when it + only needs engine metadata. The record keeps its normal shape, but the + engine slot is deliberately empty; callers that need the engine must + use ``__getstate__`` instead. + """ + serialized_info = list(self.serialized_info) + serialized_info[ENGINE_IDX] = "" + return serialized_info + def __getstate__(self) -> Tuple[List[Any], str]: """Return pickle state in the same shape as C++ ``ScriptObject.__getstate__``. From 1fd6170949ce6c5eb02bb720574b03f76946e3c4 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Tue, 25 Aug 2026 14:55:07 -0700 Subject: [PATCH 3/3] fix --- py/torch-tensorrt-executorch-runtime/setup.py | 7 ++++++- py/torch_tensorrt/executorch/_export_utils.py | 14 ++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/py/torch-tensorrt-executorch-runtime/setup.py b/py/torch-tensorrt-executorch-runtime/setup.py index dc3d853556..0a4b00f8ba 100644 --- a/py/torch-tensorrt-executorch-runtime/setup.py +++ b/py/torch-tensorrt-executorch-runtime/setup.py @@ -136,7 +136,12 @@ def build_extension(self, ext: Extension) -> None: source = built.parent / dependency if not source.is_file(): raise RuntimeError(f"Bazel did not produce {source}") - shutil.copy2(source, output.parent / dependency) + destination = output.parent / dependency + # ``build_extension`` runs once for each extension. The first copy + # preserves Bazel's read-only mode, so remove it before the second + # extension tries to copy the same dependency. + destination.unlink(missing_ok=True) + shutil.copy2(source, destination) require_cuda_13() diff --git a/py/torch_tensorrt/executorch/_export_utils.py b/py/torch_tensorrt/executorch/_export_utils.py index 89a389ea77..42b40654fb 100644 --- a/py/torch_tensorrt/executorch/_export_utils.py +++ b/py/torch_tensorrt/executorch/_export_utils.py @@ -36,13 +36,14 @@ def _seed_graph_bound_leaves(value: Any, memo: dict[int, Any]) -> None: def _warn_missing_accessor(name: str) -> None: - """Warn once per accessor that the loaded runtime predates it. + """Warn once per accessor that the engine's native runtime predates it. Without this the only symptom is that export gets slower: the fallback is correct, it just re-serializes the whole ICudaEngine and base64-encodes it. - It means the Torch-TensorRT C++ library is older than this Python package, - i.e. a source or editable build where only the Python half was rebuilt; a - wheel ships both halves together, so an unmodified wheel should not see it. + The engine may have come from a Torch-TensorRT C++ library older than this + Python package. That can be a mismatched source/editable build, or, for a + Python-only wheel, an engine loaded from an external or previously-built + native runtime. """ if name in _WARNED_MISSING: return @@ -50,8 +51,9 @@ def _warn_missing_accessor(name: str) -> None: logger.warning( "TensorRT runtime has no %s(); falling back to full engine serialization " "on every read of this engine's info. This is correct but slower, and " - "means the Torch-TensorRT C++ library loaded from torch_tensorrt/lib " - "predates this Python package -- rebuild the C++ side to avoid it.", + "means this engine was created by a Torch-TensorRT C++ runtime that " + "predates this Python package. Rebuild or reinstall the matching native " + "runtime (or regenerate the engine) to avoid it.", name, )