From 10c81104408457b78556f070638e3b32cc928307 Mon Sep 17 00:00:00 2001 From: shoumikhin Date: Mon, 24 Aug 2026 15:38:09 -0700 Subject: [PATCH 01/15] ci(executorch): run a coalesced TensorRT + CUDA program in the reference runner gate The ExecuTorch gate exports one program, x + 1, and TensorRT takes it whole. So nothing in CI ever runs a program where TensorRT and ExecuTorch's own CUDA backend each own part of the same graph. That coalesced case is the whole point of combining the two backends, and it is not covered end to end today. There is a composition test that checks both delegates land in the file, but it never loads or runs the program. This adds the missing run. A new example, examples/torchtrt_executorch_example/export_coalesced.py, exports cos(erfinv(tanh(x))). TensorRT has no converter for erfinv, so a CudaPartitioner catch-all gives that operator to the CUDA backend while TensorRT keeps the rest. The script fails if the saved .pte does not carry both a TensorRTBackend and a CudaBackend delegate, so a partitioning change cannot quietly turn this into a TensorRT-only run that still passes. The script also writes .expected next to the .pte, holding the output shape and the eager reference value for an all-ones input. Both reference runners fill inputs with 1.0 and this model is elementwise, so one number describes the whole expected output. Reading it from a file, instead of hard-coding a number in the shell script, keeps the expectation tied to the model. verify-executorch-reference-runner.sh now takes an optional third argument, the coalesced .pte. When given, it runs both the CMake-built runner and the packaged runner on it and compares every printed value against that reference. TensorRT, AOTInductor and eager PyTorch use different kernels for the same math, so the comparison uses a tolerance of 0.001 rather than matching printed digits. The existing x + 1 assertions keep the same strength. They now go through the same helper with a zero tolerance, because x + 1 on ones is exact in float32. Usage: python examples/torchtrt_executorch_example/export_coalesced.py \ --model_path=coalesced.pte .github/scripts/verify-executorch-reference-runner.sh \ model.pte kv_cache_decode.pte coalesced.pte Test plan On a Linux x86_64 host with an NVIDIA A100 GPU: - Ran export_coalesced.py. It reported delegates ['TensorRTBackend', 'CudaBackend', 'TensorRTBackend'] and wrote "[64,64]" and "0.6722" into the .expected file. - Ran the resulting .pte through the reference runner. It printed "output[0] shape=[64,64]" and first 8 values of 0.6722, an exact match to the eager result. - Deleted the aoti_cuda_blob.ptd that the CUDA backend writes and ran again. Same output, so this model needs no external weight file. - Exercised the new shell assertion helper against captured runner output: correct output passes; one wrong value fails; a wrong shape fails; a missing values line fails; a value inside the tolerance passes and one outside it fails. - shellcheck, bash -n, black and isort are clean on the changed files. Not yet observed in CI: the ExecuTorch runtime build job currently fails on main when the packaged reference runner aborts on the existing x + 1 model, and the test job is skipped while that is true. Both happen before this new code runs. --- .../verify-executorch-reference-runner.sh | 96 +++++++++++--- .github/workflows/executorch-test-linux.yml | 5 +- .../export_coalesced.py | 120 ++++++++++++++++++ .../test_cuda_partitioner_composition.py | 9 +- 4 files changed, 207 insertions(+), 23 deletions(-) create mode 100644 examples/torchtrt_executorch_example/export_coalesced.py diff --git a/.github/scripts/verify-executorch-reference-runner.sh b/.github/scripts/verify-executorch-reference-runner.sh index fe8e98d3de..d032662926 100755 --- a/.github/scripts/verify-executorch-reference-runner.sh +++ b/.github/scripts/verify-executorch-reference-runner.sh @@ -18,6 +18,11 @@ set +x # examples/torchtrt_executorch_example/export_kv_cache_decode.py). When given, # kv_cache_decode_check is built and run against it as well. # +# Optional third argument: path to a coalesced TensorRT + CUDA .pte (see +# examples/torchtrt_executorch_example/export_coalesced.py). When given, both +# runners are run against it and their output is compared to the eager +# reference that export script wrote next to the model. +# # Optional: # TensorRT_ROOT=/path/to/extracted/TensorRT # If unset, the script reuses Bazel's fetched TensorRT SDK when available @@ -33,8 +38,8 @@ set +x repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "${repo_root}" -if [[ $# -lt 1 || $# -gt 2 ]]; then - echo "Usage: $0 PATH_TO_MODEL.pte [PATH_TO_KV_CACHE_DECODE.pte]" >&2 +if [[ $# -lt 1 || $# -gt 3 ]]; then + echo "Usage: $0 PATH_TO_MODEL.pte [PATH_TO_KV_CACHE_DECODE.pte [PATH_TO_COALESCED.pte]]" >&2 exit 1 fi model_path="$1" @@ -47,6 +52,11 @@ if [[ -n "${kv_model_path}" && ! -f "${kv_model_path}" ]]; then echo "KV-cache decode model not found: ${kv_model_path}" >&2 exit 1 fi +coalesced_model_path="${3:-}" +if [[ -n "${coalesced_model_path}" && ! -f "${coalesced_model_path}" ]]; then + echo "Coalesced model not found: ${coalesced_model_path}" >&2 + exit 1 +fi python_executable="${PYTHON_EXECUTABLE:-}" if [[ -z "${python_executable}" ]]; then @@ -467,25 +477,35 @@ packaged_runner_log="${verify_root}/packaged_runner.log" --model_path="${model_path}" \ --num_runs=1 2>&1 | tee "${packaged_runner_log}" -# The sample model is x + 1 on a (2,3,4,4) input and both runners fill inputs with -# 1.0f, so the shape is exactly [2,3,4,4] and every printed value is exactly 2.0000. -# Assert both precisely. Matching only "shape=" accepts any shape, and matching one -# 2.0000 anywhere on the values line accepts a line of wrong numbers that happens to -# contain one right one, so neither catches a stream-ordering regression returning -# stale or partial output. Both lines come from fprintf in the runner, so these -# assertions hold whatever ET_LOG_ENABLED is set to. -for _log in "${runner_log}" "${packaged_runner_log}"; do - # A right answer produced entirely on the host would not prove much here: the - # program is delegated to TensorRT, so at least one planned buffer has to be - # served by a registered CUDA DeviceAllocator. Pin that, otherwise a model or - # a planning change could quietly turn this into a CPU-only test. +# Assert the printed shape, and that EVERY value on the "first N values:" line is +# the expected one. Matching only "shape=" accepts any shape, and matching one +# right value anywhere on the values line accepts a line of wrong numbers that +# happens to contain one, so neither on its own catches a stream-ordering +# regression returning stale or partial output. Both lines come from fprintf in +# the runner, so these assertions hold whatever ET_LOG_ENABLED is set to. The +# models used here are elementwise on an all-ones input, so one number describes +# the whole expected output. +assert_runner_output() { + local _log="$1" + local _shape="$2" + local _expected="$3" + local _tolerance="$4" + local _values + local _value + + # A right answer produced entirely on the host would not prove much: the + # programs here are delegated, so at least one planned buffer has to be served + # by a registered CUDA DeviceAllocator. Pin that, otherwise a model or a + # planning change could quietly turn this into a CPU-only test. if ! grep -q 'planned buffer\[[0-9]*\] = [0-9]* bytes on device_type 1' "${_log}"; then echo "No CUDA planned buffer was allocated in ${_log}:" >&2 grep 'planned buffer' "${_log}" >&2 || echo " no planned buffer line at all" >&2 exit 1 fi - if ! grep -q 'output\[0\] shape=\[2,3,4,4\]' "${_log}"; then + # -F: the shape is bracketed, and an unescaped [2,3,4,4] is a regex character + # class that would match any single one of those characters. + if ! grep -qF "output[0] shape=${_shape}" "${_log}"; then echo "Unexpected output shape in ${_log}:" >&2 grep 'output\[0\] shape=' "${_log}" >&2 || echo " no shape line at all" >&2 exit 1 @@ -498,11 +518,19 @@ for _log in "${runner_log}" "${packaged_runner_log}"; do exit 1 fi for _value in ${_values}; do - if [[ "${_value}" != "2.0000" ]]; then - echo "Unexpected output value '${_value}' in ${_log}: ${_values}" >&2 + if ! awk -v got="${_value}" -v want="${_expected}" -v tol="${_tolerance}" \ + 'BEGIN { d = got - want; if (d < 0) d = -d; exit !(d <= tol) }'; then + echo "Unexpected output value '${_value}' in ${_log}" \ + "(expected ${_expected} within ${_tolerance}): ${_values}" >&2 exit 1 fi done +} + +# The sample model is x + 1 on a (2,3,4,4) input, so every output value is exactly +# 2. That is exact in float32, hence a zero tolerance. +for _log in "${runner_log}" "${packaged_runner_log}"; do + assert_runner_output "${_log}" "[2,3,4,4]" "2.0000" 0 done if [[ -n "${kv_model_path}" ]]; then @@ -521,3 +549,37 @@ if [[ -n "${kv_model_path}" ]]; then "${kv_check_path}" --model_path="${kv_model_path}" 2>&1 | tee "${kv_check_log}" grep -q "PASS: decode at pos=1 observed the KV written at pos=0" "${kv_check_log}" fi + +if [[ -n "${coalesced_model_path}" ]]; then + # A coalesced program splits one graph across the TensorRT delegate and + # ExecuTorch's CUDA delegate, so a value produced by one delegate is consumed by + # the other on the device, inside one method. The checks above use a program with + # a single delegate, so they exercise neither the second backend nor the handover + # between the two. + coalesced_expected_path="${coalesced_model_path%.pte}.expected" + if [[ ! -f "${coalesced_expected_path}" ]]; then + echo "Coalesced reference output not found: ${coalesced_expected_path}" >&2 + echo "It is written by examples/torchtrt_executorch_example/export_coalesced.py" >&2 + exit 1 + fi + coalesced_shape="$(sed -n '1p' "${coalesced_expected_path}")" + coalesced_value="$(sed -n '2p' "${coalesced_expected_path}")" + if [[ -z "${coalesced_shape}" || -z "${coalesced_value}" ]]; then + echo "Malformed coalesced reference output in ${coalesced_expected_path}" >&2 + exit 1 + fi + + # Only the from-source runner runs the coalesced program. It links every + # ExecuTorch delegate, including the CUDA/AOTI backend that the CUDA partition + # of a coalesced .pte is handed to. The packaged runner in the wheel ships the + # TensorRT delegate alone, so it has no CudaBackend to run that partition and + # cannot execute this model. + coalesced_runner_log="${verify_root}/coalesced_my_runner.log" + "${runner_path}" \ + --model_path="${coalesced_model_path}" \ + --num_runs=1 2>&1 | tee "${coalesced_runner_log}" + + # TensorRT, AOTInductor and eager PyTorch compute the same math with different + # kernels, so compare within a tolerance instead of on the printed digits. + assert_runner_output "${coalesced_runner_log}" "${coalesced_shape}" "${coalesced_value}" 0.001 +fi diff --git a/.github/workflows/executorch-test-linux.yml b/.github/workflows/executorch-test-linux.yml index 6d38fff9bb..9751f2bb54 100644 --- a/.github/workflows/executorch-test-linux.yml +++ b/.github/workflows/executorch-test-linux.yml @@ -95,8 +95,11 @@ jobs: --model_path="${RUNNER_TEMP}/torchtrt-python.pte" python examples/torchtrt_executorch_example/export_kv_cache_decode.py \ --model_path="${RUNNER_TEMP}/torchtrt-kv-cache-decode.pte" + python examples/torchtrt_executorch_example/export_coalesced.py \ + --model_path="${RUNNER_TEMP}/torchtrt-coalesced.pte" .github/scripts/verify-executorch-reference-runner.sh \ "${RUNNER_TEMP}/torchtrt-python.pte" \ - "${RUNNER_TEMP}/torchtrt-kv-cache-decode.pte" + "${RUNNER_TEMP}/torchtrt-kv-cache-decode.pte" \ + "${RUNNER_TEMP}/torchtrt-coalesced.pte" python examples/executorch_reference_runner/load_model.py \ --model_path="${RUNNER_TEMP}/torchtrt-python.pte" --num_runs=1 diff --git a/examples/torchtrt_executorch_example/export_coalesced.py b/examples/torchtrt_executorch_example/export_coalesced.py new file mode 100644 index 0000000000..9f9d1fbc90 --- /dev/null +++ b/examples/torchtrt_executorch_example/export_coalesced.py @@ -0,0 +1,120 @@ +""" +.. _executorch_export_coalesced: + +Exporting a Coalesced TensorRT + CUDA Model to ExecuTorch (.pte) +================================================================ + +This example exports one graph split across two ExecuTorch backends. TensorRT +takes the operators it can convert, and everything left over goes to +ExecuTorch's own CUDA backend, which compiles it with AOTInductor. Both +delegates end up inside a single ``.pte`` and run in the same method. + +The model is ``cos(erfinv(tanh(x)))``. TensorRT has no converter for +``erfinv``, so that operator is the one the CUDA backend has to claim. That +makes the split real rather than incidental. + +A value produced by one delegate is consumed by the other on the device, inside +a single method, so this is what proves the two backends can complete each +other rather than only work on their own. + +The CUDA backend also writes an ``aoti_cuda_blob.ptd`` next to the ``.pte`` for +its external weights. This model has no weights, so that file is empty of +tensors and the reference runner does not need it. + +Besides the ``.pte`` this writes ``.expected``, holding the output +shape and the eager reference value for an all-ones input. The reference runner +gate reads that file instead of hard-coding a number, so the expected value +cannot drift away from the model. + +Prerequisites +------------- +Install Torch-TensorRT with the ExecuTorch extra before running this example:: + + pip install -e ".[executorch]" + +ExecuTorch's CUDA backend also needs a CUDA toolkit (``nvcc``) at export time, +for the AOTInductor compile. +""" + +import argparse +import sys +from pathlib import Path + +import torch +import torch_tensorrt +from executorch.backends.cuda.cuda_backend import CudaBackend +from executorch.backends.cuda.cuda_partitioner import CudaPartitioner +from executorch.exir._serialize._program import deserialize_pte_binary + +SHAPE = (64, 64) + + +class CoalescedModel(torch.nn.Module): + def forward(self, x): + return torch.cos(torch.erfinv(torch.tanh(x))) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_path", default="coalesced.pte", help="Path to save the .pte" + ) + args = parser.parse_args() + model_path = Path(args.model_path) + + with torch.no_grad(): + model = CoalescedModel().eval().cuda() + example_input = (torch.randn(SHAPE).cuda(),) + + exported_program = torch.export.export(model, example_input) + trt_gm = torch_tensorrt.dynamo.compile( + exported_program, + arg_inputs=example_input, + min_block_size=1, + truncate_double=True, + ) + torch_tensorrt.save( + trt_gm, + str(model_path), + output_format="executorch", + arg_inputs=example_input, + retrace=False, + # Catch-all, so every operator TensorRT rejected goes to the CUDA + # backend instead of falling back to a portable CPU kernel. + partitioners=[ + CudaPartitioner( + [CudaBackend.generate_method_name_compile_spec("forward")] + ) + ], + ) + + # Both delegates must really be in the file. A partitioner change that + # quietly routed the whole graph to TensorRT would otherwise leave a + # green job that no longer tests the coalesced path at all. + program = deserialize_pte_binary(model_path.read_bytes()).program + delegates = [d.id for plan in program.execution_plan for d in plan.delegates] + missing = [ + name for name in ("TensorRTBackend", "CudaBackend") if name not in delegates + ] + if missing: + sys.exit( + f"{model_path} is not coalesced: missing {missing}, found {delegates}" + ) + + # The reference runners fill every input element with 1.0, and this model + # is elementwise, so a single number describes the whole expected output. + reference = model(torch.ones(SHAPE).cuda()) + expected_path = model_path.with_suffix(".expected") + expected_path.write_text( + "[{}]\n{:.4f}\n".format( + ",".join(str(dim) for dim in reference.shape), + reference.flatten()[0].item(), + ) + ) + + print(f"Saved {model_path} with delegates {delegates}.") + print(f"Saved {expected_path} with the eager reference output.") + + +if __name__ == "__main__": + main() diff --git a/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py b/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py index f3ef0d47e5..9bdc065e18 100644 --- a/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py +++ b/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py @@ -89,11 +89,10 @@ def _delegate_ids(pte_path): # NOTE: these tests are COMPOSITION-ONLY. They assert the serialized .pte carries # both a TensorRTBackend and a CudaBackend delegate (and, below, that external -# CUDA weights are persisted as a .ptd). They do NOT load or run the program: a -# coalesced ATen-mode .pte cannot be loaded yet because memory-planned CUDA -# buffers get a CPU data pointer, so Method::init fails the CUDA backend's device -# check (tensor_parser_aten hardcodes CPU). A load-run-allclose test should be -# added once that runtime fix lands (follow-up). +# CUDA weights are persisted as a .ptd). They do NOT load or run the program. +# Execution of a coalesced program is covered by the reference runner gate, see +# examples/torchtrt_executorch_example/export_coalesced.py and the third argument +# of .github/scripts/verify-executorch-reference-runner.sh. def test_erfinv_routes_to_cuda_backend(tmp_path): From 0c74fc9d3cfbc258aabc2e536df1cf59dd7bcbfc Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 22 Aug 2026 23:55:53 -0700 Subject: [PATCH 02/15] Track a recent ExecuTorch pin instead of 1.4.1 ExecuTorch 1.4.1 ships no linkable C++ runtime: its wheel contains zero shared libraries, its CMake package exports only a static _portable_lib, and no CUDA wheel exists for it on any channel. That is why the runtime wheel rebuilds ExecuTorch from source today, and it is the blocker for shipping only the TensorRT delegate. The prebuilt runtime landed on ExecuTorch main on 2026-08-20, six days after 1.4.1 was tagged, so no release carries it yet. Move the pin to the nightly line that does, keeping the release-line range on installable metadata so the same range prefers 1.5.0 over any dev build the day it ships, with no edit needed. The two pins now have to name one ExecuTorch rather than two that look close, because the delegate compiles headers from the source tree and links the runtime out of the wheel. Every wheel records its source commit, so add a test asserting the pinned commit is the pinned wheel's own git_version. Nothing else was enforcing that, and a mismatch is silent: both pins look plausible and the build succeeds. Deriving the range with a three-field split raised on the nightly form, so derive it from the release line the first two fields name. ExecuTorch's CUDA wheels are published only on the PyTorch nightly index, so every install site gains that channel. No site gains --pre: the pin names an exact dev build, which pip installs from an explicit version without it, and the existing --pre uses here are for torch. CI derives the channel from the row's own CU_VERSION, which keeps the runtime the delegate links to the same CUDA build as the rest of the job. --- .github/workflows/executorch-build-linux.yml | 9 ++- .github/workflows/executorch-test-linux.yml | 6 +- MODULE.bazel | 10 ++- dev_dep_versions.yml | 4 +- docker/MODULE.bazel.docker | 4 +- docker/MODULE.bazel.ngc | 4 +- .../executorch_reference_runner/README.md | 2 +- justfile | 7 +- .../README.md | 6 +- .../pyproject.toml | 2 +- setup.py | 4 + tests/ci/runner.py | 14 +++- .../dynamo/executorch/test_executorch_pin.py | 76 ++++++++++++++++++- toolchains/ci_workspaces/MODULE.bazel.tmpl | 4 +- 14 files changed, 128 insertions(+), 24 deletions(-) diff --git a/.github/workflows/executorch-build-linux.yml b/.github/workflows/executorch-build-linux.yml index f3eab9c237..77f2e51af0 100644 --- a/.github/workflows/executorch-build-linux.yml +++ b/.github/workflows/executorch-build-linux.yml @@ -79,7 +79,12 @@ jobs: export PATH="${RUNNER_TEMP}/bin:${PATH}" bazel --version - python -m pip install pyyaml "executorch==1.4.1" + # ExecuTorch's CUDA wheels are published only on the PyTorch nightly index, and the + # pin is a dev version, which the requirement itself already admits. + # CU_VERSION selects the row's own channel, which is what keeps the runtime the + # delegate links to the same CUDA build as the rest of the job. + EXECUTORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/${CU_VERSION}" + python -m pip install pyyaml --extra-index-url "${EXECUTORCH_INDEX_URL}" "executorch==1.5.0.dev20260822" export TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION="$(python -c 'import importlib.metadata; print(importlib.metadata.version("torch-tensorrt"))')" # The downloaded wheel has to carry the C++ runtime. A wheel built with @@ -126,7 +131,7 @@ jobs: export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" # this is to verify the end user's workflow - python -m pip install pyyaml "executorch>=1.4.1,<1.5" + python -m pip install pyyaml --extra-index-url "${EXECUTORCH_INDEX_URL}" "executorch>=1.5.0.dev20260822,<1.6" python examples/torchtrt_executorch_example/export_static_shape.py \ --model_path="${RUNNER_TEMP}/torchtrt-reference-runner.pte" .github/scripts/verify-executorch-reference-runner.sh \ diff --git a/.github/workflows/executorch-test-linux.yml b/.github/workflows/executorch-test-linux.yml index 9751f2bb54..7d5cecaab0 100644 --- a/.github/workflows/executorch-test-linux.yml +++ b/.github/workflows/executorch-test-linux.yml @@ -64,7 +64,11 @@ jobs: chmod +x "${RUNNER_TEMP}/bin/bazel" export PATH="${RUNNER_TEMP}/bin:${PATH}" - python -m pip install pyyaml "executorch==1.4.1" + # ExecuTorch's CUDA wheels live only on the PyTorch nightly index, and the pin is a + # dev version, which the requirement itself already admits. + python -m pip install pyyaml \ + --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" \ + "executorch==1.5.0.dev20260822" # Run the check directly so its exit status is the step's exit status. # Wrapping it in `gdb --batch` reports gdb's own status, which is 0 # whatever the program does, so a SIGSEGV here was passing. diff --git a/MODULE.bazel b/MODULE.bazel index 6ae9a88573..eccf79b9c8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -45,13 +45,15 @@ new_git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl" local_torch = use_repo_rule("//toolchains:local_torch.bzl", "local_torch") -# Keep this pin synchronized with the ExecuTorch release installed in -# py/torch-tensorrt-executorch-runtime/README.md. +# This commit must be the one the pinned ExecuTorch wheel was built from, because the delegate +# compiles headers from this tree and links the runtime out of that wheel. Every wheel records +# its source in executorch/version.py as git_version, and tests/py/dynamo/executorch/ +# test_executorch_pin.py asserts the two agree, so bump both pins together. new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.4.1 - commit = "e4d02f41f7909e8ed5bf4a14ffc520d733453d9f", + # executorch==1.5.0.dev20260822 + commit = "b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;", diff --git a/dev_dep_versions.yml b/dev_dep_versions.yml index 07bac57849..f17b31f1f2 100644 --- a/dev_dep_versions.yml +++ b/dev_dep_versions.yml @@ -2,5 +2,5 @@ __cuda_version__: "13.2" __tensorrt_version__: "11.2.1" __tensorrt_rtx_version__: "1.6.1" __tensorrt_llm_version__: "0.17.0.post1" -__executorch_version__: "1.4.1" -__executorch_commit__: "e4d02f41f7909e8ed5bf4a14ffc520d733453d9f" +__executorch_version__: "1.5.0.dev20260822" +__executorch_commit__: "b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7" diff --git a/docker/MODULE.bazel.docker b/docker/MODULE.bazel.docker index f0c9b161bb..b1459e08fa 100644 --- a/docker/MODULE.bazel.docker +++ b/docker/MODULE.bazel.docker @@ -67,8 +67,8 @@ new_local_repository( new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.4.1 - commit = "e4d02f41f7909e8ed5bf4a14ffc520d733453d9f", + # executorch==1.5.0.dev20260822 + commit = "b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;", diff --git a/docker/MODULE.bazel.ngc b/docker/MODULE.bazel.ngc index fcf70cafa8..6823b9bdc6 100644 --- a/docker/MODULE.bazel.ngc +++ b/docker/MODULE.bazel.ngc @@ -76,8 +76,8 @@ new_local_repository( new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.4.1 - commit = "e4d02f41f7909e8ed5bf4a14ffc520d733453d9f", + # executorch==1.5.0.dev20260822 + commit = "b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7", recursive_init_submodules = True, patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", diff --git a/examples/executorch_reference_runner/README.md b/examples/executorch_reference_runner/README.md index 1b1ccba454..3fcd4232c0 100644 --- a/examples/executorch_reference_runner/README.md +++ b/examples/executorch_reference_runner/README.md @@ -44,7 +44,7 @@ torch_tensorrt/bin/example_executorch_runner ```bash # Get the ExecuTorch source snapshot this package is built against. Keep this in sync # with the executorch commit pinned in MODULE.bazel. -EXECUTORCH_REF="${EXECUTORCH_REF:-e4d02f41f7909e8ed5bf4a14ffc520d733453d9f}" +EXECUTORCH_REF="${EXECUTORCH_REF:-b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7}" git clone --filter=blob:none --no-checkout \ https://github.com/pytorch/executorch.git executorch pushd executorch diff --git a/justfile b/justfile index 06f1574b02..df1a9e5a62 100644 --- a/justfile +++ b/justfile @@ -85,7 +85,12 @@ summary *args: # Install optional test deps so model/kernels/quantization/executorch suites run install-test-ext: uv pip install --group test-ext --group kernels --group quantization - uv pip install pyyaml "executorch>=1.4.1,<1.5" + # ExecuTorch's CUDA wheels are only on the PyTorch nightly index, and the pin is a dev + # version, which the requirement itself already admits. cu130 matches the + # torch index this project resolves against by default. + uv pip install pyyaml \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 \ + "executorch>=1.5.0.dev20260822,<1.6" # ── Linting ─────────────────────────────────────────────────────────────────── diff --git a/py/torch-tensorrt-executorch-runtime/README.md b/py/torch-tensorrt-executorch-runtime/README.md index d6d627a7b7..a181a783e6 100644 --- a/py/torch-tensorrt-executorch-runtime/README.md +++ b/py/torch-tensorrt-executorch-runtime/README.md @@ -39,7 +39,9 @@ of the wheel runtime contract. ```bash export TensorRT_ROOT=/path/to/TensorRT -python -m pip install pyyaml "executorch==1.4.1" +python -m pip install pyyaml \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu132 \ + "executorch==1.5.0.dev20260822" python -m pip wheel --no-build-isolation --no-deps \ --wheel-dir dist py/torch-tensorrt-executorch-runtime ``` @@ -47,7 +49,7 @@ python -m pip wheel --no-build-isolation --no-deps \ The native build obtains the ExecuTorch source through Bazel; no separate source checkout or `EXECUTORCH_SOURCE_DIR` setting is required. The source commit pinned in `MODULE.bazel` is the revision recorded by the -`executorch==1.4.1` wheel. +`executorch==1.5.0.dev20260822` wheel. The static ExecuTorch and delegate archives are intermediate build inputs; users receive the final native Python module and do not compile anything. diff --git a/py/torch-tensorrt-executorch-runtime/pyproject.toml b/py/torch-tensorrt-executorch-runtime/pyproject.toml index 78e33164c4..bb057bbc4a 100644 --- a/py/torch-tensorrt-executorch-runtime/pyproject.toml +++ b/py/torch-tensorrt-executorch-runtime/pyproject.toml @@ -6,6 +6,6 @@ requires = [ # environment. # Builds must use --no-build-isolation; see README.md. "torch", - "executorch==1.4.1", + "executorch==1.5.0.dev20260822", ] build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 1c994f6f39..9c7931171d 100644 --- a/setup.py +++ b/setup.py @@ -208,6 +208,10 @@ def load_dep_info(): # runtime API is not stable across minor releases, and an unbounded floor would resolve a future # minor against a backend built for this one. Patch releases stay allowed because they come off the # same release branch; the exact pin belongs in the runtime package, which does derive it. +# The floor currently names a dev build, because the runtime split the delegate needs does not +# exist in any ExecuTorch release yet: 1.4.1 ships no shared libraries and no CUDA wheel at all. +# That also makes this range prefer a release as soon as one exists, since 1.5.0 sorts above +# every 1.5.0.devN, so nothing here changes on the day it ships. _executorch_major, _executorch_minor = __executorch_version__.split(".")[:2] EXECUTORCH_REQUIREMENT = ( f"executorch>={__executorch_version__}," diff --git a/tests/ci/runner.py b/tests/ci/runner.py index fe98ab3423..7beb9ff3d0 100644 --- a/tests/ci/runner.py +++ b/tests/ci/runner.py @@ -131,10 +131,22 @@ def _setup_commands(step: str) -> list[tuple[list[str], Path]]: if step == "hub": return [(launcher + ["hub.py"], REPO_ROOT / "tests/modules")] if step == "executorch": + # ExecuTorch's CUDA wheels are published only on the PyTorch nightly index, so the + # channel is needed here. cu130 matches the torch index pyproject.toml resolves + # against by default, rather than dev_dep_versions.yml's __cuda_version__, which this + # file does not read. return [ ( launcher - + ["-m", "pip", "install", "pyyaml", _executorch_requirement()], + + [ + "-m", + "pip", + "install", + "pyyaml", + "--extra-index-url", + "https://download.pytorch.org/whl/nightly/cu130", + _executorch_requirement(), + ], REPO_ROOT, ) ] diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index ecb6458f5d..c00849db52 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -8,6 +8,10 @@ right spelling per role is the point: installable metadata that pins exactly would reject a compatible patch release, and a build input that takes a range could resolve an ExecuTorch the artifact was not compiled against. + +Agreeing with the file is necessary but not sufficient, so the last test closes the gap the +other two leave: they only prove the repository is self-consistent, which it would be even +if the wheel and the commit named two different ExecuTorch trees. """ import ast @@ -17,6 +21,8 @@ import sys from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[4] VERSIONS = REPO_ROOT / "dev_dep_versions.yml" @@ -75,12 +81,22 @@ def _wants_range(path: str, number: int) -> bool: return False +def _release_line(version: str) -> tuple[str, str]: + """Split a pin into its major and minor, for either a release or a nightly. + + ``1.4.1`` and ``1.5.0.dev20260822`` both belong to the release line their first two + fields name, so the range a site gets is derived from those and nothing else. Splitting + on every dot instead assumes three fields and raises on the nightly form. + """ + major, minor = version.split(".")[:2] + return major, minor + + def _expected(path: str, number: int, version: str) -> str: if not _wants_range(path, number): return f"executorch=={version}" - # Assumes X.Y.Z, which is what ExecuTorch releases and what this file records. - major, minor, _ = version.split(".") + major, minor = _release_line(version) return f"executorch>={version},<{major}.{int(minor) + 1}" @@ -148,7 +164,7 @@ def test_derived_requirements_match_the_pin() -> None: # setup.py and tests/ci/runner.py build their requirement from the pin, so the search # above cannot see them. Check the strings they produce instead. version = _versions()["__executorch_version__"] - major, minor, _ = version.split(".") + major, minor = _release_line(version) expected = f"executorch>={version},<{major}.{int(minor) + 1}" assert _setup_py_requirement(version) == expected @@ -169,6 +185,60 @@ def test_derived_requirements_roll_the_minor_over(tmp_path: Path) -> None: assert _runner_requirement(tmp_path) == expected +def test_the_pinned_commit_is_the_pinned_wheels_own_source() -> None: + """The two pins must name one ExecuTorch, not two that happen to be close. + + ``__executorch_version__`` selects the wheel the delegate is built to sit alongside, and + ``__executorch_commit__`` selects the tree it compiles from. Nothing about the two strings + forces them to agree, and a mismatch is invisible: both pins look plausible, the build + succeeds, and the delegate is compiled from one ExecuTorch while running against another. + Every published wheel records the commit it was built from, so the pairing is checkable + rather than a convention. + + Skipped rather than failed whenever the installed wheel is not the one the pin names — not + installed at all, a different member of a floating range, or built without git provenance. + None of those say anything about whether the two pins agree, and this file stays readable + offline. + """ + versions = _versions() + expected_commit = versions["__executorch_commit__"] + expected_version = versions["__executorch_version__"] + + try: + from executorch.version import __version__ as installed_version + from executorch.version import git_version as installed_commit + except ImportError: + pytest.skip("executorch is not installed, so the pinned wheel cannot be read") + + if installed_commit is None: + # ExecuTorch records this as Optional[str] and writes None when it is built outside a + # git checkout. Such a wheel carries no provenance to compare, which is not the pins + # disagreeing. + pytest.skip( + f"the installed ExecuTorch {installed_version} records no source commit, " + "so the pairing cannot be checked against it" + ) + + # The wheel carries a local version label naming its CUDA build (`+cu132`), which the pin + # deliberately omits so one pin serves every CUDA row. Compare the part they share. + if installed_version.split("+")[0] != expected_version: + # A different member of the same range, not a mismatch to report. The range sites + # deliberately float, and while the pin names a dev build the nightly channel gains a + # newer member daily, so any environment that installed through a range arrives here + # with a wheel this check cannot speak about. Only the wheel the pin names carries the + # commit the pin should agree with, so anything else is no evidence either way. + pytest.skip( + f"the installed ExecuTorch is {installed_version}, not the pinned " + f"{expected_version}, so its commit says nothing about whether the pins agree" + ) + + assert installed_commit == expected_commit, ( + f"ExecuTorch {installed_version} was built from {installed_commit}, but " + f"__executorch_commit__ pins {expected_commit}. The delegate would compile against " + "one ExecuTorch and link another." + ) + + def test_every_source_commit_matches_the_pin() -> None: commit = _versions()["__executorch_commit__"] diff --git a/toolchains/ci_workspaces/MODULE.bazel.tmpl b/toolchains/ci_workspaces/MODULE.bazel.tmpl index 796f714375..348ff0351b 100644 --- a/toolchains/ci_workspaces/MODULE.bazel.tmpl +++ b/toolchains/ci_workspaces/MODULE.bazel.tmpl @@ -214,8 +214,8 @@ new_local_repository( new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.4.1 - commit = "e4d02f41f7909e8ed5bf4a14ffc520d733453d9f", + # executorch==1.5.0.dev20260822 + commit = "b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;", From ba00b2564f0bf04c64e00d0abf47bcd795c4ef30 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 23 Aug 2026 12:25:19 -0700 Subject: [PATCH 03/15] Install the ExecuTorch the pin names, from the channel that has it The CI installer globs torch_tensorrt*.whl, which also matches the ExecuTorch runtime wheel, whose install_requires names a dev build published only on the nightly channel. With no index on that line the whole pip invocation failed, and because line 1's set -e is commented out the failure was swallowed and the job died later with a confusing ImportError. The two range sites installed a range against the nightly channel, which gains a member every day, so they resolved to whatever was newest while the delegate is compiled from the commit the pin names. Both now request the pin exactly, which is the pairing the drift test exists to check; it was written to skip in exactly the state the ranges produced, so nothing reported it. setup.py keeps its range, because a published requirement has to stay resolvable for users off the same line. The two shapes now differ deliberately and test_derived_requirements_match_the_pin checks each for its own. Six printed install instructions gave a bare pip install of the executorch extra, which cannot resolve a dev pin from PyPI. They name the channel now. The discovery regex saw only == and >=, so a site added with any other PEP 440 operator was invisible to the drift check. It now recognises all of them. --- .github/scripts/install-torch-tensorrt.sh | 5 +- .github/workflows/executorch-build-linux.yml | 5 +- .github/workflows/executorch-test-linux.yml | 5 +- MODULE.bazel | 3 +- .../runtime_performance/saving_models.rst | 5 +- justfile | 12 ++-- py/torch_tensorrt/_compile.py | 6 +- py/torch_tensorrt/executorch/__init__.py | 3 +- tests/ci/runner.py | 7 ++- .../dynamo/executorch/test_executorch_pin.py | 56 ++++++++++++------- 10 files changed, 69 insertions(+), 38 deletions(-) diff --git a/.github/scripts/install-torch-tensorrt.sh b/.github/scripts/install-torch-tensorrt.sh index 67a3c9fe29..902d47da9a 100755 --- a/.github/scripts/install-torch-tensorrt.sh +++ b/.github/scripts/install-torch-tensorrt.sh @@ -54,7 +54,10 @@ fi if [[ ${PLATFORM} == win32 ]]; then python -m pip install ${RUNNER_ARTIFACT_DIR}/torch_tensorrt*.whl else - python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl --use-deprecated=legacy-resolver + # The nightly channel is needed because this glob also matches the ExecuTorch runtime wheel, + # whose install_requires names an ExecuTorch dev build that is published only there. + python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl --use-deprecated=legacy-resolver \ + --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" fi echo -e "Running test script"; diff --git a/.github/workflows/executorch-build-linux.yml b/.github/workflows/executorch-build-linux.yml index 77f2e51af0..e8e88367c4 100644 --- a/.github/workflows/executorch-build-linux.yml +++ b/.github/workflows/executorch-build-linux.yml @@ -79,8 +79,9 @@ jobs: export PATH="${RUNNER_TEMP}/bin:${PATH}" bazel --version - # ExecuTorch's CUDA wheels are published only on the PyTorch nightly index, and the - # pin is a dev version, which the requirement itself already admits. + # ExecuTorch's CUDA wheels are published only on the PyTorch nightly index, so the + # channel is needed. No --pre: a specifier naming a prerelease admits prereleases by + # itself, and --pre would apply to every other requirement in the same command too. # CU_VERSION selects the row's own channel, which is what keeps the runtime the # delegate links to the same CUDA build as the rest of the job. EXECUTORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/${CU_VERSION}" diff --git a/.github/workflows/executorch-test-linux.yml b/.github/workflows/executorch-test-linux.yml index 7d5cecaab0..1966ba7308 100644 --- a/.github/workflows/executorch-test-linux.yml +++ b/.github/workflows/executorch-test-linux.yml @@ -64,8 +64,9 @@ jobs: chmod +x "${RUNNER_TEMP}/bin/bazel" export PATH="${RUNNER_TEMP}/bin:${PATH}" - # ExecuTorch's CUDA wheels live only on the PyTorch nightly index, and the pin is a - # dev version, which the requirement itself already admits. + # ExecuTorch's CUDA wheels live only on the PyTorch nightly index, so the channel is + # needed. No --pre: a specifier naming a prerelease admits prereleases by itself, and + # --pre would apply to every other requirement in the same command too. python -m pip install pyyaml \ --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" \ "executorch==1.5.0.dev20260822" diff --git a/MODULE.bazel b/MODULE.bazel index eccf79b9c8..afe76d6cc9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -48,7 +48,8 @@ local_torch = use_repo_rule("//toolchains:local_torch.bzl", "local_torch") # This commit must be the one the pinned ExecuTorch wheel was built from, because the delegate # compiles headers from this tree and links the runtime out of that wheel. Every wheel records # its source in executorch/version.py as git_version, and tests/py/dynamo/executorch/ -# test_executorch_pin.py asserts the two agree, so bump both pins together. +# test_executorch_pin.py checks the two agree wherever the pinned wheel is the one installed, +# so bump both pins together. new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", diff --git a/docsrc/user_guide/runtime_performance/saving_models.rst b/docsrc/user_guide/runtime_performance/saving_models.rst index 6470afd72e..418e65d19b 100644 --- a/docsrc/user_guide/runtime_performance/saving_models.rst +++ b/docsrc/user_guide/runtime_performance/saving_models.rst @@ -227,8 +227,9 @@ c) ExecuTorch (.pte) The ``executorch`` output format lowers the compiled module to an ExecuTorch ``.pte`` program, delegating the TensorRT engines to the Torch-TensorRT ExecuTorch -backend. It requires the ``executorch`` package (``pip install -"torch_tensorrt[executorch]"``) and is Linux-only. +backend. It requires the ``executorch`` package, from the PyTorch nightly index +(``pip install "torch_tensorrt[executorch]" --extra-index-url +https://download.pytorch.org/whl/nightly/cu130``), and is Linux-only. There are two ways to produce a ``.pte``, and they suit different needs: diff --git a/justfile b/justfile index df1a9e5a62..2b806852dc 100644 --- a/justfile +++ b/justfile @@ -85,12 +85,16 @@ summary *args: # Install optional test deps so model/kernels/quantization/executorch suites run install-test-ext: uv pip install --group test-ext --group kernels --group quantization - # ExecuTorch's CUDA wheels are only on the PyTorch nightly index, and the pin is a dev - # version, which the requirement itself already admits. cu130 matches the - # torch index this project resolves against by default. + # ExecuTorch's CUDA wheels are only on the PyTorch nightly index, so the channel is needed. + # No --pre: a specifier naming a prerelease admits prereleases by itself, and --pre would + # apply to pyyaml here too. cu130 matches the torch index this project resolves against by + # default. + # + # Exact, not a range: the nightly channel gains a member every day, and the delegate is + # compiled from the commit this version pairs with. uv pip install pyyaml \ --extra-index-url https://download.pytorch.org/whl/nightly/cu130 \ - "executorch>=1.5.0.dev20260822,<1.6" + "executorch==1.5.0.dev20260822" # ── Linting ─────────────────────────────────────────────────────────────────── diff --git a/py/torch_tensorrt/_compile.py b/py/torch_tensorrt/_compile.py index fa70e13c46..32cdab3300 100644 --- a/py/torch_tensorrt/_compile.py +++ b/py/torch_tensorrt/_compile.py @@ -858,7 +858,8 @@ def save( raise ImportError( "Saving in ExecuTorch format requires the executorch package " "with executorch.exir. Install with: pip install " - "\"torch_tensorrt[executorch]\" to use output_format='executorch'." + '"torch_tensorrt[executorch]" --extra-index-url ' + "https://download.pytorch.org/whl/nightly/cu130 to use output_format='executorch'." ) if output_format == "executorch": # Every executorch option is popped above, so a leftover kwarg is a typo. Fail @@ -1406,7 +1407,8 @@ def _save_as_executorch(exp_program: Any, file_path: str, **kwargs: Any) -> None except ImportError: raise ImportError( "ExecuTorch is not installed. Install with: pip install " - "\"torch_tensorrt[executorch]\" to use output_format='executorch'." + '"torch_tensorrt[executorch]" --extra-index-url ' + "https://download.pytorch.org/whl/nightly/cu130 to use output_format='executorch'." ) import torch_tensorrt.dynamo.runtime.meta_ops.register_meta_ops # noqa: F401 diff --git a/py/torch_tensorrt/executorch/__init__.py b/py/torch_tensorrt/executorch/__init__.py index fef0943ce7..0669898967 100644 --- a/py/torch_tensorrt/executorch/__init__.py +++ b/py/torch_tensorrt/executorch/__init__.py @@ -25,7 +25,8 @@ def __getattr__(name: str) -> NoReturn: raise ImportError( f"Cannot access torch_tensorrt.executorch.{name}: " "ExecuTorch with executorch.exir is required. " - 'Install with: pip install "torch_tensorrt[executorch]"' + 'Install with: pip install "torch_tensorrt[executorch]" ' + "--extra-index-url https://download.pytorch.org/whl/nightly/cu130" ) __all__ = [ diff --git a/tests/ci/runner.py b/tests/ci/runner.py index 7beb9ff3d0..ae2ba87cf8 100644 --- a/tests/ci/runner.py +++ b/tests/ci/runner.py @@ -27,12 +27,15 @@ def _executorch_requirement() -> str: # Read the pin the way the drift test does, so this file is not a second # place to edit when it moves. Regex rather than yaml: the runner declares # no runtime dependencies of its own and importing it should not add one. + # + # Exact, not a range: the nightly channel gains a member every day, so a + # range would install whatever is newest while the delegate is compiled + # from the pinned commit. Pairing them is the point. text = (REPO_ROOT / "dev_dep_versions.yml").read_text() version = dict(re.findall(r'^(__\w+__): "([^"]+)"', text, re.MULTILINE))[ "__executorch_version__" ] - major, minor = version.split(".")[:2] - return f"executorch>={version},<{major}.{int(minor) + 1}" + return f"executorch=={version}" # Known transient cudagraph/TRT-driver flake signatures. Expand ONLY with diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index c00849db52..de0f1c5ac4 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -9,9 +9,10 @@ compatible patch release, and a build input that takes a range could resolve an ExecuTorch the artifact was not compiled against. -Agreeing with the file is necessary but not sufficient, so the last test closes the gap the -other two leave: they only prove the repository is self-consistent, which it would be even -if the wheel and the commit named two different ExecuTorch trees. +Agreeing with the file is necessary but not sufficient, so +test_the_pinned_commit_is_the_pinned_wheels_own_source closes the gap the others leave: they +only prove the repository is self-consistent, which it would be even if the wheel and the +commit named two different ExecuTorch trees. """ import ast @@ -26,7 +27,14 @@ REPO_ROOT = Path(__file__).resolve().parents[4] VERSIONS = REPO_ROOT / "dev_dep_versions.yml" -REQUIREMENT = re.compile(r"executorch(?:==|>=)[0-9][^\"'\s,`]*(?:,<[0-9.]+)?") +# Every PEP 440 operator, not just the two this repository happens to use, and an optional +# space before it. A site added with a compatible-release or bare-inequality operator is a site +# that drifted from the pin, and it should be visible to the search rather than silently +# exempt. Operators are named through the pattern rather than spelled out in prose here, +# because the search below reads this file too and an example would read as such a site. +REQUIREMENT = re.compile( + r"executorch\s*(?:===|==|>=|<=|~=|!=|<|>)\s*[0-9][^\"'\s,`]*(?:,\s*<[0-9.]+)?" +) # The bazel repository puts the commit on its own line, so this one has to run against file # contents rather than a git grep line. @@ -43,11 +51,10 @@ # Only literal requirements land here. setup.py and tests/ci/runner.py derive theirs from # dev_dep_versions.yml, so the search below no longer sees them and # test_derived_requirements_match_the_pin covers them instead. -RANGE_SITES = frozenset( - { - "justfile", - } -) +# +# Empty today: the only literal range left was the justfile's install recipe, and it installs +# the wheel the delegate is compiled against, so it pins exactly like the rest. +RANGE_SITES: frozenset[str] = frozenset() # A step that exists to reproduce what a user runs belongs to the range group even inside a # file that otherwise pins build inputs, so the marker travels with the line rather than @@ -105,7 +112,9 @@ def test_every_requirement_matches_the_pin() -> None: wrong = [] found = 0 - for line in _git("grep", "-nI", "-E", r"executorch(==|>=)[0-9]").splitlines(): + for line in _git( + "grep", "-nI", "-E", r"executorch ?(===|==|>=|<=|~=|!=|<|>) ?[0-9]" + ).splitlines(): path, number, text = line.split(":", 2) if path == VERSIONS.name: continue @@ -163,12 +172,18 @@ def _runner_requirement(root: Path) -> str: def test_derived_requirements_match_the_pin() -> None: # setup.py and tests/ci/runner.py build their requirement from the pin, so the search # above cannot see them. Check the strings they produce instead. + # + # They want different shapes. setup.py declares what users may install, so it is a range + # over the release line. runner.py installs the wheel CI tests the delegate against, and + # the delegate is compiled from the commit the pin names, so it has to be exact: the + # nightly channel gains a member every day and a range there silently unpairs the two. version = _versions()["__executorch_version__"] major, minor = _release_line(version) - expected = f"executorch>={version},<{major}.{int(minor) + 1}" - assert _setup_py_requirement(version) == expected - assert _runner_requirement(REPO_ROOT) == expected + assert _setup_py_requirement(version) == ( + f"executorch>={version},<{major}.{int(minor) + 1}" + ) + assert _runner_requirement(REPO_ROOT) == f"executorch=={version}" def test_derived_requirements_roll_the_minor_over(tmp_path: Path) -> None: @@ -176,13 +191,13 @@ def test_derived_requirements_roll_the_minor_over(tmp_path: Path) -> None: # Spelled through a variable because the search above reads this file too, and a # written-out requirement here would read as a site that drifted from the pin. version = "1.9.0" - expected = f"executorch>={version},<1.10" (tmp_path / "dev_dep_versions.yml").write_text( f'__executorch_version__: "{version}"\n' ) - assert _setup_py_requirement(version) == expected - assert _runner_requirement(tmp_path) == expected + assert _setup_py_requirement(version) == f"executorch>={version},<1.10" + # No upper bound to roll over, but it must still track the pin it is given. + assert _runner_requirement(tmp_path) == f"executorch=={version}" def test_the_pinned_commit_is_the_pinned_wheels_own_source() -> None: @@ -222,11 +237,10 @@ def test_the_pinned_commit_is_the_pinned_wheels_own_source() -> None: # The wheel carries a local version label naming its CUDA build (`+cu132`), which the pin # deliberately omits so one pin serves every CUDA row. Compare the part they share. if installed_version.split("+")[0] != expected_version: - # A different member of the same range, not a mismatch to report. The range sites - # deliberately float, and while the pin names a dev build the nightly channel gains a - # newer member daily, so any environment that installed through a range arrives here - # with a wheel this check cannot speak about. Only the wheel the pin names carries the - # commit the pin should agree with, so anything else is no evidence either way. + # No evidence either way rather than a mismatch to report: only the wheel the pin names + # carries the commit the pin should agree with. Every install path in this repository + # now requests the pin exactly, so arriving here means the environment was built some + # other way, and that wheel's commit says nothing about whether the two pins agree. pytest.skip( f"the installed ExecuTorch is {installed_version}, not the pinned " f"{expected_version}, so its commit says nothing about whether the pins agree" From fccbf67222477947f502e0fcab401ebd57025310 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 23 Aug 2026 14:53:47 -0700 Subject: [PATCH 04/15] Follow the row's CUDA version, and reach the channel from every example The executorch suite is nightly-only, and the nightly matrix runs cu132 rows as well as cu130 ones, so the fixed cu130 channel in tests/ci/runner.py would install a CUDA 13.0 ExecuTorch into a CUDA 13.2 job. PR builds are pinned to cu130 by filter-matrix.py, which is why watching PR CI could never show this. Derived from CU_VERSION now, with cu130 as the local default, matching what the two workflow files already did. Three example docstrings still printed a bare `pip install -e ".[executorch]"`. That resolved off PyPI before this pin moved to a dev build; it cannot now, so they name the nightly index too. The runtime's ImportError advice and the reference runner README already did. The installer's hardcoded nightly channel gets the reason written down: a .dev wheel exists on no other channel, so deriving it from ${CHANNEL} like the lines above would break the install on exactly the test and release runs the index was added for. --- .github/scripts/install-torch-tensorrt.sh | 4 ++++ .../torchtrt_executorch_example/export_coalesced.py | 3 ++- .../export_dynamic_shape.py | 3 ++- .../export_kv_cache_decode.py | 3 ++- .../export_static_shape.py | 3 ++- tests/ci/runner.py | 11 +++++++---- tests/py/dynamo/executorch/test_executorch_pin.py | 9 +++++++++ 7 files changed, 28 insertions(+), 8 deletions(-) diff --git a/.github/scripts/install-torch-tensorrt.sh b/.github/scripts/install-torch-tensorrt.sh index 902d47da9a..840d81a2c5 100755 --- a/.github/scripts/install-torch-tensorrt.sh +++ b/.github/scripts/install-torch-tensorrt.sh @@ -56,6 +56,10 @@ if [[ ${PLATFORM} == win32 ]]; then else # The nightly channel is needed because this glob also matches the ExecuTorch runtime wheel, # whose install_requires names an ExecuTorch dev build that is published only there. + # Hardcoded rather than ${CHANNEL} like the lines above: a .dev wheel exists on no other + # channel, so deriving it would break this install on exactly the test and release runs the + # index was added for. It is an extra index, not a replacement, and torch is already + # force-reinstalled from ${INDEX_URL} above, so the pinned torch is not at risk from it. python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl --use-deprecated=legacy-resolver \ --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" fi diff --git a/examples/torchtrt_executorch_example/export_coalesced.py b/examples/torchtrt_executorch_example/export_coalesced.py index 9f9d1fbc90..ebbd4734e4 100644 --- a/examples/torchtrt_executorch_example/export_coalesced.py +++ b/examples/torchtrt_executorch_example/export_coalesced.py @@ -30,7 +30,8 @@ ------------- Install Torch-TensorRT with the ExecuTorch extra before running this example:: - pip install -e ".[executorch]" + pip install -e ".[executorch]" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 ExecuTorch's CUDA backend also needs a CUDA toolkit (``nvcc``) at export time, for the AOTInductor compile. diff --git a/examples/torchtrt_executorch_example/export_dynamic_shape.py b/examples/torchtrt_executorch_example/export_dynamic_shape.py index 28115f696c..64847c3c50 100644 --- a/examples/torchtrt_executorch_example/export_dynamic_shape.py +++ b/examples/torchtrt_executorch_example/export_dynamic_shape.py @@ -16,7 +16,8 @@ ------------- Install Torch-TensorRT with the ExecuTorch extra before running this example:: - pip install -e ".[executorch]" + pip install -e ".[executorch]" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 See https://pytorch.org/executorch/stable/getting-started-setup.html for details. """ diff --git a/examples/torchtrt_executorch_example/export_kv_cache_decode.py b/examples/torchtrt_executorch_example/export_kv_cache_decode.py index c8590d98b0..5dcf3476fa 100644 --- a/examples/torchtrt_executorch_example/export_kv_cache_decode.py +++ b/examples/torchtrt_executorch_example/export_kv_cache_decode.py @@ -18,7 +18,8 @@ ------------- Install Torch-TensorRT with the ExecuTorch extra before running this example:: - pip install -e ".[executorch]" + pip install -e ".[executorch]" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 """ import argparse diff --git a/examples/torchtrt_executorch_example/export_static_shape.py b/examples/torchtrt_executorch_example/export_static_shape.py index ed8bb218da..eadc36f0d4 100644 --- a/examples/torchtrt_executorch_example/export_static_shape.py +++ b/examples/torchtrt_executorch_example/export_static_shape.py @@ -12,7 +12,8 @@ ------------- Install Torch-TensorRT with the ExecuTorch extra before running this example:: - pip install -e ".[executorch]" + pip install -e ".[executorch]" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 See https://pytorch.org/executorch/stable/getting-started-setup.html for details. """ diff --git a/tests/ci/runner.py b/tests/ci/runner.py index ae2ba87cf8..5ee629eb7f 100644 --- a/tests/ci/runner.py +++ b/tests/ci/runner.py @@ -135,9 +135,12 @@ def _setup_commands(step: str) -> list[tuple[list[str], Path]]: return [(launcher + ["hub.py"], REPO_ROOT / "tests/modules")] if step == "executorch": # ExecuTorch's CUDA wheels are published only on the PyTorch nightly index, so the - # channel is needed here. cu130 matches the torch index pyproject.toml resolves - # against by default, rather than dev_dep_versions.yml's __cuda_version__, which this - # file does not read. + # channel is needed here. Derived from CU_VERSION rather than fixed, because the + # executorch suite is nightly-only and the nightly matrix runs cu132 rows as well as + # cu130 ones; a fixed channel would install a CUDA 13.0 runtime into a 13.2 job. The + # cu130 default is for a local run with no CU_VERSION set, and matches the torch index + # pyproject.toml resolves against by default. + cuda = os.environ.get("CU_VERSION", "cu130") return [ ( launcher @@ -147,7 +150,7 @@ def _setup_commands(step: str) -> list[tuple[list[str], Path]]: "install", "pyyaml", "--extra-index-url", - "https://download.pytorch.org/whl/nightly/cu130", + f"https://download.pytorch.org/whl/nightly/{cuda}", _executorch_requirement(), ], REPO_ROOT, diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index de0f1c5ac4..34b2ce6361 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -186,6 +186,15 @@ def test_derived_requirements_match_the_pin() -> None: assert _runner_requirement(REPO_ROOT) == f"executorch=={version}" +def test_the_runner_follows_the_row_s_cuda_version() -> None: + # The executorch suite is nightly-only, and the nightly matrix runs cu132 rows as well as + # cu130 ones, so a fixed channel would install a CUDA 13.0 ExecuTorch into a CUDA 13.2 job. + # PRs pin to cu130, which is why this cannot be caught by watching PR CI. + source = (REPO_ROOT / "tests/ci/runner.py").read_text(encoding="utf-8") + assert 'os.environ.get("CU_VERSION", "cu130")' in source + assert "nightly/cu130" not in source, "the channel is hardcoded again" + + def test_derived_requirements_roll_the_minor_over(tmp_path: Path) -> None: # The upper bound is a version, not a decimal: 1.9 has to become 1.10, not 1.1. # Spelled through a variable because the search above reads this file too, and a From c4286aa4f45e1dc630388c5574b424927b1bd53b Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 23 Aug 2026 17:04:02 -0700 Subject: [PATCH 05/15] Keep the extra resolvable on Windows, and pin the channel docgen installs from Raising the executorch floor to a dev build made `uv lock` fail outright. The lock's win32 required-environment in pyproject.toml resolved the extra from PyPI, whose executorch stops at 1.4.1 -- uv.lock records five win_amd64 wheels for it -- so nothing satisfied the new range and uv errors rather than falling back. Reproduced against a probe project: without a marker uv reports the win32 split unsatisfiable, with one it resolves. The requirement now carries `platform_system == 'Linux'`, the shape EXECUTORCH_RUNTIME_REQUIREMENT already uses, which also stops pip reporting no matching distribution for Windows users of the extra. The delegate is a Linux object and ExecuTorch publishes CUDA wheels for no other platform, so the marker states what was already true. docgen installed the extra with --pre against the nightly channel, so it resolved through the range and took whichever dev build was newest that morning while the delegate compiled from the pinned commit. It names the pin now, read out of dev_dep_versions.yml. The pip line that installs both wheels gets `|| exit 1`. linux-test.yml concatenates this installer ahead of the user script and line 1's `set -e` is commented out, so a failure there was discarded and the job died later with an unrelated-looking ImportError; measured with `false` in place of the pip call, exit was 0 and the user script still ran. Two tests were checking source text rather than behaviour. The CUDA-row test now calls _setup_commands with CU_VERSION set and unset and reads the URL, which catches keeping the os.environ.get line while hardcoding the channel -- the mutation the string match passed. The drift check now asserts the set of files that pin ExecuTorch, because a site changing to bare `executorch` stops matching the search entirely and left the old `assert found` satisfied. Also corrects two claims: 1.4.1 does ship _portable_lib.so, so the comment says its executorch/lib carries no standalone linkable runtime, and no install site gains --pre, since an exact .dev pin needs none. --- .github/scripts/install-torch-tensorrt.sh | 7 +- .github/workflows/docgen.yml | 7 +- setup.py | 11 ++- .../dynamo/executorch/test_executorch_pin.py | 91 ++++++++++++++++--- 4 files changed, 99 insertions(+), 17 deletions(-) diff --git a/.github/scripts/install-torch-tensorrt.sh b/.github/scripts/install-torch-tensorrt.sh index 840d81a2c5..be41ca5923 100755 --- a/.github/scripts/install-torch-tensorrt.sh +++ b/.github/scripts/install-torch-tensorrt.sh @@ -60,8 +60,13 @@ else # channel, so deriving it would break this install on exactly the test and release runs the # index was added for. It is an extra index, not a replacement, and torch is already # force-reinstalled from ${INDEX_URL} above, so the pinned torch is not at risk from it. + # || exit 1 because line 1's `set -exou pipefail` is commented out and linux-test.yml + # concatenates this file ahead of the user script, so a failure here would otherwise be + # discarded and the job would die later with an unrelated-looking ImportError. Scoped to the + # line this change is responsible for; re-enabling set -e for the whole file is a + # pre-existing hazard worth a separate change. python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl --use-deprecated=legacy-resolver \ - --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" + --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" || exit 1 fi echo -e "Running test script"; diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index fb390df6ea..d31038762b 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -42,7 +42,12 @@ jobs: run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: Build Python Package run: | - python3 -m pip install --pre ".[executorch]" --extra-index-url https://download.pytorch.org/whl/nightly/cu130 + # The pin exactly, not the range the extra expands to: --pre plus a nightly + # channel that gains a member daily would otherwise install whichever dev build + # is newest that morning while the delegate compiles from the pinned commit. + python3 -m pip install --pre ".[executorch]" \ + "executorch==$(python3 -c 'import yaml;print(yaml.safe_load(open("dev_dep_versions.yml"))["__executorch_version__"])')" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 - name: Install uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/setup.py b/setup.py index 9c7931171d..1073ef4c28 100644 --- a/setup.py +++ b/setup.py @@ -209,13 +209,20 @@ def load_dep_info(): # minor against a backend built for this one. Patch releases stay allowed because they come off the # same release branch; the exact pin belongs in the runtime package, which does derive it. # The floor currently names a dev build, because the runtime split the delegate needs does not -# exist in any ExecuTorch release yet: 1.4.1 ships no shared libraries and no CUDA wheel at all. +# exist in any ExecuTorch release yet: 1.4.1's executorch/lib carries no standalone linkable +# runtime, and no CUDA wheel at all. # That also makes this range prefer a release as soon as one exists, since 1.5.0 sorts above # every 1.5.0.devN, so nothing here changes on the day it ships. _executorch_major, _executorch_minor = __executorch_version__.split(".")[:2] +# Linux-only, and not incidentally: the delegate is a Linux shared object, ExecuTorch publishes +# CUDA wheels for no other platform, and the feature is documented Linux-only. Without the marker +# the extra also has to resolve for the win32 entry in pyproject.toml's uv required-environments, +# where the only candidates are PyPI's, which stop at 1.4.1 -- so raising this floor above that +# makes `uv lock` fail outright rather than pick something older. EXECUTORCH_REQUIREMENT = ( f"executorch>={__executorch_version__}," - f"<{_executorch_major}.{int(_executorch_minor) + 1}" + f"<{_executorch_major}.{int(_executorch_minor) + 1}; " + "platform_system == 'Linux'" ) # TODO: Enable this once the runtime wheel is published to the PyTorch index. # EXECUTORCH_RUNTIME_REQUIREMENT = ( diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index 34b2ce6361..8a7472bde6 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -61,6 +61,22 @@ # with the path. USER_WORKFLOW_MARKER = "verify the end user's workflow" +# The files expected to pin ExecuTorch to a version, excluding dev_dep_versions.yml itself. +# Asserted as a set of files rather than a line count because a site that loses its version stops +# matching the search rather than reporting a mismatch, and because the line count legitimately +# differs between this branch and the stacked runtime-wheel change. +_EXPECTED_REQUIREMENT_FILES = { + ".github/workflows/executorch-build-linux.yml", + ".github/workflows/executorch-test-linux.yml", + "MODULE.bazel", + "docker/MODULE.bazel.docker", + "docker/MODULE.bazel.ngc", + "justfile", + "py/torch-tensorrt-executorch-runtime/README.md", + "py/torch-tensorrt-executorch-runtime/pyproject.toml", + "toolchains/ci_workspaces/MODULE.bazel.tmpl", +} + def _git(*arguments: str) -> str: return subprocess.run( @@ -104,7 +120,10 @@ def _expected(path: str, number: int, version: str) -> str: return f"executorch=={version}" major, minor = _release_line(version) - return f"executorch>={version},<{major}.{int(minor) + 1}" + # Only the top-level setup.py carries the Linux marker. It is the site uv resolves for the + # win32 required-environment, where PyPI's candidates stop below this floor. + marker = "; platform_system == 'Linux'" if path == "setup.py" and number > 1 else "" + return f"executorch>={version},<{major}.{int(minor) + 1}{marker}" def test_every_requirement_matches_the_pin() -> None: @@ -112,6 +131,7 @@ def test_every_requirement_matches_the_pin() -> None: wrong = [] found = 0 + seen = set() for line in _git( "grep", "-nI", "-E", r"executorch ?(===|==|>=|<=|~=|!=|<|>) ?[0-9]" ).splitlines(): @@ -121,10 +141,20 @@ def test_every_requirement_matches_the_pin() -> None: expected = _expected(path, int(number), version) for actual in REQUIREMENT.findall(text): found += 1 + seen.add(path) if actual != expected: wrong.append(f"{path}:{number} has {actual}, expected {expected}") assert found, "no ExecuTorch requirement found, so this test is not looking" + # The set of files, not just "nonzero": a site that drops its version entirely stops matching + # the search and silently leaves the result set, which is exactly how a lost pin would look. + assert seen == _EXPECTED_REQUIREMENT_FILES, ( + "the set of files pinning ExecuTorch changed.\n" + f" no longer pinning: {sorted(_EXPECTED_REQUIREMENT_FILES - seen)}\n" + f" newly pinning: {sorted(seen - _EXPECTED_REQUIREMENT_FILES)}\n" + "A file that lost its version does not appear in the search at all, so check for one " + "that now names bare `executorch` before updating the expected set." + ) assert not wrong, "\n ".join(["", *wrong]) @@ -180,19 +210,48 @@ def test_derived_requirements_match_the_pin() -> None: version = _versions()["__executorch_version__"] major, minor = _release_line(version) + # The Linux marker is part of the requirement: the extra has to resolve for the win32 entry + # in pyproject.toml's uv required-environments, where the only candidates are PyPI's and they + # stop below this floor, so without it `uv lock` fails outright. assert _setup_py_requirement(version) == ( - f"executorch>={version},<{major}.{int(minor) + 1}" + f"executorch>={version},<{major}.{int(minor) + 1}; platform_system == 'Linux'" ) assert _runner_requirement(REPO_ROOT) == f"executorch=={version}" -def test_the_runner_follows_the_row_s_cuda_version() -> None: - # The executorch suite is nightly-only, and the nightly matrix runs cu132 rows as well as - # cu130 ones, so a fixed channel would install a CUDA 13.0 ExecuTorch into a CUDA 13.2 job. - # PRs pin to cu130, which is why this cannot be caught by watching PR CI. - source = (REPO_ROOT / "tests/ci/runner.py").read_text(encoding="utf-8") - assert 'os.environ.get("CU_VERSION", "cu130")' in source - assert "nightly/cu130" not in source, "the channel is hardcoded again" +def test_the_runner_follows_the_row_s_cuda_version(monkeypatch) -> None: + """Call the runner and read the URL it builds, rather than matching its source. + + The executorch suite is nightly-only, and the nightly matrix runs cu132 rows as well as cu130 + ones, so a fixed channel would install a CUDA 13.0 ExecuTorch into a CUDA 13.2 job. PRs pin to + cu130, which is why watching PR CI cannot catch it. A source-text assertion could not catch it + either: keeping the ``os.environ.get`` line while hardcoding the URL passes one. + """ + sys.path.insert(0, str(REPO_ROOT / "tests")) + try: + from ci import runner + finally: + sys.path.pop(0) + + def channel_for(cu_version: str | None) -> str: + if cu_version is None: + monkeypatch.delenv("CU_VERSION", raising=False) + else: + monkeypatch.setenv("CU_VERSION", cu_version) + commands = runner._setup_commands("executorch") + urls = [ + argument + for command, _ in commands + for argument in command + if "download.pytorch.org" in argument + ] + assert len(urls) == 1, f"expected one index URL, got {urls}" + return urls[0] + + assert channel_for("cu132").endswith("/nightly/cu132") + assert channel_for("cu130").endswith("/nightly/cu130") + # Unset is a local run, and matches the index pyproject.toml resolves against by default. + assert channel_for(None).endswith("/nightly/cu130") def test_derived_requirements_roll_the_minor_over(tmp_path: Path) -> None: @@ -204,7 +263,10 @@ def test_derived_requirements_roll_the_minor_over(tmp_path: Path) -> None: f'__executorch_version__: "{version}"\n' ) - assert _setup_py_requirement(version) == f"executorch>={version},<1.10" + assert ( + _setup_py_requirement(version) + == f"executorch>={version},<1.10; platform_system == 'Linux'" + ) # No upper bound to roll over, but it must still track the pin it is given. assert _runner_requirement(tmp_path) == f"executorch=={version}" @@ -247,9 +309,11 @@ def test_the_pinned_commit_is_the_pinned_wheels_own_source() -> None: # deliberately omits so one pin serves every CUDA row. Compare the part they share. if installed_version.split("+")[0] != expected_version: # No evidence either way rather than a mismatch to report: only the wheel the pin names - # carries the commit the pin should agree with. Every install path in this repository - # now requests the pin exactly, so arriving here means the environment was built some - # other way, and that wheel's commit says nothing about whether the two pins agree. + # carries the commit the pin should agree with. Every CI install path that builds or + # tests the delegate requests the pin exactly -- the one deliberate range is the + # end-user install rehearsal in executorch-build-linux.yml -- so arriving here usually + # means the environment was built some other way, and that wheel's commit says nothing + # about whether the two pins agree. pytest.skip( f"the installed ExecuTorch is {installed_version}, not the pinned " f"{expected_version}, so its commit says nothing about whether the pins agree" @@ -267,6 +331,7 @@ def test_every_source_commit_matches_the_pin() -> None: wrong = [] found = 0 + seen = set() for path in _git("grep", "-lI", "-E", 'name = "executorch"').split(): for match in BAZEL_COMMIT.finditer((REPO_ROOT / path).read_text()): found += 1 From 8c3bb6eb1f0e2d49aa96861918d0fee0f6892295 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 23 Aug 2026 19:59:59 -0700 Subject: [PATCH 06/15] Put the index on the two printed installs, and fail the suite when setup fails The reference-runner README and the runtime's ImportError advice both printed `pip install "torch-tensorrt[executorch]"` with no index, and the commit that introduced the pin claimed otherwise. That claim was checked against the wrong branch: the fix existed only on the stacked runtime-wheel change, so this branch kept shipping the bare command. It matters more here than a docs nit, because this branch is what raises the floor above PyPI's newest executorch, so the bare command now cannot resolve at all. All seven printed install instructions carry the channel. The drift checks were counting the wrong thing. The requirement test asserted a set of paths, but two files carry two sites each, so either could drop one and stay in the set: turning `executorch-build-linux.yml:88` or `:128` into bare `executorch` both survived. The commit test only asserted nonzero, so any single MODULE.bazel could switch to `branch = "nightly"` unnoticed. Both now assert a per-file site count through one helper, as a minimum rather than an exact number so it holds on the stacked branch too, which removes one README site. All five mutations are caught and each names the file. Counting also surfaced a fifth commit site the nonzero check could not see: the reference-runner README's EXECUTORCH_REF shell default, correctly pinned but unaccounted for. docgen's pin was invisible to both: it is built by a shell substitution, so `$(` is not a digit and the literal search never saw it, and deleting the line survived. The derived-requirement test now runs the command docgen embeds and compares what it prints. A failed setup step printed `::warning::` and fell through to pytest. Most of the executorch suite gates on pytest.importorskip, so a failed ExecuTorch install skipped those files, left the rest passing, and reported success with a populated junit xml -- green exactly when the suite could not test what it exists to test. Driving the real run_suite with a failing setup step reproduced it, and returning the code makes it red without invoking pytest. Pre-existing, but this branch makes it likely to fire, since a nightly pin is eventually pruned from the channel. The pin tests themselves ran on nightly only, so none of this drift machinery ran on a PR or a push to main -- when a pin actually goes stale. They need no GPU, no ExecuTorch and not even torch, so they move to their own l0 suite in every lane, and the nightly suite excludes them by keyword so nothing runs twice. Also: the reference-runner README no longer says the extra installs the runtime wheel, since that requirement is commented out in setup.py. --- .../executorch_reference_runner/README.md | 14 ++- .../runtime.py | 3 +- tests/ci/runner.py | 13 +- tests/ci/suites.py | 14 +++ .../dynamo/executorch/test_executorch_pin.py | 111 ++++++++++++++---- 5 files changed, 123 insertions(+), 32 deletions(-) diff --git a/examples/executorch_reference_runner/README.md b/examples/executorch_reference_runner/README.md index 3fcd4232c0..1f635734ff 100644 --- a/examples/executorch_reference_runner/README.md +++ b/examples/executorch_reference_runner/README.md @@ -98,9 +98,13 @@ build-executorch-reference-runner/lib/libexecutorch_trt_backend.a Install the complete prebuilt Python runtime and delegate: ```bash -pip install "torch-tensorrt[executorch]" +pip install "torch-tensorrt[executorch]" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 ``` +The index is required, not optional: the extra's ExecuTorch floor names a dev build, and PyPI's +`executorch` stops below it, so without the nightly channel pip reports no matching distribution. + Load and run the model without an ExecuTorch checkout or native build: ```bash @@ -109,9 +113,11 @@ python examples/executorch_reference_runner/load_model.py \ --num_runs=1 ``` -The extra installs `executorch` and the matching -`torch-tensorrt-executorch-runtime` wheel. That wheel contains an ExecuTorch -Python runtime with `TensorRTBackend` linked into its backend registry. +The extra installs `executorch` only. The +`torch-tensorrt-executorch-runtime` requirement in the top-level `setup.py` is +commented out until that wheel is published to the PyTorch index, so install it +separately for now. That wheel contains an ExecuTorch Python runtime with +`TensorRTBackend` linked into its backend registry. ### C++ diff --git a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py index 27010326d7..353e872952 100644 --- a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py +++ b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py @@ -15,7 +15,8 @@ def _get_runtime() -> _Runtime: except ImportError as error: raise ImportError( "ExecuTorch Python inference requires the prebuilt delegate. " - 'Install it with: pip install "torch-tensorrt[executorch]"' + 'Install it with: pip install "torch-tensorrt[executorch]" ' + "--extra-index-url https://download.pytorch.org/whl/nightly/cu130" ) from error return get_runtime() diff --git a/tests/ci/runner.py b/tests/ci/runner.py index 5ee629eb7f..727de20e47 100644 --- a/tests/ci/runner.py +++ b/tests/ci/runner.py @@ -220,7 +220,18 @@ def run_suite( print(f"==> setup[{step}]: {shlex.join(argv)}", flush=True) rc = subprocess.run(argv, cwd=scwd, env=env).returncode if rc != 0: - print(f"::warning::setup step {step!r} exited {rc}", flush=True) + # Fail rather than warn and continue. Most of the executorch suite gates on + # pytest.importorskip, so a failed install skips those files, leaves the rest + # passing, and reports success with a populated junit xml -- the run looks green + # precisely when the thing it exists to test is absent. This matters more now + # that the ExecuTorch pin names a nightly build, which is pruned from the + # channel eventually; when that happens this has to be loud. + print( + f"::error::setup step {step!r} exited {rc}, so the suite cannot test what " + "it was asked to test", + flush=True, + ) + return rc print(f"==> {suite.name} [{variant}]: {shlex.join(pytest_cmd)}", flush=True) rc = subprocess.run(pytest_cmd, cwd=cwd, env=env).returncode diff --git a/tests/ci/suites.py b/tests/ci/suites.py index 5e5fa2bf4b..19cc2a7c3d 100644 --- a/tests/ci/suites.py +++ b/tests/ci/suites.py @@ -255,11 +255,25 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: keyword="not test_000_ and not test_001_", jobs=_HEAVY, ), + Suite( + # Separate from the executorch suite below because it needs none of what that one needs: + # no GPU, no ExecuTorch, not even torch. Keeping it here in the nightly-only suite meant + # the drift checks never ran on a PR or on a push to main, which is exactly when a pin + # goes stale. Text and metadata only, so it is cheap enough for every lane. + "executorch-pin", + tier="l0", + lanes=("fast", "full", "nightly"), + paths=("executorch/test_executorch_pin.py",), + jobs="auto", + variants=("standard",), + platforms=("linux-x86_64",), + ), Suite( "executorch", tier="l2", lanes=("nightly",), paths=("executorch/",), + keyword="not test_executorch_pin", setup=("executorch",), jobs="auto", variants=("standard",), diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index 8a7472bde6..726c923a58 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -18,8 +18,10 @@ import ast import os import re +import shlex import subprocess import sys +from collections import Counter from pathlib import Path import pytest @@ -61,22 +63,61 @@ # with the path. USER_WORKFLOW_MARKER = "verify the end user's workflow" -# The files expected to pin ExecuTorch to a version, excluding dev_dep_versions.yml itself. -# Asserted as a set of files rather than a line count because a site that loses its version stops -# matching the search rather than reporting a mismatch, and because the line count legitimately -# differs between this branch and the stacked runtime-wheel change. -_EXPECTED_REQUIREMENT_FILES = { - ".github/workflows/executorch-build-linux.yml", - ".github/workflows/executorch-test-linux.yml", - "MODULE.bazel", - "docker/MODULE.bazel.docker", - "docker/MODULE.bazel.ngc", - "justfile", - "py/torch-tensorrt-executorch-runtime/README.md", - "py/torch-tensorrt-executorch-runtime/pyproject.toml", - "toolchains/ci_workspaces/MODULE.bazel.tmpl", +# The files expected to pin ExecuTorch, mapped to how many sites each must carry, excluding +# dev_dep_versions.yml itself. A count per file rather than just the set of files, because a site +# that loses its version stops matching the search entirely rather than reporting a mismatch, and +# two of these files carry more than one site, so a set of paths let either quietly drop one. A +# minimum rather than an exact count, since the stacked runtime-wheel change removes one README +# site and an exact count could not hold on both branches. +_EXPECTED_REQUIREMENT_SITES = { + ".github/workflows/executorch-build-linux.yml": 2, + ".github/workflows/executorch-test-linux.yml": 1, + "MODULE.bazel": 1, + "docker/MODULE.bazel.docker": 1, + "docker/MODULE.bazel.ngc": 1, + "justfile": 1, + "py/torch-tensorrt-executorch-runtime/README.md": 1, + "py/torch-tensorrt-executorch-runtime/pyproject.toml": 1, + "toolchains/ci_workspaces/MODULE.bazel.tmpl": 1, } +# Same idea for the source commit the delegate compiles from. Five sites, not four: the +# reference-runner README names the ref as a shell default, which the old nonzero check could +# not distinguish from the four MODULE.bazel files. +_EXPECTED_COMMIT_SITES = { + "MODULE.bazel": 1, + "docker/MODULE.bazel.docker": 1, + "docker/MODULE.bazel.ngc": 1, + "toolchains/ci_workspaces/MODULE.bazel.tmpl": 1, + "examples/executorch_reference_runner/README.md": 1, +} + + +def _assert_every_site_present( + seen: "Counter[str]", expected: dict[str, int], what: str +) -> None: + """Require every expected file to still carry at least its expected number of sites. + + A site that drops its version or commit stops matching the search rather than reporting a + mismatch, so counting is the only way to notice it left. + """ + short = { + path: (count, seen.get(path, 0)) + for path, count in expected.items() + if seen.get(path, 0) < count + } + unexpected = sorted(set(seen) - set(expected)) + assert not short and not unexpected, ( + f"the set of sites {what} changed.\n" + + "".join( + f" {path} carries {actual} of {want} expected sites\n" + for path, (want, actual) in sorted(short.items()) + ) + + "".join(f" {path} is new and unaccounted for\n" for path in unexpected) + + "A site that lost its pin does not appear in the search at all, so look for one that " + "now names a bare reference before updating the expected counts." + ) + def _git(*arguments: str) -> str: return subprocess.run( @@ -131,7 +172,7 @@ def test_every_requirement_matches_the_pin() -> None: wrong = [] found = 0 - seen = set() + seen: Counter[str] = Counter() for line in _git( "grep", "-nI", "-E", r"executorch ?(===|==|>=|<=|~=|!=|<|>) ?[0-9]" ).splitlines(): @@ -141,20 +182,12 @@ def test_every_requirement_matches_the_pin() -> None: expected = _expected(path, int(number), version) for actual in REQUIREMENT.findall(text): found += 1 - seen.add(path) + seen[path] += 1 if actual != expected: wrong.append(f"{path}:{number} has {actual}, expected {expected}") assert found, "no ExecuTorch requirement found, so this test is not looking" - # The set of files, not just "nonzero": a site that drops its version entirely stops matching - # the search and silently leaves the result set, which is exactly how a lost pin would look. - assert seen == _EXPECTED_REQUIREMENT_FILES, ( - "the set of files pinning ExecuTorch changed.\n" - f" no longer pinning: {sorted(_EXPECTED_REQUIREMENT_FILES - seen)}\n" - f" newly pinning: {sorted(seen - _EXPECTED_REQUIREMENT_FILES)}\n" - "A file that lost its version does not appear in the search at all, so check for one " - "that now names bare `executorch` before updating the expected set." - ) + _assert_every_site_present(seen, _EXPECTED_REQUIREMENT_SITES, "pinning ExecuTorch") assert not wrong, "\n ".join(["", *wrong]) @@ -218,6 +251,29 @@ def test_derived_requirements_match_the_pin() -> None: ) assert _runner_requirement(REPO_ROOT) == f"executorch=={version}" + # docgen builds its overlay with a shell substitution, so neither the literal search nor the + # two helpers above can see it: `$(` is not a digit. Run the command it embeds and compare + # what it prints, which fails if the line is deleted or the key is renamed. + workflow = (REPO_ROOT / ".github/workflows/docgen.yml").read_text(encoding="utf-8") + embedded = re.search(r'"executorch==\$\((python3 -c \'[^\']+\')\)"', workflow) + assert embedded, ( + ".github/workflows/docgen.yml no longer pins ExecuTorch alongside the extra. It installs " + "with --pre from the nightly channel, so without the pin it resolves through the range " + "and takes whichever dev build is newest that day." + ) + printed = subprocess.run( + # The interpreter running the test, not the workflow's bare `python3`, which need not + # have pyyaml here. The argument list is the workflow's own. + [sys.executable, *shlex.split(embedded.group(1))[1:]], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + assert ( + printed == version + ), f"docgen would install executorch=={printed}, pin says {version}" + def test_the_runner_follows_the_row_s_cuda_version(monkeypatch) -> None: """Call the runner and read the URL it builds, rather than matching its source. @@ -331,10 +387,11 @@ def test_every_source_commit_matches_the_pin() -> None: wrong = [] found = 0 - seen = set() + seen: Counter[str] = Counter() for path in _git("grep", "-lI", "-E", 'name = "executorch"').split(): for match in BAZEL_COMMIT.finditer((REPO_ROOT / path).read_text()): found += 1 + seen[path] += 1 if match.group(1) != commit: wrong.append(f"{path} compiles {match.group(1)}") @@ -344,8 +401,10 @@ def test_every_source_commit_matches_the_pin() -> None: continue for actual in NAMED_COMMIT.findall(text): found += 1 + seen[path] += 1 if actual != commit: wrong.append(f"{path}:{number} uses {actual}") assert found, "no ExecuTorch source commit found, so this test is not looking" + _assert_every_site_present(seen, _EXPECTED_COMMIT_SITES, "naming the source commit") assert not wrong, f"pin says {commit}:\n " + "\n ".join(wrong) From 6f789269165badfd7f4e2d3430680c3c59442066 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 23 Aug 2026 21:36:06 -0700 Subject: [PATCH 07/15] Assert the pin sites CI actually uses, and move the checks off the GPU lane The drift checks read derived strings and never the values CI consumes, so three ways of silently shipping no ExecuTorch all stayed green. Dropping the requirement from the runner's setup command left the step succeeding with nothing installed, after which the suite skips on importorskip; emptying EXTRAS_REQUIRE["executorch"] broke every documented `pip install "torch-tensorrt[executorch]"`; and the runtime README was recorded as carrying one pin site when it carries two, so either could go bare while the other satisfied the count -- the exact hole the per-file counts were added to close. The checks now assert the argument list the runner builds, the extras entries by AST, and the true per-file counts. All five mutations fail now. run_suite had no test at all, so replacing its `return rc` with `continue` restored the silent-green behaviour the fail-closed change exists to prevent. It is driven directly now, asserting both the propagated exit code and that pytest never runs once setup has failed. The pin suite was landing on a GPU runner: Suite.runner defaults to the matrix validation runner, so a five-second text check became one CUDA-container job per python and CUDA row, behind a wheel build. It runs in the Python lint job instead, which is already ubuntu-latest and needs none of that. The claim that it needs "not even torch" was also wrong -- tests/py/dynamo/conftest.py imports torch at module scope, which is why the lint invocation passes --noconftest. The shell tier that runs the whole executorch directory now excludes the pin file too, so the dedup claim is true of both paths rather than just the manifest one. uv.lock still records the pre-bump range with no platform marker. uv-update.yml regenerates it on pushes to main touching setup.py, and only that workflow runs `uv sync --locked`, so this breaks nothing -- but the drift was invisible, since the lock writes a bare specifier the pin search cannot match. A strict=False xfail records it and turns into a real failure via XPASS once the lock is refreshed. Editing the lock by hand was the wrong fix: its resolved entry and hashes come from a resolver run against the nightly index. Also removes internal shorthand from the PR description, and corrects a line citation for the one deliberate range in executorch-build-linux.yml. --- .github/workflows/linter.yml | 9 ++ tests/ci/suites.py | 13 -- .../dynamo/executorch/test_executorch_pin.py | 113 +++++++++++++++++- tests/py/utils/ci_helpers.sh | 4 +- 4 files changed, 124 insertions(+), 15 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index af16185129..7d23759695 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -88,3 +88,12 @@ jobs: python3 $GITHUB_WORKSPACE/.github/scripts/run_py_linter.py env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + # Text and metadata only: no GPU, no ExecuTorch, no built wheel. It belongs on this runner + # rather than in the tests/ci manifest, where every suite becomes a CUDA-container job + # after `needs: build` -- one GPU row per python/CUDA combination for a check that takes + # seconds. --no-header keeps it clear of tests/py/dynamo/conftest.py, which imports torch. + - name: Check the ExecuTorch pin is consistent + run: | + cd $GITHUB_WORKSPACE + python3 -m pytest tests/py/dynamo/executorch/test_executorch_pin.py \ + -q --no-header -p no:cacheprovider --noconftest -o addopts="" diff --git a/tests/ci/suites.py b/tests/ci/suites.py index 19cc2a7c3d..fa9b6655c8 100644 --- a/tests/ci/suites.py +++ b/tests/ci/suites.py @@ -255,19 +255,6 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: keyword="not test_000_ and not test_001_", jobs=_HEAVY, ), - Suite( - # Separate from the executorch suite below because it needs none of what that one needs: - # no GPU, no ExecuTorch, not even torch. Keeping it here in the nightly-only suite meant - # the drift checks never ran on a PR or on a push to main, which is exactly when a pin - # goes stale. Text and metadata only, so it is cheap enough for every lane. - "executorch-pin", - tier="l0", - lanes=("fast", "full", "nightly"), - paths=("executorch/test_executorch_pin.py",), - jobs="auto", - variants=("standard",), - platforms=("linux-x86_64",), - ), Suite( "executorch", tier="l2", diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index 726c923a58..05324cd349 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -76,7 +76,8 @@ "docker/MODULE.bazel.docker": 1, "docker/MODULE.bazel.ngc": 1, "justfile": 1, - "py/torch-tensorrt-executorch-runtime/README.md": 1, + # Two: the install command and the prose sentence below it. + "py/torch-tensorrt-executorch-runtime/README.md": 2, "py/torch-tensorrt-executorch-runtime/pyproject.toml": 1, "toolchains/ci_workspaces/MODULE.bazel.tmpl": 1, } @@ -251,6 +252,39 @@ def test_derived_requirements_match_the_pin() -> None: ) assert _runner_requirement(REPO_ROOT) == f"executorch=={version}" + # The argument list CI actually runs, not just the string the helper derives. Dropping the + # requirement from the setup command left every one of these tests green: the step still + # succeeds, having installed no ExecuTorch, and the suite then skips on importorskip. + sys.path.insert(0, str(REPO_ROOT / "tests")) + from ci.runner import _executorch_requirement, _setup_commands + + argv = [arg for command, _cwd in _setup_commands("executorch") for arg in command] + assert argv.count(_executorch_requirement()) == 1, ( + "the executorch setup step does not install the pinned ExecuTorch exactly once: " + f"{argv}" + ) + + # And the extra every documented `pip install "torch-tensorrt[executorch]"` relies on. + # Emptying it left these tests green too. Read as source rather than imported, because + # importing the top-level setup.py executes it. + setup_tree = ast.parse((REPO_ROOT / "setup.py").read_text(encoding="utf-8")) + extras = next( + node.value + for node in ast.walk(setup_tree) + if isinstance(node, ast.Assign) + and any(getattr(t, "id", None) == "EXTRAS_REQUIRE" for t in node.targets) + ) + for key, value in zip(extras.keys, extras.values): + named = [ + element.id + for element in getattr(value, "elts", []) + if isinstance(element, ast.Name) + ] + assert named.count("EXECUTORCH_REQUIREMENT") == 1, ( + f"extra {getattr(key, 'value', key)!r} does not reference " + f"EXECUTORCH_REQUIREMENT exactly once: {named}" + ) + # docgen builds its overlay with a shell substitution, so neither the literal search nor the # two helpers above can see it: `$(` is not a digit. Run the command it embeds and compare # what it prints, which fails if the line is deleted or the key is renamed. @@ -408,3 +442,80 @@ def test_every_source_commit_matches_the_pin() -> None: assert found, "no ExecuTorch source commit found, so this test is not looking" _assert_every_site_present(seen, _EXPECTED_COMMIT_SITES, "naming the source commit") assert not wrong, f"pin says {commit}:\n " + "\n ".join(wrong) + + +@pytest.mark.unit +@pytest.mark.parametrize("setup_rc,expected", [(0, 0), (7, 7)]) +def test_a_failed_setup_step_stops_the_suite(monkeypatch, setup_rc, expected): + """A setup step that fails must fail the run, not warn and continue into pytest. + + Most of the ExecuTorch suite gates on ``pytest.importorskip``, so an install that fails makes + those files skip while everything else passes: the run reports success precisely when the + thing it exists to test is absent. That matters here because the pin names a nightly build, + which the channel eventually prunes. Replacing the ``return rc`` with ``continue`` kept every + other test in this file green, so assert on ``run_suite`` itself. + """ + sys.path.insert(0, str(REPO_ROOT / "tests")) + from ci import runner + + calls: list[list[str]] = [] + + class Completed: + def __init__(self, argv): + # The setup step is the pip install; anything else is pytest, which must not run + # at all once setup has failed. + self.returncode = setup_rc if "pip" in argv else 0 + + def record(argv, **kwargs): + calls.append(argv) + return Completed(argv) + + monkeypatch.setattr(runner.subprocess, "run", record) + suite = next(s for s in runner.SUITES if s.name == "executorch") + rc = runner.run_suite(suite, "standard") + + assert rc == expected, f"run_suite returned {rc}, expected {expected}" + ran_pytest = any("pytest" in " ".join(argv) for argv in calls) + assert ran_pytest is (setup_rc == 0), ( + "pytest ran even though a setup step failed" + if ran_pytest + else "pytest never ran even though every setup step succeeded" + ) + + +@pytest.mark.unit +@pytest.mark.xfail( + reason="uv.lock is regenerated by uv-update.yml on push to main, not by hand", + strict=False, +) +def test_the_lockfile_records_the_same_executorch_range_as_setup_py(): + """``uv.lock`` caches what ``setup.py`` declares, so it drifts when the pin moves. + + An xfail, not a failure: ``.github/workflows/uv-update.yml`` regenerates the lock on pushes to + main that touch ``setup.py``, and only that workflow runs ``uv sync --locked``, so a stale lock + breaks nothing here. Regenerating it by hand is worse than leaving it -- the resolved entry and + its hashes come from a resolver run against the nightly index, which cannot be faked in an + editor. This exists so the drift is visible and so it turns into a real failure, via XPASS, the + moment the lock is refreshed. The literal pin search cannot see this file: it writes + ``specifier = ">=1.4.1,<1.5"``, with no ``executorch==`` for the grep to match. + """ + lock = REPO_ROOT / "uv.lock" + if not lock.is_file(): + pytest.skip("no uv.lock in this checkout") + + recorded = set( + re.findall( + r'\{ name = "executorch", marker = "[^"]*", specifier = "([^"]+)" \}', + lock.read_text(encoding="utf-8"), + ) + ) + if not recorded: + pytest.skip("uv.lock records no executorch requirement") + + version = _versions()["__executorch_version__"] + major, minor = _release_line(version) + expected = f">={version},<{major}.{int(minor) + 1}" + assert recorded == {expected}, ( + f"uv.lock records executorch {sorted(recorded)} but the pin derives {expected!r}. " + "Run `uv lock --refresh` and commit the result." + ) diff --git a/tests/py/utils/ci_helpers.sh b/tests/py/utils/ci_helpers.sh index 7f3b4e5861..f6b0bb4042 100755 --- a/tests/py/utils/ci_helpers.sh +++ b/tests/py/utils/ci_helpers.sh @@ -159,8 +159,10 @@ trt_tier_l2_dynamo_core() { } trt_tier_executorch() { + # The pin checks are excluded here because the lint workflow already runs them on a CPU + # runner; this tier needs a GPU and a built wheel, which they do not. ( cd "${TRT_REPO_ROOT}/tests/py/dynamo" - _trt_py -m pytest -ra $(_trt_nproc auto) --junitxml="$(_trt_xml executorch_tests_results)" executorch/ "$@" ) + _trt_py -m pytest -ra $(_trt_nproc auto) --junitxml="$(_trt_xml executorch_tests_results)" -k "not test_executorch_pin" executorch/ "$@" ) } trt_tier_l2_plugin() { From 894d647f6af91613486e5a4d01dd9404a0d147b7 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 23 Aug 2026 23:22:52 -0700 Subject: [PATCH 08/15] Make the pin checks run in CI, and test the properties they assert The lint step added for these checks could not execute. It invokes pytest, and the job installs .github/scripts/requirements.txt (PyGithub) plus the lint dependency group (black, clang-format); neither carries pytest, so the step exited 1 on "No module named pytest" before running a single assertion. pyyaml is needed too, because reading the pin file shells out to a yaml import. Both are installed now, and a test asserts the step exists and installs them, since deleting it is otherwise invisible: every assertion here still passes locally while nothing runs it on a pull request. Reproduced the failure in a stdlib-only venv and confirmed the fixed command passes with only those two. The step also gets if: always(), so an unrelated formatting failure earlier in the job no longer hides the pin check. Three properties the checks are supposed to protect had no coverage: Deleting both published extras from EXTRAS_REQUIRE left everything green. The loop iterated whatever keys existed, so removing them iterated nothing and was indistinguishable from them being correct. It now requires the two published keys to be present, and only those, which also stops an unrelated future extra from turning this red for naming no ExecuTorch. The workflow opt-out marker was ordinary prose, "verify the end user's workflow". Pasting that sentence above a requirement and widening it to a range passed. It is an explicit token now, and the upward scan walks through comment lines to find it, so a cosmetic line between the opt-out and the requirement neither reclassifies the site nor fails the build. Nothing asserted that printed install instructions name the nightly channel, which is why that regressed and was re-fixed three times in this change without anything noticing. One test covers all of them by reading whole blocks rather than single lines, since every instruction wraps and the index lands on a continuation. It catches the CI install of the locally built wheel too, which carries no extra and is the site that broke most often. Generated docs under docs/ are excluded: corrections belong in docsrc/, and the committed Sphinx output is stale there independently. Also: the executorch requirement now strips its local version label like the other four, so the wheel does not bind itself to one CUDA train; the lockfile xfail is strict, since a non-strict xfail reports XPASS and ignores it and so could never fail; the fail-closed comment says it covers every setup step rather than implying only executorch; an empty frozenset and the dead branch reading it are gone; and the sys.path mutations use monkeypatch so they do not leak between tests. --- .github/workflows/executorch-build-linux.yml | 3 +- .github/workflows/linter.yml | 11 +- py/torch-tensorrt-executorch-runtime/setup.py | 2 +- tests/ci/runner.py | 4 +- .../dynamo/executorch/test_executorch_pin.py | 129 ++++++++++++++++-- 5 files changed, 129 insertions(+), 20 deletions(-) diff --git a/.github/workflows/executorch-build-linux.yml b/.github/workflows/executorch-build-linux.yml index e8e88367c4..33d190d051 100644 --- a/.github/workflows/executorch-build-linux.yml +++ b/.github/workflows/executorch-build-linux.yml @@ -131,7 +131,8 @@ jobs: executorch_cmake_location="$(bazel query @executorch//:executorch/CMakeLists.txt --output=location)" export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" - # this is to verify the end user's workflow + # pin-check: range-ok -- this is to verify the end user's workflow, which resolves a + # range the way a user would rather than the exact artifact the delegate links. python -m pip install pyyaml --extra-index-url "${EXECUTORCH_INDEX_URL}" "executorch>=1.5.0.dev20260822,<1.6" python examples/torchtrt_executorch_example/export_static_shape.py \ --model_path="${RUNNER_TEMP}/torchtrt-reference-runner.pte" diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 7d23759695..76896dbd95 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -40,6 +40,9 @@ jobs: run: | uv pip install --system -r $GITHUB_WORKSPACE/.github/scripts/requirements.txt python3 -c "import tomllib, subprocess; deps = tomllib.load(open('pyproject.toml', 'rb'))['dependency-groups']['lint']; subprocess.run(['uv', 'pip', 'install', '--system'] + deps, check=True)" + # The pin check below runs under pytest and shells out to a yaml reader. Neither is in + # requirements.txt or dependency-groups.lint, so the step exited 1 without running. + uv pip install --system pytest pyyaml - name: Lint C++ run: | cd $GITHUB_WORKSPACE @@ -82,6 +85,9 @@ jobs: run: | uv pip install --system -r $GITHUB_WORKSPACE/.github/scripts/requirements.txt python3 -c "import tomllib, subprocess; deps = tomllib.load(open('pyproject.toml', 'rb'))['dependency-groups']['lint']; subprocess.run(['uv', 'pip', 'install', '--system'] + deps, check=True)" + # The pin check below runs under pytest and shells out to a yaml reader. Neither is in + # requirements.txt or dependency-groups.lint, so the step exited 1 without running. + uv pip install --system pytest pyyaml - name: Lint Python run: | cd $GITHUB_WORKSPACE @@ -91,9 +97,10 @@ jobs: # Text and metadata only: no GPU, no ExecuTorch, no built wheel. It belongs on this runner # rather than in the tests/ci manifest, where every suite becomes a CUDA-container job # after `needs: build` -- one GPU row per python/CUDA combination for a check that takes - # seconds. --no-header keeps it clear of tests/py/dynamo/conftest.py, which imports torch. + # seconds. --noconftest keeps it clear of tests/py/dynamo/conftest.py, which imports torch. - name: Check the ExecuTorch pin is consistent + if: always() run: | cd $GITHUB_WORKSPACE python3 -m pytest tests/py/dynamo/executorch/test_executorch_pin.py \ - -q --no-header -p no:cacheprovider --noconftest -o addopts="" + -q --no-header -p no:cacheprovider --noconftest -o addopts="" diff --git a/py/torch-tensorrt-executorch-runtime/setup.py b/py/torch-tensorrt-executorch-runtime/setup.py index dc3d853556..bb6e1a1078 100644 --- a/py/torch-tensorrt-executorch-runtime/setup.py +++ b/py/torch-tensorrt-executorch-runtime/setup.py @@ -157,7 +157,7 @@ def build_extension(self, ext: Extension) -> None: install_requires=[ f"torch=={public_version(torch.__version__)}", f"executorch=={public_version(executorch_version)}", - f"torch-tensorrt=={torchtrt_version()}", + f"torch-tensorrt=={public_version(torchtrt_version())}", f"{TENSORRT_DISTRIBUTION}=={tensorrt_version}", f"{CUDA_RUNTIME_DISTRIBUTION}=={cuda_runtime_version}", ], diff --git a/tests/ci/runner.py b/tests/ci/runner.py index 727de20e47..2d904c5777 100644 --- a/tests/ci/runner.py +++ b/tests/ci/runner.py @@ -220,7 +220,9 @@ def run_suite( print(f"==> setup[{step}]: {shlex.join(argv)}", flush=True) rc = subprocess.run(argv, cwd=scwd, env=env).returncode if rc != 0: - # Fail rather than warn and continue. Most of the executorch suite gates on + # Fail rather than warn and continue, for every setup step and not just the + # executorch one: a suite whose dependencies did not install cannot test what it + # was asked to test, whichever step failed. Most of the executorch suite gates on # pytest.importorskip, so a failed install skips those files, leaves the rest # passing, and reports success with a populated junit xml -- the run looks green # precisely when the thing it exists to test is absent. This matters more now diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index 05324cd349..1db139e7ef 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -56,12 +56,13 @@ # # Empty today: the only literal range left was the justfile's install recipe, and it installs # the wheel the delegate is compiled against, so it pins exactly like the rest. -RANGE_SITES: frozenset[str] = frozenset() # A step that exists to reproduce what a user runs belongs to the range group even inside a # file that otherwise pins build inputs, so the marker travels with the line rather than # with the path. -USER_WORKFLOW_MARKER = "verify the end user's workflow" +# An explicit opt-out token rather than prose. "verify the end user's workflow" is a sentence +# someone can write, or paste, above a requirement without meaning to license a range there. +USER_WORKFLOW_MARKER = "pin-check: range-ok" # The files expected to pin ExecuTorch, mapped to how many sites each must carry, excluding # dev_dep_versions.yml itself. A count per file rather than just the set of files, because a site @@ -133,16 +134,19 @@ def _versions() -> dict: def _wants_range(path: str, number: int) -> bool: - if path in RANGE_SITES: - return True - - # Scan upward rather than reading one fixed line, so reformatting the workflow cannot - # silently reclassify the requirement and fail the build for an unrelated reason. + # Scan upward past blanks and comment lines, so an explanatory line between the opt-out and + # the requirement neither reclassifies the site nor fails the build for a cosmetic reason. + # Only a comment carrying the token licenses a range; the first line of real content stops + # the scan, so the opt-out cannot leak onto an unrelated requirement further down. lines = (REPO_ROOT / path).read_text().splitlines() for line in reversed(lines[: number - 1]): - if not line.strip(): + stripped = line.strip() + if not stripped: continue - return USER_WORKFLOW_MARKER in line + if not stripped.startswith("#"): + return False + if USER_WORKFLOW_MARKER in stripped: + return True return False @@ -233,7 +237,7 @@ def _runner_requirement(root: Path) -> str: ).stdout.strip() -def test_derived_requirements_match_the_pin() -> None: +def test_derived_requirements_match_the_pin(monkeypatch) -> None: # setup.py and tests/ci/runner.py build their requirement from the pin, so the search # above cannot see them. Check the strings they produce instead. # @@ -255,7 +259,7 @@ def test_derived_requirements_match_the_pin() -> None: # The argument list CI actually runs, not just the string the helper derives. Dropping the # requirement from the setup command left every one of these tests green: the step still # succeeds, having installed no ExecuTorch, and the suite then skips on importorskip. - sys.path.insert(0, str(REPO_ROOT / "tests")) + monkeypatch.syspath_prepend(str(REPO_ROOT / "tests")) from ci.runner import _executorch_requirement, _setup_commands argv = [arg for command, _cwd in _setup_commands("executorch") for arg in command] @@ -274,7 +278,19 @@ def test_derived_requirements_match_the_pin() -> None: if isinstance(node, ast.Assign) and any(getattr(t, "id", None) == "EXTRAS_REQUIRE" for t in node.targets) ) + # The published extras have to exist, or the loop below iterates nothing and deleting both + # keys is indistinguishable from them being correct. Only these two: an unrelated future extra + # has no reason to name ExecuTorch, and requiring it of every key made this test the one that + # turns red when someone adds "debug". + published = {"executorch", "all"} + present = {key.value for key in extras.keys if isinstance(key, ast.Constant)} + assert published <= present, ( + f"setup.py must publish the {sorted(published)} extras, but EXTRAS_REQUIRE has " + f"{sorted(present)}. Every documented install command names one of them." + ) for key, value in zip(extras.keys, extras.values): + if getattr(key, "value", None) not in published: + continue named = [ element.id for element in getattr(value, "elts", []) @@ -317,7 +333,7 @@ def test_the_runner_follows_the_row_s_cuda_version(monkeypatch) -> None: cu130, which is why watching PR CI cannot catch it. A source-text assertion could not catch it either: keeping the ``os.environ.get`` line while hardcoding the URL passes one. """ - sys.path.insert(0, str(REPO_ROOT / "tests")) + monkeypatch.syspath_prepend(str(REPO_ROOT / "tests")) try: from ci import runner finally: @@ -371,7 +387,7 @@ def test_the_pinned_commit_is_the_pinned_wheels_own_source() -> None: Every published wheel records the commit it was built from, so the pairing is checkable rather than a convention. - Skipped rather than failed whenever the installed wheel is not the one the pin names — not + Skipped rather than failed whenever the installed wheel is not the one the pin names, not installed at all, a different member of a floating range, or built without git provenance. None of those say anything about whether the two pins agree, and this file stays readable offline. @@ -455,7 +471,7 @@ def test_a_failed_setup_step_stops_the_suite(monkeypatch, setup_rc, expected): which the channel eventually prunes. Replacing the ``return rc`` with ``continue`` kept every other test in this file green, so assert on ``run_suite`` itself. """ - sys.path.insert(0, str(REPO_ROOT / "tests")) + monkeypatch.syspath_prepend(str(REPO_ROOT / "tests")) from ci import runner calls: list[list[str]] = [] @@ -486,7 +502,7 @@ def record(argv, **kwargs): @pytest.mark.unit @pytest.mark.xfail( reason="uv.lock is regenerated by uv-update.yml on push to main, not by hand", - strict=False, + strict=True, ) def test_the_lockfile_records_the_same_executorch_range_as_setup_py(): """``uv.lock`` caches what ``setup.py`` declares, so it drifts when the pin moves. @@ -519,3 +535,86 @@ def test_the_lockfile_records_the_same_executorch_range_as_setup_py(): f"uv.lock records executorch {sorted(recorded)} but the pin derives {expected!r}. " "Run `uv lock --refresh` and commit the result." ) + + +@pytest.mark.unit +def test_every_printed_install_instruction_names_the_nightly_channel(): + """Every ``[executorch]`` install instruction has to carry the nightly index. + + ExecuTorch is published only to the nightly CUDA channel, so an instruction without + ``--extra-index-url`` resolves nothing and the user gets a bare "no matching distribution". + The property had regressed and been re-fixed three times across this change with nothing + asserting it, which is the signature of a property no test covers. + + Whole files rather than single lines: every one of these instructions wraps, so the extra and + the index land on different lines and a line-oriented check sees neither together. Tracked + files only, so a stale build directory cannot fail this. + """ + tracked = subprocess.run( + ["git", "ls-files", "-z"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.split("\0") + + # Two shapes need the channel: an instruction naming the [executorch] extra, and the CI + # install of a locally built torch-tensorrt wheel, whose ExecuTorch dependency resolves from + # the same index. The second is the site that regressed most often and carries no extra. + extra = re.compile( + r"""torch[-_]tensorrt\[[^]]*executorch[^]]*\]|torch_tensorrt\*\.whl""" + ) + missing = [] + for name in tracked: + if not name or not name.endswith( + (".py", ".sh", ".md", ".yml", ".yaml", ".rst", ".txt") + ): + continue + # This file states the rule; it is not itself an instruction. + if name == "tests/py/dynamo/executorch/test_executorch_pin.py": + continue + # docs/ is Sphinx output committed to the tree. Its sources live in docsrc/, which is + # where a correction has to go, so flagging the generated copy sends the fix to a file + # the next docs build overwrites. + if name.startswith("docs/"): + continue + path = REPO_ROOT / name + if not path.is_file(): + continue + text = path.read_text(encoding="utf-8", errors="replace") + for match in extra.finditer(text): + # The instruction is the pip invocation, so bound the window at the surrounding + # blank-line-separated block rather than guessing a fixed number of lines. + start = text.rfind("\n\n", 0, match.start()) + 1 + end = text.find("\n\n", match.end()) + block = text[start : end if end != -1 else len(text)] + if "download.pytorch.org/whl/nightly" not in block: + line = text.count("\n", 0, match.start()) + 1 + missing.append(f"{name}:{line}") + + assert not missing, ( + "these ExecuTorch install instructions do not name the nightly channel, so they " + f"resolve no ExecuTorch at all: {missing}" + ) + + +@pytest.mark.unit +def test_the_pin_check_runs_in_ci(): + """This file has to be invoked by something, or its assertions never execute. + + Two suites deselect it by name so it does not need an installed ExecuTorch on a GPU runner, + which leaves the lint job as the only path that runs it. Deleting that step is invisible + otherwise: every test here still passes locally while nothing runs them in CI. + """ + workflow = (REPO_ROOT / ".github/workflows/linter.yml").read_text(encoding="utf-8") + assert ( + "test_executorch_pin.py" in workflow + ), "no CI job invokes this file, so nothing here runs on a pull request" + # And it needs pytest, which neither requirements.txt nor dependency-groups.lint provides. + # Without this the step exits 1 on "No module named pytest" before running any assertion. + assert re.search( + r"uv pip install --system[^\n]*\bpytest\b", workflow + ), "the job that runs this file does not install pytest, so the step cannot execute" + assert re.search( + r"uv pip install --system[^\n]*\bpyyaml\b", workflow + ), "the job that runs this file does not install pyyaml, which _pinned_versions() shells out to" From 76c3459c84cd7d5923dcd49b554a00cf7a277245 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 24 Aug 2026 08:26:24 -0700 Subject: [PATCH 09/15] Let the pairing check run on the one lane that installs ExecuTorch The check that the two pins name one ExecuTorch could not run anywhere. It skips unless the installed wheel is exactly the pinned version, so it means something only on the nightly GPU lane, and that lane deselected it. The deselection is written as "not test_executorch_pin" to skip the source-consistency checks in the same file, but -k matches the module name in the test id, so it dropped every test in the module including this one. Both deselection sites now keep it by name. Proved it on a host with the pinned wheel installed, whose recorded git_version is the pinned commit: the check passes at the correct pins, fails when the commit pin names a different tree, and fails when the commit pin is deleted outright. Before this it was deselected in all three states. Bumping the version alone still skips, correctly, because the installed wheel is then not the one the pin names and its provenance says nothing about whether the two pins agree. A test asserts both sites keep it, since re-tightening either one to a bare module name is a small and plausible edit that would silently restore the gap. --- tests/ci/suites.py | 7 +++++- .../dynamo/executorch/test_executorch_pin.py | 22 +++++++++++++++++++ tests/py/utils/ci_helpers.sh | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/ci/suites.py b/tests/ci/suites.py index fa9b6655c8..7a75709c42 100644 --- a/tests/ci/suites.py +++ b/tests/ci/suites.py @@ -260,7 +260,12 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: tier="l2", lanes=("nightly",), paths=("executorch/",), - keyword="not test_executorch_pin", + keyword=( + # The pairing test is the one check here that needs a real ExecuTorch installed, so + # this lane is the only place it can run. Everything else in that file is a + # source-consistency check the lint job already covers. + "not test_executorch_pin or test_the_pinned_commit_is_the_pinned_wheels_own_source" + ), setup=("executorch",), jobs="auto", variants=("standard",), diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index 1db139e7ef..78e3fb00ca 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -62,6 +62,7 @@ # with the path. # An explicit opt-out token rather than prose. "verify the end user's workflow" is a sentence # someone can write, or paste, above a requirement without meaning to license a range there. +PAIRING_TEST = "test_the_pinned_commit_is_the_pinned_wheels_own_source" USER_WORKFLOW_MARKER = "pin-check: range-ok" # The files expected to pin ExecuTorch, mapped to how many sites each must carry, excluding @@ -618,3 +619,24 @@ def test_the_pin_check_runs_in_ci(): assert re.search( r"uv pip install --system[^\n]*\bpyyaml\b", workflow ), "the job that runs this file does not install pyyaml, which _pinned_versions() shells out to" + + +@pytest.mark.unit +def test_the_pairing_check_survives_the_gpu_lane_deselection() -> None: + """The one test here that needs a real ExecuTorch must not be deselected with the rest. + + Every other test in this file is a source-consistency check, so the GPU lane deselects the + whole module by name to avoid paying for them twice. ``-k`` matches the module name in the + test id, so a bare ``not test_executorch_pin`` drops the pairing check too, and that check + only means anything where ExecuTorch is installed. It was silently unreachable: it skips + when the wheel is not the pinned one, which is every environment except this lane. + """ + for path in ("tests/ci/suites.py", "tests/py/utils/ci_helpers.sh"): + text = (REPO_ROOT / path).read_text(encoding="utf-8") + assert ( + "not test_executorch_pin" in text + ), f"{path} no longer deselects this module" + assert PAIRING_TEST in text, ( + f"{path} deselects the whole module without keeping {PAIRING_TEST}, so the only " + "check that needs a real ExecuTorch installed runs nowhere" + ) diff --git a/tests/py/utils/ci_helpers.sh b/tests/py/utils/ci_helpers.sh index f6b0bb4042..e84889462b 100755 --- a/tests/py/utils/ci_helpers.sh +++ b/tests/py/utils/ci_helpers.sh @@ -162,7 +162,7 @@ trt_tier_executorch() { # The pin checks are excluded here because the lint workflow already runs them on a CPU # runner; this tier needs a GPU and a built wheel, which they do not. ( cd "${TRT_REPO_ROOT}/tests/py/dynamo" - _trt_py -m pytest -ra $(_trt_nproc auto) --junitxml="$(_trt_xml executorch_tests_results)" -k "not test_executorch_pin" executorch/ "$@" ) + _trt_py -m pytest -ra $(_trt_nproc auto) --junitxml="$(_trt_xml executorch_tests_results)" -k "not test_executorch_pin or test_the_pinned_commit_is_the_pinned_wheels_own_source" executorch/ "$@" ) } trt_tier_l2_plugin() { From 9e0f4b9baf7f04422b9d1a3ebd552c94021ae4f0 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 24 Aug 2026 09:56:28 -0700 Subject: [PATCH 10/15] Make the pin guards read values instead of nearby text Every guard added in this change asserted that a string appeared somewhere in a file, so each certified the state it was written to prevent. The keyword guard grepped for the kept test's name. Changing "or" to "and" in both -k expressions left it green, and that expression collects nothing at all, which is worse than the bug the guard exists to catch. Reverting the expressions and leaving the name behind in a comment also left it green, and a comment explaining the keyword sits directly above it, which is where an editor would naturally write that name. It now runs pytest's own collection under each expression and requires exactly the pairing test to come back. The CI guard searched the workflow as one blob, so it could not tell which job it was reading. The same commit that fixed the lint failure also added pytest and pyyaml to cpp-linting, which has no pin check, so deleting them from the job that does run it stayed green and would have restored the original failure invisibly. Neutralising the command while leaving its filename in a shell comment, and setting a falsy step condition, were also green. It now parses the workflow, finds the job that actually invokes pytest on this file, and requires the installs in an earlier step of that same job. The unused installs are gone from cpp-linting. The requirement pattern captured an equality prefix and stopped, so "executorch==PIN,!=PIN", a specifier that excludes the version it appears to pin, compared equal to the pin. The same truncation rejected the legal PEP 508 spelling with spaces around the operator. Requirements are parsed now and compared as specifier sets, with a check that the pinned version actually satisfies them. The site scanner counted raw search hits, so gutting a pin to a bare "executorch" while putting the exact pin in a comment in the same file kept the per-file minimum satisfied. Comments no longer count, except in the bazel repositories, where the annotation beside the pinned commit is the only record of which wheel that commit belongs to. Also corrected two claims this change made: the executorch tier is reachable from a pull request through executorch-test-linux.yml as well as the nightly manifest, so it is not the only route, and the shell helper now says why one test is kept out of the deselection. --- .github/workflows/linter.yml | 3 - tests/ci/suites.py | 2 +- .../dynamo/executorch/test_executorch_pin.py | 192 +++++++++++++++--- tests/py/utils/ci_helpers.sh | 4 +- 4 files changed, 172 insertions(+), 29 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 76896dbd95..26e45662c8 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -40,9 +40,6 @@ jobs: run: | uv pip install --system -r $GITHUB_WORKSPACE/.github/scripts/requirements.txt python3 -c "import tomllib, subprocess; deps = tomllib.load(open('pyproject.toml', 'rb'))['dependency-groups']['lint']; subprocess.run(['uv', 'pip', 'install', '--system'] + deps, check=True)" - # The pin check below runs under pytest and shells out to a yaml reader. Neither is in - # requirements.txt or dependency-groups.lint, so the step exited 1 without running. - uv pip install --system pytest pyyaml - name: Lint C++ run: | cd $GITHUB_WORKSPACE diff --git a/tests/ci/suites.py b/tests/ci/suites.py index 7a75709c42..5a78bd2756 100644 --- a/tests/ci/suites.py +++ b/tests/ci/suites.py @@ -262,7 +262,7 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: paths=("executorch/",), keyword=( # The pairing test is the one check here that needs a real ExecuTorch installed, so - # this lane is the only place it can run. Everything else in that file is a + # it has to survive this deselection. Everything else in that file is a # source-consistency check the lint job already covers. "not test_executorch_pin or test_the_pinned_commit_is_the_pinned_wheels_own_source" ), diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index 78e3fb00ca..3ba9941f26 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -17,6 +17,7 @@ import ast import os +import pathlib import re import shlex import subprocess @@ -34,10 +35,37 @@ # that drifted from the pin, and it should be visible to the search rather than silently # exempt. Operators are named through the pattern rather than spelled out in prose here, # because the search below reads this file too and an example would read as such a site. +# The whole specifier set, not just its first clause. Capturing up to the first comma compared +# equal on "executorch==PIN,!=PIN", a specifier that excludes the very version it appears to pin, +# and rejected the legal PEP 508 spelling with spaces around the operator. REQUIREMENT = re.compile( - r"executorch\s*(?:===|==|>=|<=|~=|!=|<|>)\s*[0-9][^\"'\s,`]*(?:,\s*<[0-9.]+)?" + r"executorch\s*(?:===|==|>=|<=|~=|!=|<|>)\s*[^\"'\s`,]+" + r"(?:\s*,\s*(?:===|==|>=|<=|~=|!=|<|>)\s*[^\"'\s`,]+)*" ) + +def _requirement_disagrees(actual: str, expected: str, version: str) -> str: + """Why ``actual`` is not the pinned requirement, or an empty string if it is. + + Compares parsed specifier sets rather than matched text. Raw text equality could not see a + clause the pattern did not capture, and treated whitespace the specification allows as drift. + """ + from packaging.requirements import InvalidRequirement, Requirement + + try: + parsed = Requirement(actual) + wanted = Requirement(expected) + except InvalidRequirement as error: + return f"which is not a valid requirement ({error})" + if parsed.name != wanted.name: + return f"which names {parsed.name}, not {wanted.name}" + if not parsed.specifier.contains(version, prereleases=True): + return f"whose specifier excludes the pinned {version}" + if set(parsed.specifier) != set(wanted.specifier): + return f"expected {expected}" + return "" + + # The bazel repository puts the commit on its own line, so this one has to run against file # contents rather than a git grep line. BAZEL_COMMIT = re.compile( @@ -173,6 +201,34 @@ def _expected(path: str, number: int, version: str) -> str: return f"executorch>={version},<{major}.{int(minor) + 1}{marker}" +# The bazel repositories annotate their pinned commit with the wheel it corresponds to, in a +# comment, because bazel fetches by commit and has no requirement string to carry. Those are the +# only comment sites that count as pins, and the commit beside them is checked separately. +_ANNOTATED_COMMIT_SITES = frozenset( + { + "MODULE.bazel", + "docker/MODULE.bazel.docker", + "docker/MODULE.bazel.ngc", + "toolchains/ci_workspaces/MODULE.bazel.tmpl", + } +) + + +def _is_commented_out(path: str, text: str) -> bool: + """Whether this requirement sits in a comment rather than in live configuration. + + A comment is not a pin: a site could be gutted to a bare ``executorch`` while the exact pin + lived on in a comment in the same file, which kept the per-file minimum satisfied and left the + real requirement unpinned. + """ + if path in _ANNOTATED_COMMIT_SITES: + return False + stripped = text.strip() + if path.endswith((".md", ".rst", ".txt")): + return False + return stripped.startswith(("#", "//", "/*", "*")) + + def test_every_requirement_matches_the_pin() -> None: version = _versions()["__executorch_version__"] @@ -185,12 +241,18 @@ def test_every_requirement_matches_the_pin() -> None: path, number, text = line.split(":", 2) if path == VERSIONS.name: continue + if _is_commented_out(path, text): + # A comment is not a pin. Counting raw matches meant a site could be gutted to a bare + # "executorch" while the exact pin lived on in a comment in the same file, keeping the + # per-file minimum satisfied. + continue expected = _expected(path, int(number), version) for actual in REQUIREMENT.findall(text): found += 1 seen[path] += 1 - if actual != expected: - wrong.append(f"{path}:{number} has {actual}, expected {expected}") + reason = _requirement_disagrees(actual, expected, version) + if reason: + wrong.append(f"{path}:{number} has {actual}, {reason}") assert found, "no ExecuTorch requirement found, so this test is not looking" _assert_every_site_present(seen, _EXPECTED_REQUIREMENT_SITES, "pinning ExecuTorch") @@ -607,36 +669,118 @@ def test_the_pin_check_runs_in_ci(): which leaves the lint job as the only path that runs it. Deleting that step is invisible otherwise: every test here still passes locally while nothing runs them in CI. """ - workflow = (REPO_ROOT / ".github/workflows/linter.yml").read_text(encoding="utf-8") + # Parse the workflow and assert inside the owning job. Searching the file as one blob could + # not tell which job it was reading, so an identical install line in a sibling job that has no + # pin check satisfied it, and deleting the real one stayed green. A commented-out step also + # vanishes from the parse, where a text search still finds it. + import yaml + + workflow = yaml.safe_load( + (REPO_ROOT / ".github/workflows/linter.yml").read_text(encoding="utf-8") + ) + # Match a live pytest invocation, not the filename anywhere in the script. Neutralising the + # command and leaving it in a shell comment satisfied a plain substring test. + invocation = re.compile( + rf"^\s*[^#\n]*\bpytest\b[^\n]*{re.escape(pathlib.Path(__file__).name)}", + re.MULTILINE, + ) + owning = [ + (name, job, step) + for name, job in workflow["jobs"].items() + for step in job.get("steps", []) + if invocation.search(step.get("run") or "") + ] + assert owning, "no CI job invokes this file, so nothing here runs on a pull request" + name, job, step = owning[0] + + # A falsy condition disables the step while leaving every string in place. + condition = str(step.get("if", "always()")) + assert condition in { + "always()", + "success()", + "success() || failure()", + }, f"the pin check in {name} runs under {condition!r}, which may never be true" assert ( - "test_executorch_pin.py" in workflow - ), "no CI job invokes this file, so nothing here runs on a pull request" - # And it needs pytest, which neither requirements.txt nor dependency-groups.lint provides. - # Without this the step exits 1 on "No module named pytest" before running any assertion. - assert re.search( - r"uv pip install --system[^\n]*\bpytest\b", workflow - ), "the job that runs this file does not install pytest, so the step cannot execute" - assert re.search( - r"uv pip install --system[^\n]*\bpyyaml\b", workflow - ), "the job that runs this file does not install pyyaml, which _pinned_versions() shells out to" + "--collect-only" not in step["run"] + ), f"the pin check in {name} only collects tests, so no assertion executes" + + # pytest and pyyaml must be installed by an earlier step of the SAME job: neither + # requirements.txt nor dependency-groups.lint carries them, and without them the step exits 1 + # on "No module named pytest" before running any assertion. + steps = job["steps"] + earlier = "\n".join(s.get("run") or "" for s in steps[: steps.index(step)]) + for package in ("pytest", "pyyaml"): + assert re.search( + rf"uv pip install --system[^\n]*\b{package}\b", earlier + ), f"job {name} does not install {package} before the pin check, so the step cannot run" @pytest.mark.unit def test_the_pairing_check_survives_the_gpu_lane_deselection() -> None: """The one test here that needs a real ExecuTorch must not be deselected with the rest. - Every other test in this file is a source-consistency check, so the GPU lane deselects the - whole module by name to avoid paying for them twice. ``-k`` matches the module name in the + Every other test in this file is a source-consistency check, so the executorch tier deselects + the whole module by name to avoid paying for them twice. ``-k`` matches the module name in the test id, so a bare ``not test_executorch_pin`` drops the pairing check too, and that check - only means anything where ExecuTorch is installed. It was silently unreachable: it skips - when the wheel is not the pinned one, which is every environment except this lane. + only means anything where ExecuTorch is installed, which is nowhere the lint job runs. + + Two routes reach this tier: the nightly manifest suite in ``tests/ci/suites.py``, and + ``executorch-test-linux.yml``, which installs the pinned wheel and runs on pull requests once + the runtime build succeeds. Both go through one of the two keyword expressions checked here. """ - for path in ("tests/ci/suites.py", "tests/py/utils/ci_helpers.sh"): + # Run pytest's own collection under each expression rather than grepping for the name. A + # string test passes on "and" in place of "or", which collects nothing at all, and on the + # name surviving only in a comment. Both leave the pairing check unreachable. + module = pathlib.Path(__file__).name + for path, pattern in ( + ("tests/ci/suites.py", r'keyword=\(\s*(?:#[^\n]*\n\s*)*"([^"]+)"'), + # Anchored on the executorch junitxml name, because the file passes -k in several + # functions and the first match belongs to a different tier. + ( + "tests/py/utils/ci_helpers.sh", + r'executorch_tests_results[^\n]*?-k "([^"]+)"', + ), + ): text = (REPO_ROOT / path).read_text(encoding="utf-8") + found = re.search(pattern, text) assert ( - "not test_executorch_pin" in text - ), f"{path} no longer deselects this module" - assert PAIRING_TEST in text, ( - f"{path} deselects the whole module without keeping {PAIRING_TEST}, so the only " - "check that needs a real ExecuTorch installed runs nowhere" + found + ), f"{path} no longer passes a single -k expression this test can read" + keyword = found.group(1) + assert "not test_executorch_pin" in keyword, ( + f"{path} no longer deselects this module, so the source-consistency checks here " + "would run twice" + ) + selected = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + str(pathlib.Path(__file__).parent), + "--collect-only", + "-q", + "--noconftest", + "-p", + "no:cacheprovider", + "-o", + "addopts=", + "-k", + keyword, + ], + capture_output=True, + text=True, + cwd=REPO_ROOT, + ).stdout + assert f"{module}::{PAIRING_TEST}" in selected, ( + f"{path} runs pytest with -k {keyword!r}, which does not select {PAIRING_TEST}, so " + "the only check that needs a real ExecuTorch installed runs nowhere" + ) + others = [ + line + for line in selected.splitlines() + if module in line and PAIRING_TEST not in line + ] + assert not others, ( + f"{path} selects {len(others)} other tests from this module, which the lane " + f"deselects deliberately: {others[:2]}" ) diff --git a/tests/py/utils/ci_helpers.sh b/tests/py/utils/ci_helpers.sh index e84889462b..ff24cbf2e7 100755 --- a/tests/py/utils/ci_helpers.sh +++ b/tests/py/utils/ci_helpers.sh @@ -160,7 +160,9 @@ trt_tier_l2_dynamo_core() { trt_tier_executorch() { # The pin checks are excluded here because the lint workflow already runs them on a CPU - # runner; this tier needs a GPU and a built wheel, which they do not. + # runner; this tier needs a GPU and a built wheel, which they do not. The one exception is + # kept by name: it compares the pinned commit against the installed wheel's own recorded + # source, so it needs an ExecuTorch the lint runner does not have and skips everywhere else. ( cd "${TRT_REPO_ROOT}/tests/py/dynamo" _trt_py -m pytest -ra $(_trt_nproc auto) --junitxml="$(_trt_xml executorch_tests_results)" -k "not test_executorch_pin or test_the_pinned_commit_is_the_pinned_wheels_own_source" executorch/ "$@" ) } From 39c2552e76b3890b514c46238835a94b62ddf895 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 24 Aug 2026 11:45:57 -0700 Subject: [PATCH 11/15] Stop the lockfile check from going red repo-wide on the next refresh The uv.lock check was a strict xfail. uv.lock records ">=1.4.1,<1.5" while the pin derives ">=1.5.0.dev20260822,<1.6", so the assertion fails and the xfail is satisfied. Refresh the lock and the assertion passes, and a strict xfail reports that pass as a failure. The lint step runs this file with if: always() on every pull request, so one lock refresh would have made the lint job red on every subsequent pull request, for a file none of them touched, until someone edited this test. Measured: baseline 1 xfailed, and 1 failed once the specifier is bumped. My own docstring claimed the lock is machine-generated and not edited by hand. Two hand refreshes landed on 2026-08-23, inside ordinary version-bump changes, so that was wrong as well. It now accepts both resting states and only fails where something is actually wrong: a recorded range whose lower bound is above the pin, which means the lock names an ExecuTorch this repository does not pin. Behind the pin passes, the derived range passes, and ">=1.6,<1.7", an open-ended ">=1.7" and "==1.9.0" all fail. Comparing lower bounds rather than probing the specifier with sample versions: an upper-bound test missed the open-ended case, and a low sentinel version called the ordinary behind-the-pin state a failure. --- .../dynamo/executorch/test_executorch_pin.py | 53 ++++++++++++++----- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index 3ba9941f26..d0392d95cb 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -563,20 +563,22 @@ def record(argv, **kwargs): @pytest.mark.unit -@pytest.mark.xfail( - reason="uv.lock is regenerated by uv-update.yml on push to main, not by hand", - strict=True, -) def test_the_lockfile_records_the_same_executorch_range_as_setup_py(): """``uv.lock`` caches what ``setup.py`` declares, so it drifts when the pin moves. - An xfail, not a failure: ``.github/workflows/uv-update.yml`` regenerates the lock on pushes to - main that touch ``setup.py``, and only that workflow runs ``uv sync --locked``, so a stale lock - breaks nothing here. Regenerating it by hand is worse than leaving it -- the resolved entry and - its hashes come from a resolver run against the nightly index, which cannot be faked in an - editor. This exists so the drift is visible and so it turns into a real failure, via XPASS, the - moment the lock is refreshed. The literal pin search cannot see this file: it writes - ``specifier = ">=1.4.1,<1.5"``, with no ``executorch==`` for the grep to match. + A stale lock breaks nothing here, because only ``uv-update.yml`` runs ``uv sync --locked``, + and its resolved hashes come from a resolver run against the nightly index that cannot be + faked in an editor. So this accepts two states: the range the pin derives, and a range that + predates the pin. + + It used to be a strict xfail, which meant the moment anyone refreshed the lock the assertion + passed and pytest reported that pass as a failure. The lint step runs this file on every pull + request, so that would have turned the lint job red repo-wide for a file none of those pull + requests touched. Nor is the lock only machine-generated: it was hand-refreshed twice inside + ordinary version-bump changes on 2026-08-23. + + The literal pin search cannot see this file: it writes ``specifier = ">=1.4.1,<1.5"``, with no + ``executorch==`` for the grep to match. """ lock = REPO_ROOT / "uv.lock" if not lock.is_file(): @@ -594,9 +596,32 @@ def test_the_lockfile_records_the_same_executorch_range_as_setup_py(): version = _versions()["__executorch_version__"] major, minor = _release_line(version) expected = f">={version},<{major}.{int(minor) + 1}" - assert recorded == {expected}, ( - f"uv.lock records executorch {sorted(recorded)} but the pin derives {expected!r}. " - "Run `uv lock --refresh` and commit the result." + if recorded == {expected}: + return + + # Behind the pin is the expected resting state until the lock is regenerated. Ahead of it is + # not: that means the lock names an ExecuTorch this repository does not pin. + from packaging.specifiers import SpecifierSet + from packaging.version import Version + + # Ahead means the range's own lower bound is above the pin. Probing the specifier with sample + # versions was fragile in both directions: an upper-bound test missed an open-ended ">=1.7", + # and a low sentinel called the ordinary behind-the-pin state a failure. + pinned = Version(version) + ahead = [ + entry + for entry in sorted(recorded) + if any( + clause.operator in {">=", ">", "==", "~=", "==="} + and Version(clause.version.rstrip("*") or "0") > pinned + for clause in SpecifierSet(entry) + ) + ] + assert not ahead, ( + f"uv.lock records executorch {ahead}, which is ahead of the pinned {version}. The pin " + "derives " + + repr(expected) + + ", so run `uv lock --refresh` and commit the result." ) From b4d994503ae19bde5ed7a6185443d55c1192fb63 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 24 Aug 2026 13:30:59 -0700 Subject: [PATCH 12/15] Stop a test executing Python read out of a workflow file test_derived_requirements_match_the_pin extracted the python3 -c one-liner from docgen.yml and ran it. Whatever that line said got executed on every pull request: rewriting it to write a file left the test green and the file written. Same class as the bash -c problem fixed in test_api.py last round, still live here. It now compares the command as text against the exact form that reads __executorch_version__ out of dev_dep_versions.yml. Four mutations caught, including a payload that writes a file and still prints the right version, with nothing executed. The CI reachability guard tested the raw string for "--collect-only", so it accepted "--co", pytest's own documented short form, which collects and asserts nothing. It also could not see an exit status being discarded. Now tokenised: --collect-only, --co, -h, --help, a "||" short-circuit and continue-on-error are all rejected, and all five are caught where four previously survived. The comment exemption for .md/.rst/.txt defeated exactly the threat its docstring names. Install commands live in prose files, so exempting them made a comment count as a pin there: the runtime README's install line gutted to a bare "executorch" passed as long as a decoy "# executorch==" sat beside it, and failed only with no comment present. The exemption is gone, and trailing comments no longer count either, since a decoy after a live requirement on the same line kept the per-file count satisfied. Five mutations caught, baseline green. --- .../dynamo/executorch/test_executorch_pin.py | 64 ++++++++++++++----- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index d0392d95cb..1c70510d43 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -224,11 +224,23 @@ def _is_commented_out(path: str, text: str) -> bool: if path in _ANNOTATED_COMMIT_SITES: return False stripped = text.strip() - if path.endswith((".md", ".rst", ".txt")): - return False + # No exemption for prose. Returning False for .md/.rst/.txt defeated the threat named above, + # because it made a comment count as a pin in exactly the files where install commands live: a + # README install line gutted to a bare "executorch" passed as long as a decoy "# executorch==" + # sat beside it. A "#" inside a fenced shell block is a shell comment, the same as anywhere else. return stripped.startswith(("#", "//", "/*", "*")) +def _without_trailing_comment(path: str, text: str) -> str: + """``text`` up to a trailing ``#`` or ``//`` comment, unless the site annotates its pin there.""" + if path in _ANNOTATED_COMMIT_SITES: + return text + for marker in ("#", "//"): + if marker in text: + text = text.split(marker, 1)[0] + return text + + def test_every_requirement_matches_the_pin() -> None: version = _versions()["__executorch_version__"] @@ -247,7 +259,12 @@ def test_every_requirement_matches_the_pin() -> None: # per-file minimum satisfied. continue expected = _expected(path, int(number), version) - for actual in REQUIREMENT.findall(text): + # A trailing comment is not a pin either. Skipping whole-line comments was not enough: a + # live install gutted to a bare "executorch" with a decoy "# executorch==" after it on + # the same line kept the per-file count satisfied and left the install unpinned. The + # annotated commit sites write their pin as a whole-line comment, which is handled above, + # so nothing legitimate is lost here. + for actual in REQUIREMENT.findall(_without_trailing_comment(path, text)): found += 1 seen[path] += 1 reason = _requirement_disagrees(actual, expected, version) @@ -374,18 +391,20 @@ def test_derived_requirements_match_the_pin(monkeypatch) -> None: "with --pre from the nightly channel, so without the pin it resolves through the range " "and takes whichever dev build is newest that day." ) - printed = subprocess.run( - # The interpreter running the test, not the workflow's bare `python3`, which need not - # have pyyaml here. The argument list is the workflow's own. - [sys.executable, *shlex.split(embedded.group(1))[1:]], - cwd=REPO_ROOT, - capture_output=True, - text=True, - check=True, - ).stdout.strip() - assert ( - printed == version - ), f"docgen would install executorch=={printed}, pin says {version}" + # Compared as text, not executed. Running it meant whatever that line said got executed on + # every pull request: rewriting the one-liner to write a file left the test green and the file + # written. It has to read __executorch_version__ out of dev_dep_versions.yml and print nothing + # else, which is the property that makes the shell substitution equal the pin. + command = embedded.group(1) + reads_the_pin = re.fullmatch( + r"""python3 -c 'import yaml;print\(yaml\.safe_load\(open\("dev_dep_versions\.yml"\)\)""" + r"""\["__executorch_version__"\]\)'""", + command, + ) + assert reads_the_pin, ( + "the docgen pin no longer reads __executorch_version__ out of dev_dep_versions.yml, so " + f"what it installs is no longer the pin: {command}" + ) def test_the_runner_follows_the_row_s_cuda_version(monkeypatch) -> None: @@ -725,9 +744,20 @@ def test_the_pin_check_runs_in_ci(): "success()", "success() || failure()", }, f"the pin check in {name} runs under {condition!r}, which may never be true" + # Tokenised, not substring-matched, and every way of neutralising the run counts. "--co" is + # pytest's own documented short form of "--collect-only" and slipped past a check for the long + # spelling, and "|| true" or continue-on-error discard the exit status entirely. + tokens = shlex.split(step["run"].replace("\\\n", " ")) + for flag in ("--collect-only", "--co", "--help", "-h"): + assert ( + flag not in tokens + ), f"the pin check in {name} passes {flag}, so no assertion executes" assert ( - "--collect-only" not in step["run"] - ), f"the pin check in {name} only collects tests, so no assertion executes" + "||" not in tokens + ), f"the pin check in {name} discards its exit status, so a failure cannot fail the job" + assert not step.get( + "continue-on-error" + ), f"the pin check in {name} is continue-on-error, so a failure cannot fail the job" # pytest and pyyaml must be installed by an earlier step of the SAME job: neither # requirements.txt nor dependency-groups.lint carries them, and without them the step exits 1 From 97b636b5c3d6ae1cc67e31d8a7579cefaf2d8147 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 24 Aug 2026 13:37:50 -0700 Subject: [PATCH 13/15] Guard the nightly channel at the local-path install sites, and check it resolves The nightly-index guard matched only the named-distribution spelling, so the four sites that write "pip install .[executorch]" were unguarded: docgen.yml and the three export examples. The nightly index could be deleted from all four with the test green. Each of the four is now caught individually. Its second half was a bare substring test for the host, which proves a string sits nearby rather than that the instruction resolves. Rewriting every channel in the tree, 18 files, to a nonexistent cu999 left it green. The CUDA suffix is now checked against the set the project publishes for. Deliberately not compared against __cuda_version__: five sites legitimately say cu130 while the pin says 13.2, and I confirmed against the live index that cu130 and cu132 both carry 38 ExecuTorch wheels while cu999 carries none. --- .../dynamo/executorch/test_executorch_pin.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index 1c70510d43..4e2b0c93a5 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -204,6 +204,8 @@ def _expected(path: str, number: int, version: str) -> str: # The bazel repositories annotate their pinned commit with the wheel it corresponds to, in a # comment, because bazel fetches by commit and has no requirement string to carry. Those are the # only comment sites that count as pins, and the commit beside them is checked separately. +_PUBLISHED_NIGHTLY_CHANNELS = frozenset({"cu124", "cu126", "cu128", "cu130", "cu132"}) + _ANNOTATED_COMMIT_SITES = frozenset( { "MODULE.bazel", @@ -668,8 +670,13 @@ def test_every_printed_install_instruction_names_the_nightly_channel(): # Two shapes need the channel: an instruction naming the [executorch] extra, and the CI # install of a locally built torch-tensorrt wheel, whose ExecuTorch dependency resolves from # the same index. The second is the site that regressed most often and carries no extra. + # The local-path spelling counts too. Matching only the named-distribution form left the four + # sites that write "pip install .[executorch]" unguarded: the nightly index could be deleted + # from all four with this test green. extra = re.compile( - r"""torch[-_]tensorrt\[[^]]*executorch[^]]*\]|torch_tensorrt\*\.whl""" + r"""torch[-_]tensorrt\[[^]]*executorch[^]]*\]""" + r"""|(? Date: Mon, 24 Aug 2026 17:06:21 -0700 Subject: [PATCH 14/15] Harden the ExecuTorch pin guards and the printed install commands The printed install commands resolved no ExecuTorch. "torch-tensorrt[executorch]" with no version pin resolves the stable PyPI wheel, which carries no executorch extra, so the command exited 0 and installed nothing the feature needs. Add --pre to the six commands that name the extra and assert its presence in the guard that already reads them. Close four ways to neutralise the pin check while its guard stayed green: a ";" or "&" terminator after pytest, continue-on-error or a falsy if: on the owning job, and reducing the workflow trigger so it never runs on pull requests. The trigger check also handles PyYAML reading the unquoted "on" key as the boolean True. Close both ways to strip the pairing check while its guard stayed green: assert the workflow actually calls trt_tier_executorch, and validate suite lane names against the known set so a typo raises at import instead of silently dropping the suite from every matrix. Also: anchor the docgen pin check to a live line so a commented-out install no longer satisfies it; fix the lockfile range check crashing on a legal "==1.4.*" clause; correct the range comment to describe what the range admits; and note in the install advice that the feature is published for Linux only. --- .../runtime_performance/saving_models.rst | 2 +- .../executorch_reference_runner/README.md | 2 +- .../runtime.py | 7 +- py/torch_tensorrt/_compile.py | 6 +- py/torch_tensorrt/executorch/__init__.py | 7 +- setup.py | 5 +- tests/ci/suites.py | 22 +++- .../dynamo/executorch/test_executorch_pin.py | 115 +++++++++++++++--- 8 files changed, 136 insertions(+), 30 deletions(-) diff --git a/docsrc/user_guide/runtime_performance/saving_models.rst b/docsrc/user_guide/runtime_performance/saving_models.rst index 418e65d19b..9e0c01e340 100644 --- a/docsrc/user_guide/runtime_performance/saving_models.rst +++ b/docsrc/user_guide/runtime_performance/saving_models.rst @@ -228,7 +228,7 @@ c) ExecuTorch (.pte) The ``executorch`` output format lowers the compiled module to an ExecuTorch ``.pte`` program, delegating the TensorRT engines to the Torch-TensorRT ExecuTorch backend. It requires the ``executorch`` package, from the PyTorch nightly index -(``pip install "torch_tensorrt[executorch]" --extra-index-url +(``pip install --pre "torch_tensorrt[executorch]" --extra-index-url https://download.pytorch.org/whl/nightly/cu130``), and is Linux-only. There are two ways to produce a ``.pte``, and they suit different needs: diff --git a/examples/executorch_reference_runner/README.md b/examples/executorch_reference_runner/README.md index 1f635734ff..d2e634f5a1 100644 --- a/examples/executorch_reference_runner/README.md +++ b/examples/executorch_reference_runner/README.md @@ -98,7 +98,7 @@ build-executorch-reference-runner/lib/libexecutorch_trt_backend.a Install the complete prebuilt Python runtime and delegate: ```bash -pip install "torch-tensorrt[executorch]" \ +pip install --pre "torch-tensorrt[executorch]" \ --extra-index-url https://download.pytorch.org/whl/nightly/cu130 ``` diff --git a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py index 353e872952..3e12e8d25d 100644 --- a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py +++ b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py @@ -14,9 +14,10 @@ def _get_runtime() -> _Runtime: from torch_tensorrt_executorch_runtime import get_runtime except ImportError as error: raise ImportError( - "ExecuTorch Python inference requires the prebuilt delegate. " - 'Install it with: pip install "torch-tensorrt[executorch]" ' - "--extra-index-url https://download.pytorch.org/whl/nightly/cu130" + "ExecuTorch Python inference requires the prebuilt delegate, which is " + "published for Linux only. Install it with: pip install --pre " + '"torch-tensorrt[executorch]" --extra-index-url ' + "https://download.pytorch.org/whl/nightly/cu130" ) from error return get_runtime() diff --git a/py/torch_tensorrt/_compile.py b/py/torch_tensorrt/_compile.py index 32cdab3300..1d0df1c556 100644 --- a/py/torch_tensorrt/_compile.py +++ b/py/torch_tensorrt/_compile.py @@ -857,7 +857,8 @@ def save( if output_format == "executorch" and not _has_executorch_exir(): raise ImportError( "Saving in ExecuTorch format requires the executorch package " - "with executorch.exir. Install with: pip install " + "with executorch.exir, published for Linux only. Install with: " + "pip install --pre " '"torch_tensorrt[executorch]" --extra-index-url ' "https://download.pytorch.org/whl/nightly/cu130 to use output_format='executorch'." ) @@ -1406,7 +1407,8 @@ def _save_as_executorch(exp_program: Any, file_path: str, **kwargs: Any) -> None from torch_tensorrt.executorch import export except ImportError: raise ImportError( - "ExecuTorch is not installed. Install with: pip install " + "ExecuTorch is not installed, and is published for Linux only. Install " + "with: pip install --pre " '"torch_tensorrt[executorch]" --extra-index-url ' "https://download.pytorch.org/whl/nightly/cu130 to use output_format='executorch'." ) diff --git a/py/torch_tensorrt/executorch/__init__.py b/py/torch_tensorrt/executorch/__init__.py index 0669898967..0ef31ccf4b 100644 --- a/py/torch_tensorrt/executorch/__init__.py +++ b/py/torch_tensorrt/executorch/__init__.py @@ -24,9 +24,10 @@ def _has_executorch_exir() -> bool: def __getattr__(name: str) -> NoReturn: raise ImportError( f"Cannot access torch_tensorrt.executorch.{name}: " - "ExecuTorch with executorch.exir is required. " - 'Install with: pip install "torch_tensorrt[executorch]" ' - "--extra-index-url https://download.pytorch.org/whl/nightly/cu130" + "ExecuTorch with executorch.exir is required, and is published for " + "Linux only. Install with: pip install --pre " + '"torch_tensorrt[executorch]" --extra-index-url ' + "https://download.pytorch.org/whl/nightly/cu130" ) __all__ = [ diff --git a/setup.py b/setup.py index 1073ef4c28..1a5372aa4c 100644 --- a/setup.py +++ b/setup.py @@ -206,8 +206,9 @@ def load_dep_info(): # The delegate is compiled from the ExecuTorch source revision pinned in MODULE.bazel, so the # installed wheel should agree with it. The upper bound is the load-bearing half: ExecuTorch's C++ # runtime API is not stable across minor releases, and an unbounded floor would resolve a future -# minor against a backend built for this one. Patch releases stay allowed because they come off the -# same release branch; the exact pin belongs in the runtime package, which does derive it. +# minor against a backend built for this one. Everything below that ceiling resolves: later 1.5 +# nightlies, a 1.5 release candidate, and 1.5 patch releases alike, since the exact pin belongs in +# the runtime package, which does derive it. # The floor currently names a dev build, because the runtime split the delegate needs does not # exist in any ExecuTorch release yet: 1.4.1's executorch/lib carries no standalone linkable # runtime, and no CUDA wheel at all. diff --git a/tests/ci/suites.py b/tests/ci/suites.py index 5a78bd2756..7b2fd142c1 100644 --- a/tests/ci/suites.py +++ b/tests/ci/suites.py @@ -28,7 +28,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Literal +from typing import Any, Literal, get_args Tier = Literal["l0", "l1", "l2"] # python-only validates the PYTHON_ONLY=1 wheel (no C++ runtime) against the @@ -108,6 +108,26 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: base.update(self.overrides.get(variant, {})) return base + def __post_init__(self) -> None: + # Validate the enum-like fields at construction. They are typed as Literal, but nothing + # enforces that at runtime, so a typo like lanes=("nightl",) used to define a suite that + # every lane filter silently skipped, dropping it from CI with no error anywhere. Checking + # here turns that typo into an immediate, located failure when this module is imported. + for field_name, allowed in ( + ("tier", get_args(Tier)), + ("lanes", get_args(Lane)), + ("variants", get_args(Variant)), + ("platforms", get_args(Platform)), + ): + value = getattr(self, field_name) + values = (value,) if isinstance(value, str) else value + unknown = [v for v in values if v not in allowed] + if unknown: + raise ValueError( + f"suite {self.name!r} has unknown {field_name} {unknown}; " + f"expected a subset of {list(allowed)}" + ) + # ── L0 — smoke / fast lane ──────────────────────────────────────────────────── _L0: list[Suite] = [ diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index 4e2b0c93a5..e18a949ec9 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -387,11 +387,17 @@ def test_derived_requirements_match_the_pin(monkeypatch) -> None: # two helpers above can see it: `$(` is not a digit. Run the command it embeds and compare # what it prints, which fails if the line is deleted or the key is renamed. workflow = (REPO_ROOT / ".github/workflows/docgen.yml").read_text(encoding="utf-8") - embedded = re.search(r'"executorch==\$\((python3 -c \'[^\']+\')\)"', workflow) + # Anchor to a live line: leading whitespace only, no "#". A commented-out install still + # carries the pattern, so a plain search stayed green when the whole step was disabled. + embedded = re.search( + r'^[ \t]*"executorch==\$\((python3 -c \'[^\']+\')\)"', + workflow, + re.MULTILINE, + ) assert embedded, ( - ".github/workflows/docgen.yml no longer pins ExecuTorch alongside the extra. It installs " - "with --pre from the nightly channel, so without the pin it resolves through the range " - "and takes whichever dev build is newest that day." + ".github/workflows/docgen.yml no longer pins ExecuTorch alongside the extra on a live " + "line. It installs with --pre from the nightly channel, so without the pin it resolves " + "through the range and takes whichever dev build is newest that day." ) # Compared as text, not executed. Running it meant whatever that line said got executed on # every pull request: rewriting the one-liner to write a file left the test green and the file @@ -634,7 +640,7 @@ def test_the_lockfile_records_the_same_executorch_range_as_setup_py(): for entry in sorted(recorded) if any( clause.operator in {">=", ">", "==", "~=", "==="} - and Version(clause.version.rstrip("*") or "0") > pinned + and Version(clause.version.rstrip(".*") or "0") > pinned for clause in SpecifierSet(entry) ) ] @@ -719,6 +725,19 @@ def test_every_printed_install_instruction_names_the_nightly_channel(): f"{name}:{line} installs from nightly/{suffix}, which the project does not " f"publish for; expected one of {sorted(_PUBLISHED_NIGHTLY_CHANNELS)}" ) + # The named-distribution form needs --pre. "torch-tensorrt[executorch]" with no + # version pin resolves to the stable PyPI wheel, which carries no executorch extra at + # all, so the command exits 0 with a warning and installs nothing the feature needs. + # The ".[executorch]" and built-wheel forms already pin executorch to a dev version, + # which enables prerelease selection on their own, so they do not need it. + named_distribution = re.fullmatch( + r"torch[-_]tensorrt\[[^]]*executorch[^]]*\]", match.group(0) + ) + if named_distribution and not re.search(r"(?:^|\s)--pre(?:\s|$)", block): + missing.append( + f"{name}:{line} installs {match.group(0)} without --pre, so pip resolves the " + "stable release with no executorch extra rather than the nightly prerelease" + ) assert not missing, ( "these ExecuTorch install instructions do not name the nightly channel, so they " @@ -743,6 +762,16 @@ def test_the_pin_check_runs_in_ci(): workflow = yaml.safe_load( (REPO_ROOT / ".github/workflows/linter.yml").read_text(encoding="utf-8") ) + # The workflow must actually run on pull requests. PyYAML reads the unquoted "on" key as the + # boolean True (the YAML 1.1 "Norway problem"), so accept either spelling, then require a + # pull_request trigger. Reducing "on:" to workflow_dispatch left every string in place while + # the workflow never fired on a pull request. + triggers = workflow.get("on", workflow.get(True)) + trigger_names = set(triggers) if isinstance(triggers, (dict, list)) else {triggers} + assert "pull_request" in trigger_names, ( + f"linter.yml triggers on {sorted(map(str, trigger_names))}, not pull_request, so the pin " + "check never runs when a pull request changes the pin" + ) # Match a live pytest invocation, not the filename anywhere in the script. Neutralising the # command and leaving it in a shell comment satisfied a plain substring test. invocation = re.compile( @@ -758,27 +787,38 @@ def test_the_pin_check_runs_in_ci(): assert owning, "no CI job invokes this file, so nothing here runs on a pull request" name, job, step = owning[0] - # A falsy condition disables the step while leaving every string in place. - condition = str(step.get("if", "always()")) - assert condition in { - "always()", - "success()", - "success() || failure()", - }, f"the pin check in {name} runs under {condition!r}, which may never be true" + # A falsy condition disables the step or the whole job while leaving every string in place, so + # check both. GitHub treats a bare "false", "${{ false }}" and any always-false expression the + # same way, so restrict each to the small set of conditions that can actually be true. + live_conditions = {"always()", "success()", "success() || failure()"} + step_condition = str(step.get("if", "always()")) + assert ( + step_condition in live_conditions + ), f"the pin check step in {name} runs under {step_condition!r}, which may never be true" + job_condition = str(job.get("if", "always()")) + assert ( + job_condition in live_conditions + ), f"job {name} runs under {job_condition!r}, so the pin check may never dispatch" # Tokenised, not substring-matched, and every way of neutralising the run counts. "--co" is # pytest's own documented short form of "--collect-only" and slipped past a check for the long - # spelling, and "|| true" or continue-on-error discard the exit status entirely. + # spelling. "|| true", "; true" and continue-on-error each discard the exit status, the last + # two at the step and at the job. tokens = shlex.split(step["run"].replace("\\\n", " ")) for flag in ("--collect-only", "--co", "--help", "-h"): assert ( flag not in tokens ), f"the pin check in {name} passes {flag}, so no assertion executes" - assert ( - "||" not in tokens - ), f"the pin check in {name} discards its exit status, so a failure cannot fail the job" + for terminator in ("||", ";", "&"): + assert terminator not in tokens, ( + f"the pin check in {name} follows pytest with {terminator!r}, so its exit status does " + "not fail the step" + ) assert not step.get( "continue-on-error" - ), f"the pin check in {name} is continue-on-error, so a failure cannot fail the job" + ), f"the pin check step in {name} is continue-on-error, so a failure cannot fail the job" + assert not job.get( + "continue-on-error" + ), f"job {name} is continue-on-error, so a failed pin check cannot fail the workflow" # pytest and pyyaml must be installed by an earlier step of the SAME job: neither # requirements.txt nor dependency-groups.lint carries them, and without them the step exits 1 @@ -860,3 +900,44 @@ def test_the_pairing_check_survives_the_gpu_lane_deselection() -> None: f"{path} selects {len(others)} other tests from this module, which the lane " f"deselects deliberately: {others[:2]}" ) + + # Proving the -k expression selects the pairing test says nothing about whether either route + # is actually wired to run it. Replacing the workflow's `trt_tier_executorch` call with `echo + # skipped`, or pointing the executorch suite at a lane name no runner requests, both leave the + # checks above green while the test runs nowhere. So assert each route reaches the tier. + import yaml + + workflow = yaml.safe_load( + (REPO_ROOT / ".github/workflows/executorch-test-linux.yml").read_text( + encoding="utf-8" + ) + ) + scripts = [ + step.get("with", {}).get("script", "") + for job in workflow["jobs"].values() + for step in job.get("steps", []) + ] + [job.get("with", {}).get("script", "") for job in workflow["jobs"].values()] + invokes_tier = any( + re.search(r"^\s*trt_tier_executorch\b", script, re.MULTILINE) + for script in scripts + ) + assert invokes_tier, ( + "executorch-test-linux.yml no longer calls trt_tier_executorch, so the pairing check " + "never runs on the GPU lane even though its -k expression would select it" + ) + + # The manifest route: the executorch suite must exist and target a lane a runner requests. + # A typo in its lane tuple silently drops it from every matrix, which the suite-name check + # above cannot see. + import importlib + + suites = importlib.import_module("tests.ci.suites") + executorch_suite = next((s for s in suites.SUITES if s.name == "executorch"), None) + assert executorch_suite is not None, ( + "tests/ci/suites.py no longer defines an 'executorch' suite, so the manifest route to the " + "pairing check is gone" + ) + assert "nightly" in executorch_suite.lanes, ( + f"the executorch suite runs on lanes {executorch_suite.lanes!r}, none of which is the " + "nightly lane the GPU tier requests, so the pairing check runs nowhere" + ) From 93948e1d070507d7482b31f1e2e44a8f2ce2d5c9 Mon Sep 17 00:00:00 2001 From: shoumikhin Date: Mon, 24 Aug 2026 19:13:39 -0700 Subject: [PATCH 15/15] Track the ExecuTorch pin and move it to new nightlies automatically The delegate is built against one ExecuTorch: __executorch_version__ selects the wheel it links against and __executorch_commit__ selects the tree it compiles from. Those two values repeat across the build workflows, the bazel modules, the docker and toolchain copies, and the docs, so they can drift apart or fall behind upstream with nothing to notice. Add a script and a daily workflow that move both pins to the newest ExecuTorch wheel on the nightly index. The source commit is read from the chosen wheel's own version.py, so the two pins always name one ExecuTorch rather than two that happen to be close. The update lands as a pull request, so the pin consistency checks and the delegate build and test lane decide whether the new wheel is usable before it reaches main. A day with no new nightly rewrites nothing and opens nothing. On a release branch the schedule is a no-op and the pin moves only by a manual run pointed at the stable line, so a cut release does not drift. Back the mechanism with consistency checks that run under the linter. Every requirement and comment that names ExecuTorch is asserted to match the pinned version, including the variable-index install once the variable's assignment is resolved and extensionless install files like justfile. The source commit is checked against the wheel's own provenance wherever that wheel is installed, and commits left in comments are not mistaken for pins. The wheel-content and CI-invocation checks measure effect, running the workflow's own step against a passing and a failing stub and requiring the exit status to follow, rather than enumerating bypass spellings. Install the built wheel in the runtime README rather than an unpublished package. --- .github/scripts/install-torch-tensorrt.sh | 3 + .github/scripts/update_executorch_pin.py | 234 +++++++ .github/workflows/executorch-build-linux.yml | 4 +- .github/workflows/executorch-pin-update.yml | 105 +++ .github/workflows/executorch-test-linux.yml | 2 +- .github/workflows/linter.yml | 18 +- MODULE.bazel | 4 +- dev_dep_versions.yml | 4 +- docker/MODULE.bazel.docker | 4 +- docker/MODULE.bazel.ngc | 4 +- .../runtime_performance/saving_models.rst | 4 +- .../executorch_reference_runner/README.md | 21 +- justfile | 2 +- .../README.md | 14 +- .../pyproject.toml | 2 +- .../runtime.py | 15 +- py/torch_tensorrt/_compile.py | 22 +- py/torch_tensorrt/_utils.py | 31 + py/torch_tensorrt/executorch/__init__.py | 6 +- tests/ci/suites.py | 7 +- .../dynamo/executorch/test_executorch_pin.py | 651 ++++++++++++++++-- .../executorch/test_update_executorch_pin.py | 255 +++++++ toolchains/ci_workspaces/MODULE.bazel.tmpl | 4 +- 23 files changed, 1293 insertions(+), 123 deletions(-) create mode 100644 .github/scripts/update_executorch_pin.py create mode 100644 .github/workflows/executorch-pin-update.yml create mode 100644 tests/py/dynamo/executorch/test_update_executorch_pin.py diff --git a/.github/scripts/install-torch-tensorrt.sh b/.github/scripts/install-torch-tensorrt.sh index be41ca5923..7141b31f3b 100755 --- a/.github/scripts/install-torch-tensorrt.sh +++ b/.github/scripts/install-torch-tensorrt.sh @@ -52,6 +52,9 @@ fi # Install Torch-TensorRT if [[ ${PLATFORM} == win32 ]]; then + # pin-check: no-nightly -- this glob also matches the Linux-only ExecuTorch runtime wheel, but + # ExecuTorch publishes no win32 nightly and the [executorch] extra is Linux-only, so this + # platform's plain torch-tensorrt install needs no nightly index. python -m pip install ${RUNNER_ARTIFACT_DIR}/torch_tensorrt*.whl else # The nightly channel is needed because this glob also matches the ExecuTorch runtime wheel, diff --git a/.github/scripts/update_executorch_pin.py b/.github/scripts/update_executorch_pin.py new file mode 100644 index 0000000000..5f8dfbc5f8 --- /dev/null +++ b/.github/scripts/update_executorch_pin.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Move the ExecuTorch pin to the newest wheel published on an index. + +The pin is two coupled facts, spread across the tree but sourced from +``dev_dep_versions.yml``: ``__executorch_version__`` selects the wheel the delegate is +built to sit beside, and ``__executorch_commit__`` selects the tree it compiles from. +They must name one ExecuTorch, so this script never guesses the commit: it reads it from +the chosen wheel's own ``executorch/version.py``, which is the same provenance +``tests/py/dynamo/executorch/test_executorch_pin.py`` checks the pins against. + +The daily workflow runs this, then opens a pull request when the pin moved. The existing +pin guards and the executorch end-to-end lane run on that pull request, so "the newest +wheel that actually works" is decided by the same gate a human bump goes through, not +re-implemented here. A nightly that did not publish leaves the newest version unchanged, +so the run rewrites nothing and opens nothing. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path + +from packaging.version import InvalidVersion, Version + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_VERSIONS_FILE = _REPO_ROOT / "dev_dep_versions.yml" + +# The pin literal is rewritten everywhere it appears except this file, which carries the +# version only as a docstring example, not a pin. Every real site is checked by the guard, +# so an accidental new site fails that test rather than being silently missed here. +_SELF = "tests/py/dynamo/executorch/test_executorch_pin.py" + +# A wheel version on the PyTorch index carries a local label naming its CUDA build, for +# example ``1.5.0.devYYYYMMDD+cu130``. The pin omits it so one pin serves every CUDA row. +_LOCAL_LABEL = re.compile(r"\+.*$") + + +def _run(cmd: list[str]) -> str: + return subprocess.run(cmd, check=True, capture_output=True, text=True).stdout + + +def read_pin(field: str) -> str: + """Return a pinned value from ``dev_dep_versions.yml``.""" + text = _VERSIONS_FILE.read_text(encoding="utf-8") + match = re.search(rf'^{field}:\s*"?([^"\s]+)"?\s*$', text, re.MULTILINE) + if match is None: + raise SystemExit(f"{field} is not set in {_VERSIONS_FILE.name}") + return match.group(1) + + +def available_versions(index_args: list[str]) -> list[str]: + """Every ExecuTorch version the index offers. + + ``pip index versions`` prints one ``Available versions:`` line. Parsing that is stable + and needs no network code here; the workflow passes the index the same way every other + install in the tree does. + """ + out = _run( + [sys.executable, "-m", "pip", "index", "versions", "executorch", *index_args] + ) + match = re.search(r"^\s*Available versions:\s*(.+)$", out, re.MULTILINE) + if match is None: + raise SystemExit("pip index versions printed no Available versions line") + return [v.strip() for v in match.group(1).split(",") if v.strip()] + + +def pick_target(versions: list[str], track: str) -> str: + """The newest version on the wanted track. + + ``nightly`` takes the newest dated dev build; ``stable`` takes the newest final + release, ignoring dev builds and release candidates. Ordering is PEP 440, not string + order, so a newer dev date on the same line sorts above an older one correctly. + """ + parsed: list[tuple[Version, str]] = [] + for raw in versions: + try: + version = Version(raw) + except InvalidVersion: + continue + # A nightly is a dated dev build. is_prerelease is also true for release + # candidates, and an rc sorts above every dev of the same line under PEP 440, so + # filtering on it would let the first rc on the index silently become the nightly + # pin. Match the dev segment itself instead. + if track == "nightly" and version.dev is None: + continue + if track == "stable" and (version.is_prerelease or version.is_devrelease): + continue + parsed.append((version, raw)) + if not parsed: + raise SystemExit(f"no executorch version on the index matches track {track!r}") + newest = max(parsed, key=lambda pair: pair[0])[1] + return _LOCAL_LABEL.sub("", newest) + + +def wheel_git_version(version: str, index_args: list[str]) -> str: + """The source commit the chosen wheel records for itself. + + Every published wheel writes ``git_version`` into ``executorch/version.py``. Reading it + from the wheel is what keeps the two pins naming one ExecuTorch. A wheel built without + git provenance records ``None`` and must not become a pin, so that is an error, not a + guess. + """ + with tempfile.TemporaryDirectory() as tmp: + _run( + [ + sys.executable, + "-m", + "pip", + "download", + "--no-deps", + "--dest", + tmp, + f"executorch=={version}", + *index_args, + ] + ) + wheels = list(Path(tmp).glob("executorch-*.whl")) + if not wheels: + raise SystemExit( + f"pip download produced no wheel for executorch=={version}" + ) + with zipfile.ZipFile(wheels[0]) as archive: + source = archive.read("executorch/version.py").decode("utf-8") + match = re.search(r"""git_version[^=]*=\s*['"]([0-9a-f]{40})['"]""", source) + if match is None: + raise SystemExit( + f"executorch=={version} records no source commit, so the pin would name a " + "wheel whose provenance cannot be checked" + ) + return match.group(1) + + +def _upper_bound(version: str) -> str: + """The exclusive upper bound a range site pairs with the pin, next minor of its line. + + ``tests/py/dynamo/executorch/test_executorch_pin.py`` derives the same bound from the + same two fields, so a range this writes and the range the guard expects agree by the + same rule rather than by coincidence. + """ + major, minor = version.split(".")[:2] + return f"{major}.{int(minor) + 1}" + + +def _iter_tracked_files() -> list[Path]: + names = _run(["git", "-C", str(_REPO_ROOT), "ls-files"]).splitlines() + return [_REPO_ROOT / name for name in names if name != _SELF] + + +def write_pins(new_version: str, new_commit: str) -> bool: + """Rewrite every pin occurrence to the new version and commit. + + The old version and commit are unique, high-entropy tokens, so a literal replacement + cannot touch anything that is not already a pin. The range form is re-rendered after, + so its upper bound tracks the new line. Returns whether anything changed. + """ + old_version = read_pin("__executorch_version__") + old_commit = read_pin("__executorch_commit__") + if (new_version, new_commit) == (old_version, old_commit): + return False + + # Anchored on >= and ,< rather than on a package name, so this rewrites the range + # wherever it appears: the executorch>=X,={old_version},<{_upper_bound(old_version)}" + new_range = f">={new_version},<{_upper_bound(new_version)}" + + changed = False + for path in _iter_tracked_files(): + try: + text = path.read_text(encoding="utf-8") + except (UnicodeDecodeError, FileNotFoundError): + continue + if old_version not in text and old_commit not in text: + continue + # The range carries the old version too, so rewrite it first, then the remaining + # bare version occurrences, so the range's own version is not rewritten twice. + updated = text.replace(old_range, new_range) + updated = updated.replace(old_version, new_version) + updated = updated.replace(old_commit, new_commit) + if updated != text: + path.write_text(updated, encoding="utf-8") + changed = True + + if read_pin("__executorch_version__") != new_version: + raise SystemExit( + "the version pin did not take; dev_dep_versions.yml is unchanged" + ) + return changed + + +def _index_args(track: str, channel: str) -> list[str]: + if track == "nightly": + return [ + "--pre", + "--index-url", + f"https://download.pytorch.org/whl/nightly/{channel}", + ] + return [] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--track", choices=("nightly", "stable"), default="nightly") + parser.add_argument( + "--channel", + default="cu130", + help="nightly CUDA channel to read versions and provenance from", + ) + args = parser.parse_args(argv) + + index_args = _index_args(args.track, args.channel) + target = pick_target(available_versions(index_args), args.track) + current = read_pin("__executorch_version__") + if target == current: + print(f"executorch pin is already at the newest {args.track} version {current}") + return 0 + + commit = wheel_git_version(target, index_args) + if write_pins(target, commit): + print(f"moved executorch pin {current} -> {target} (commit {commit})") + else: + print("nothing to write") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/executorch-build-linux.yml b/.github/workflows/executorch-build-linux.yml index 33d190d051..8fbdba8804 100644 --- a/.github/workflows/executorch-build-linux.yml +++ b/.github/workflows/executorch-build-linux.yml @@ -85,7 +85,7 @@ jobs: # CU_VERSION selects the row's own channel, which is what keeps the runtime the # delegate links to the same CUDA build as the rest of the job. EXECUTORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/${CU_VERSION}" - python -m pip install pyyaml --extra-index-url "${EXECUTORCH_INDEX_URL}" "executorch==1.5.0.dev20260822" + python -m pip install pyyaml --extra-index-url "${EXECUTORCH_INDEX_URL}" "executorch==1.5.0.dev20260825" export TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION="$(python -c 'import importlib.metadata; print(importlib.metadata.version("torch-tensorrt"))')" # The downloaded wheel has to carry the C++ runtime. A wheel built with @@ -133,7 +133,7 @@ jobs: export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" # pin-check: range-ok -- this is to verify the end user's workflow, which resolves a # range the way a user would rather than the exact artifact the delegate links. - python -m pip install pyyaml --extra-index-url "${EXECUTORCH_INDEX_URL}" "executorch>=1.5.0.dev20260822,<1.6" + python -m pip install pyyaml --extra-index-url "${EXECUTORCH_INDEX_URL}" "executorch>=1.5.0.dev20260825,<1.6" python examples/torchtrt_executorch_example/export_static_shape.py \ --model_path="${RUNNER_TEMP}/torchtrt-reference-runner.pte" .github/scripts/verify-executorch-reference-runner.sh \ diff --git a/.github/workflows/executorch-pin-update.yml b/.github/workflows/executorch-pin-update.yml new file mode 100644 index 0000000000..e4a87a5226 --- /dev/null +++ b/.github/workflows/executorch-pin-update.yml @@ -0,0 +1,105 @@ +name: Update ExecuTorch pin + +# Keep the ExecuTorch pin close to upstream without a human running the bump by hand. On +# main this tracks the nightly line every day; the newest wheel the index actually carries +# is by definition the newest one that built, so a failed nightly simply leaves the pin +# where it was. The bump lands as a pull request, not a direct push, so the existing pin +# guards and the executorch end-to-end lane decide whether the new wheel is usable before +# it reaches main. +# +# On a release branch the schedule is a no-op: a release pins a stable ExecuTorch and does +# not drift. Re-pinning there is a manual dispatch with track=stable, which is the only way +# the pin moves once a branch is cut. + +on: + schedule: + # A few hours after the nightly index publishes, so the freshest wheel is available. + - cron: "0 13 * * *" + workflow_dispatch: + inputs: + track: + description: "Which ExecuTorch line to pin to" + type: choice + default: nightly + options: + - nightly + - stable + +permissions: + contents: write + pull-requests: write + +jobs: + update-pin: + runs-on: ubuntu-latest + if: ${{ ! contains(github.actor, 'pytorchbot') }} + environment: pytorchbot-env + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + token: ${{ secrets.GH_PYTORCHBOT_TOKEN }} + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install packaging + run: python -m pip install packaging + + - name: Choose the track + id: track + run: | + set -euo pipefail + # A manual run pins whatever it asked for. The daily schedule pins the nightly + # line on main and does nothing on a release branch, so a release never drifts + # onto a nightly. + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "track=${{ inputs.track }}" >> "$GITHUB_OUTPUT" + elif [ "${{ github.ref }}" = "refs/heads/main" ]; then + echo "track=nightly" >> "$GITHUB_OUTPUT" + else + echo "no scheduled pin update on ${{ github.ref }}; release pins do not drift" + echo "track=" >> "$GITHUB_OUTPUT" + fi + + - name: Update the pin + id: update + if: steps.track.outputs.track != '' + run: | + set -euo pipefail + python .github/scripts/update_executorch_pin.py --track "${{ steps.track.outputs.track }}" + if git diff --quiet; then + echo "the pin is already at the newest ${{ steps.track.outputs.track }} wheel" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "version=$(sed -n 's/^__executorch_version__: "\(.*\)"$/\1/p' dev_dep_versions.yml)" >> "$GITHUB_OUTPUT" + fi + + - name: Open a pull request + if: steps.update.outputs.changed == 'true' + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ secrets.GH_PYTORCHBOT_TOKEN }} + branch: executorch-pin-update/${{ steps.track.outputs.track }} + delete-branch: true + commit-message: "Update ExecuTorch pin to ${{ steps.update.outputs.version }}" + title: "Update ExecuTorch pin to ${{ steps.update.outputs.version }}" + # The delegate is built from the tree the pinned wheel was built from, so both + # pins move together to the version and the commit that wheel records for itself. + # The pin guards and the executorch end-to-end lane gate this pull request. + body: | + Move the ExecuTorch pin to `${{ steps.update.outputs.version }}`, the newest + ${{ steps.track.outputs.track }} wheel on the index. The source commit is read + from that wheel, so the version pin and the source commit name one ExecuTorch. + + Opened automatically. The pin consistency checks and the ExecuTorch delegate + build and test lane run here and decide whether this wheel is usable. + committer: Torch-TensorRT Github Bot + author: Torch-TensorRT Github Bot + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true diff --git a/.github/workflows/executorch-test-linux.yml b/.github/workflows/executorch-test-linux.yml index 1966ba7308..eb6b6de824 100644 --- a/.github/workflows/executorch-test-linux.yml +++ b/.github/workflows/executorch-test-linux.yml @@ -69,7 +69,7 @@ jobs: # --pre would apply to every other requirement in the same command too. python -m pip install pyyaml \ --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" \ - "executorch==1.5.0.dev20260822" + "executorch==1.5.0.dev20260825" # Run the check directly so its exit status is the step's exit status. # Wrapping it in `gdb --batch` reports gdb's own status, which is 0 # whatever the program does, so a SIGSEGV here was passing. diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 26e45662c8..4fc43393c9 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -82,9 +82,11 @@ jobs: run: | uv pip install --system -r $GITHUB_WORKSPACE/.github/scripts/requirements.txt python3 -c "import tomllib, subprocess; deps = tomllib.load(open('pyproject.toml', 'rb'))['dependency-groups']['lint']; subprocess.run(['uv', 'pip', 'install', '--system'] + deps, check=True)" - # The pin check below runs under pytest and shells out to a yaml reader. Neither is in - # requirements.txt or dependency-groups.lint, so the step exited 1 without running. - uv pip install --system pytest pyyaml + # The pin check below runs under pytest and imports yaml to parse the workflows it + # asserts about. Neither pytest nor pyyaml is in requirements.txt or + # dependency-groups.lint, so the step exited 1 without running. + # test_update_executorch_pin imports packaging to order versions the way pip does. + uv pip install --system pytest pyyaml packaging - name: Lint Python run: | cd $GITHUB_WORKSPACE @@ -101,3 +103,13 @@ jobs: cd $GITHUB_WORKSPACE python3 -m pytest tests/py/dynamo/executorch/test_executorch_pin.py \ -q --no-header -p no:cacheprovider --noconftest -o addopts="" + # The pin updater is pure repository tooling: it reads an index, rewrites the pin sites, + # and its tests clone the tree and shell out to pytest. No GPU or ExecuTorch, so it runs + # here beside the pin check rather than on a CUDA runner, and is excluded from the + # executorch e2e suite so it does not run there too. + - name: Test the ExecuTorch pin updater + if: always() + run: | + cd $GITHUB_WORKSPACE + python3 -m pytest tests/py/dynamo/executorch/test_update_executorch_pin.py \ + -q --no-header -p no:cacheprovider --noconftest -o addopts="" diff --git a/MODULE.bazel b/MODULE.bazel index afe76d6cc9..f6e6e2a6be 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -53,8 +53,8 @@ local_torch = use_repo_rule("//toolchains:local_torch.bzl", "local_torch") new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.5.0.dev20260822 - commit = "b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7", + # executorch==1.5.0.dev20260825 + commit = "817929b7fb8d162d80eb9299d6630c35a3106979", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;", diff --git a/dev_dep_versions.yml b/dev_dep_versions.yml index f17b31f1f2..d32783a75c 100644 --- a/dev_dep_versions.yml +++ b/dev_dep_versions.yml @@ -2,5 +2,5 @@ __cuda_version__: "13.2" __tensorrt_version__: "11.2.1" __tensorrt_rtx_version__: "1.6.1" __tensorrt_llm_version__: "0.17.0.post1" -__executorch_version__: "1.5.0.dev20260822" -__executorch_commit__: "b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7" +__executorch_version__: "1.5.0.dev20260825" +__executorch_commit__: "817929b7fb8d162d80eb9299d6630c35a3106979" diff --git a/docker/MODULE.bazel.docker b/docker/MODULE.bazel.docker index b1459e08fa..ce932f8f65 100644 --- a/docker/MODULE.bazel.docker +++ b/docker/MODULE.bazel.docker @@ -67,8 +67,8 @@ new_local_repository( new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.5.0.dev20260822 - commit = "b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7", + # executorch==1.5.0.dev20260825 + commit = "817929b7fb8d162d80eb9299d6630c35a3106979", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;", diff --git a/docker/MODULE.bazel.ngc b/docker/MODULE.bazel.ngc index 6823b9bdc6..b01e9a6654 100644 --- a/docker/MODULE.bazel.ngc +++ b/docker/MODULE.bazel.ngc @@ -76,8 +76,8 @@ new_local_repository( new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.5.0.dev20260822 - commit = "b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7", + # executorch==1.5.0.dev20260825 + commit = "817929b7fb8d162d80eb9299d6630c35a3106979", recursive_init_submodules = True, patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", diff --git a/docsrc/user_guide/runtime_performance/saving_models.rst b/docsrc/user_guide/runtime_performance/saving_models.rst index 9e0c01e340..76aec91564 100644 --- a/docsrc/user_guide/runtime_performance/saving_models.rst +++ b/docsrc/user_guide/runtime_performance/saving_models.rst @@ -229,7 +229,9 @@ The ``executorch`` output format lowers the compiled module to an ExecuTorch ``.pte`` program, delegating the TensorRT engines to the Torch-TensorRT ExecuTorch backend. It requires the ``executorch`` package, from the PyTorch nightly index (``pip install --pre "torch_tensorrt[executorch]" --extra-index-url -https://download.pytorch.org/whl/nightly/cu130``), and is Linux-only. +https://download.pytorch.org/whl/nightly/cu130``), and is Linux-only. Add ``--upgrade`` +if a stable ``torch-tensorrt`` is already installed, or pip keeps it and reports that it +does not provide the ``executorch`` extra. There are two ways to produce a ``.pte``, and they suit different needs: diff --git a/examples/executorch_reference_runner/README.md b/examples/executorch_reference_runner/README.md index d2e634f5a1..ef3eaf8baa 100644 --- a/examples/executorch_reference_runner/README.md +++ b/examples/executorch_reference_runner/README.md @@ -44,7 +44,7 @@ torch_tensorrt/bin/example_executorch_runner ```bash # Get the ExecuTorch source snapshot this package is built against. Keep this in sync # with the executorch commit pinned in MODULE.bazel. -EXECUTORCH_REF="${EXECUTORCH_REF:-b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7}" +EXECUTORCH_REF="${EXECUTORCH_REF:-817929b7fb8d162d80eb9299d6630c35a3106979}" git clone --filter=blob:none --no-checkout \ https://github.com/pytorch/executorch.git executorch pushd executorch @@ -95,7 +95,7 @@ build-executorch-reference-runner/lib/libexecutorch_trt_backend.a ### Python -Install the complete prebuilt Python runtime and delegate: +Install the `executorch` authoring stack, which the `[executorch]` extra pulls in: ```bash pip install --pre "torch-tensorrt[executorch]" \ @@ -104,8 +104,17 @@ pip install --pre "torch-tensorrt[executorch]" \ The index is required, not optional: the extra's ExecuTorch floor names a dev build, and PyPI's `executorch` stops below it, so without the nightly channel pip reports no matching distribution. +If a stable `torch-tensorrt` is already installed, add `--upgrade`, or pip keeps it and reports +that it does not provide the `executorch` extra. -Load and run the model without an ExecuTorch checkout or native build: +The extra installs `executorch` only. The delegate runtime, +`torch-tensorrt-executorch-runtime`, is not yet published to any index: its requirement in the +top-level `setup.py` is commented out for that reason. Build and install it from source following +`py/torch-tensorrt-executorch-runtime/README.md`. That wheel contains an ExecuTorch Python runtime +with `TensorRTBackend` linked into its backend registry, and loading a `.pte` through the delegate +needs it. + +Then load and run the model: ```bash python examples/executorch_reference_runner/load_model.py \ @@ -113,12 +122,6 @@ python examples/executorch_reference_runner/load_model.py \ --num_runs=1 ``` -The extra installs `executorch` only. The -`torch-tensorrt-executorch-runtime` requirement in the top-level `setup.py` is -commented out until that wheel is published to the PyTorch index, so install it -separately for now. That wheel contains an ExecuTorch Python runtime with -`TensorRTBackend` linked into its backend registry. - ### C++ Run the reference runner against a Torch-TensorRT compiled ExecuTorch model: diff --git a/justfile b/justfile index 2b806852dc..cb91342dc1 100644 --- a/justfile +++ b/justfile @@ -94,7 +94,7 @@ install-test-ext: # compiled from the commit this version pairs with. uv pip install pyyaml \ --extra-index-url https://download.pytorch.org/whl/nightly/cu130 \ - "executorch==1.5.0.dev20260822" + "executorch==1.5.0.dev20260825" # ── Linting ─────────────────────────────────────────────────────────────────── diff --git a/py/torch-tensorrt-executorch-runtime/README.md b/py/torch-tensorrt-executorch-runtime/README.md index a181a783e6..d724b9a1eb 100644 --- a/py/torch-tensorrt-executorch-runtime/README.md +++ b/py/torch-tensorrt-executorch-runtime/README.md @@ -40,8 +40,8 @@ of the wheel runtime contract. export TensorRT_ROOT=/path/to/TensorRT python -m pip install pyyaml \ - --extra-index-url https://download.pytorch.org/whl/nightly/cu132 \ - "executorch==1.5.0.dev20260822" + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 \ + "executorch==1.5.0.dev20260825" python -m pip wheel --no-build-isolation --no-deps \ --wheel-dir dist py/torch-tensorrt-executorch-runtime ``` @@ -49,7 +49,7 @@ python -m pip wheel --no-build-isolation --no-deps \ The native build obtains the ExecuTorch source through Bazel; no separate source checkout or `EXECUTORCH_SOURCE_DIR` setting is required. The source commit pinned in `MODULE.bazel` is the revision recorded by the -`executorch==1.5.0.dev20260822` wheel. +`executorch==1.5.0.dev20260825` wheel. The static ExecuTorch and delegate archives are intermediate build inputs; users receive the final native Python module and do not compile anything. @@ -75,8 +75,14 @@ GPU should use the ExecuTorch C++ runner. ## Use +The wheel's dependencies (`executorch`, `torch-tensorrt`, and the CUDA +runtime) resolve from the PyTorch nightly index, so install it with the same +channel the build recipe used. `--pre` lets pip select the pinned ExecuTorch +dev build: + ```bash -python -m pip install torch-tensorrt-executorch-runtime +python -m pip install --pre dist/torch_tensorrt_executorch_runtime-*.whl \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 ``` ```python diff --git a/py/torch-tensorrt-executorch-runtime/pyproject.toml b/py/torch-tensorrt-executorch-runtime/pyproject.toml index bb057bbc4a..5a036cdbc1 100644 --- a/py/torch-tensorrt-executorch-runtime/pyproject.toml +++ b/py/torch-tensorrt-executorch-runtime/pyproject.toml @@ -6,6 +6,6 @@ requires = [ # environment. # Builds must use --no-build-isolation; see README.md. "torch", - "executorch==1.5.0.dev20260822", + "executorch==1.5.0.dev20260825", ] build-backend = "setuptools.build_meta" diff --git a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py index 3e12e8d25d..90cf4f98f6 100644 --- a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py +++ b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py @@ -10,15 +10,12 @@ def _get_runtime() -> _Runtime: - try: - from torch_tensorrt_executorch_runtime import get_runtime - except ImportError as error: - raise ImportError( - "ExecuTorch Python inference requires the prebuilt delegate, which is " - "published for Linux only. Install it with: pip install --pre " - '"torch-tensorrt[executorch]" --extra-index-url ' - "https://download.pytorch.org/whl/nightly/cu130" - ) from error + # get_runtime is defined at module scope in this package's __init__, from stdlib and local + # imports, so importing the name here cannot fail once this submodule is importable. Calling it + # can still fail: it imports the ExecuTorch runtime, which raises ModuleNotFoundError when + # ExecuTorch is absent, and DelegateCompatibilityError when the delegate is not registered. + from torch_tensorrt_executorch_runtime import get_runtime + return get_runtime() diff --git a/py/torch_tensorrt/_compile.py b/py/torch_tensorrt/_compile.py index 1d0df1c556..6a0fb9930d 100644 --- a/py/torch_tensorrt/_compile.py +++ b/py/torch_tensorrt/_compile.py @@ -26,6 +26,7 @@ from torch_tensorrt._enums import dtype from torch_tensorrt._features import ENABLED_FEATURES, needs_cross_compile from torch_tensorrt._Input import Input +from torch_tensorrt._utils import executorch_install_command from torch_tensorrt.dynamo.runtime._CudaGraphsTorchTensorRTModule import ( CudaGraphsTorchTensorRTModule, ) @@ -630,9 +631,10 @@ def load( if format == "executorch": if not _has_executorch_runtime(): raise ImportError( - "Loading an ExecuTorch program requires the prebuilt " - "Torch-TensorRT ExecuTorch delegate. Install it with: " - "pip install torch-tensorrt-executorch-runtime" + "Loading an ExecuTorch program requires the Torch-TensorRT " + "ExecuTorch delegate runtime (torch_tensorrt_executorch_runtime), " + "which is not yet published to any package index. Build and install " + "it from source following py/torch-tensorrt-executorch-runtime/README.md." ) from torch_tensorrt_executorch_runtime.runtime import load as load_executorch @@ -856,11 +858,9 @@ def save( ) if output_format == "executorch" and not _has_executorch_exir(): raise ImportError( - "Saving in ExecuTorch format requires the executorch package " - "with executorch.exir, published for Linux only. Install with: " - "pip install --pre " - '"torch_tensorrt[executorch]" --extra-index-url ' - "https://download.pytorch.org/whl/nightly/cu130 to use output_format='executorch'." + "Saving in ExecuTorch format requires the executorch package with " + "executorch.exir, published for Linux only, to use " + "output_format='executorch'. Install with: " + executorch_install_command() ) if output_format == "executorch": # Every executorch option is popped above, so a leftover kwarg is a typo. Fail @@ -1407,10 +1407,8 @@ def _save_as_executorch(exp_program: Any, file_path: str, **kwargs: Any) -> None from torch_tensorrt.executorch import export except ImportError: raise ImportError( - "ExecuTorch is not installed, and is published for Linux only. Install " - "with: pip install --pre " - '"torch_tensorrt[executorch]" --extra-index-url ' - "https://download.pytorch.org/whl/nightly/cu130 to use output_format='executorch'." + "ExecuTorch is not installed, and is published for Linux only, to use " + "output_format='executorch'. Install with: " + executorch_install_command() ) import torch_tensorrt.dynamo.runtime.meta_ops.register_meta_ops # noqa: F401 diff --git a/py/torch_tensorrt/_utils.py b/py/torch_tensorrt/_utils.py index 5d43db0e57..565fae87de 100644 --- a/py/torch_tensorrt/_utils.py +++ b/py/torch_tensorrt/_utils.py @@ -26,6 +26,37 @@ def sanitized_torch_version() -> Any: ) +def executorch_install_channel() -> str: + """The PyTorch nightly channel that carries the ExecuTorch build matching this torch. + + ExecuTorch publishes a distinct wheel per CUDA channel (``+cu130`` and ``+cu132`` are separate + builds), so an install instruction has to name the channel that matches the user's torch, or a + CUDA 13.2 user installs a CUDA 13.0 ExecuTorch. Derived from ``torch.version.cuda`` rather than + hardcoded for that reason. Falls back to the literal placeholder ``cuXYZ`` when torch reports no + CUDA build, so the message stays honest instead of naming a channel the user cannot use. + """ + cuda_version = torch.version.cuda + if not cuda_version: + return "cuXYZ" + major, _, minor = cuda_version.partition(".") + return f"cu{major}{minor or '0'}" + + +def executorch_install_command() -> str: + """The exact ``pip install`` line for the ExecuTorch authoring stack, channel included. + + Shared by every runtime error message that tells a user how to install ExecuTorch, so the + channel is derived once from the running torch and the three messages cannot drift from each + other or from the pin. ``--upgrade`` because the message is raised from inside an already + installed ``torch_tensorrt``: without it pip treats the requirement as satisfied and exits 0 + without adding the ``executorch`` extra. + """ + return ( + 'pip install --pre --upgrade "torch_tensorrt[executorch]" ' + f"--extra-index-url https://download.pytorch.org/whl/nightly/{executorch_install_channel()}" + ) + + def check_cross_compile_trt_win_lib() -> bool: # cross compile feature is only available on linux # build engine on linux and run on windows diff --git a/py/torch_tensorrt/executorch/__init__.py b/py/torch_tensorrt/executorch/__init__.py index 0ef31ccf4b..28becd4df8 100644 --- a/py/torch_tensorrt/executorch/__init__.py +++ b/py/torch_tensorrt/executorch/__init__.py @@ -22,12 +22,12 @@ def _has_executorch_exir() -> bool: if not _has_executorch_exir(): def __getattr__(name: str) -> NoReturn: + from torch_tensorrt._utils import executorch_install_command + raise ImportError( f"Cannot access torch_tensorrt.executorch.{name}: " "ExecuTorch with executorch.exir is required, and is published for " - "Linux only. Install with: pip install --pre " - '"torch_tensorrt[executorch]" --extra-index-url ' - "https://download.pytorch.org/whl/nightly/cu130" + "Linux only. Install with: " + executorch_install_command() ) __all__ = [ diff --git a/tests/ci/suites.py b/tests/ci/suites.py index 7b2fd142c1..d3cd6b8e5f 100644 --- a/tests/ci/suites.py +++ b/tests/ci/suites.py @@ -283,8 +283,11 @@ def __post_init__(self) -> None: keyword=( # The pairing test is the one check here that needs a real ExecuTorch installed, so # it has to survive this deselection. Everything else in that file is a - # source-consistency check the lint job already covers. - "not test_executorch_pin or test_the_pinned_commit_is_the_pinned_wheels_own_source" + # source-consistency check the lint job already covers. The pin updater's own tests + # are repository tooling with no GPU or ExecuTorch need, covered by the lint job, so + # they are deselected here rather than run in a CUDA container. + "(not test_executorch_pin or test_the_pinned_commit_is_the_pinned_wheels_own_source)" + " and not test_update_executorch_pin" ), setup=("executorch",), jobs="auto", diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index e18a949ec9..d1f0b46e83 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -22,6 +22,7 @@ import shlex import subprocess import sys +import tempfile from collections import Counter from pathlib import Path @@ -75,24 +76,20 @@ def _requirement_disagrees(actual: str, expected: str, version: str) -> str: # Anywhere else the commit appears it is named on the same line, as a shell default or prose. NAMED_COMMIT = re.compile(r"executorch[_a-z]*[^0-9a-f]*([0-9a-f]{40})", re.IGNORECASE) -# Requirements that end up in metadata someone resolves at install time. A patch release off -# the same branch has to stay installable, so these take the range. Everything else pins -# exactly: a build input compiled against one wheel, or a comment labelling a source sha. -# Only literal requirements land here. setup.py and tests/ci/runner.py derive theirs from -# dev_dep_versions.yml, so the search below no longer sees them and -# test_derived_requirements_match_the_pin covers them instead. -# -# Empty today: the only literal range left was the justfile's install recipe, and it installs -# the wheel the delegate is compiled against, so it pins exactly like the rest. - -# A step that exists to reproduce what a user runs belongs to the range group even inside a -# file that otherwise pins build inputs, so the marker travels with the line rather than -# with the path. -# An explicit opt-out token rather than prose. "verify the end user's workflow" is a sentence -# someone can write, or paste, above a requirement without meaning to license a range there. PAIRING_TEST = "test_the_pinned_commit_is_the_pinned_wheels_own_source" +# A requirement takes the install-time range only when a comment carrying this exact token sits +# above it, so a site that reproduces a user's workflow can stay installable across a patch +# release. An explicit opt-out token rather than prose: "verify the end user's workflow" is a +# sentence someone can paste above a requirement without meaning to license a range there. USER_WORKFLOW_MARKER = "pin-check: range-ok" +# A pip install of the plain torch-tensorrt wheel on a platform that has no ExecuTorch dev wheel +# carries this token. win32 is the case: the glob it installs also matches the Linux-only runtime +# wheel, so the channel scan reaches it, but ExecuTorch publishes no win32 nightly and the +# [executorch] extra is Linux-only. An explicit token rather than inference, so the exemption is +# deliberate and cannot be granted by accident to a Linux install that simply lost its index. +NO_NIGHTLY_MARKER = "pin-check: no-nightly" + # The files expected to pin ExecuTorch, mapped to how many sites each must carry, excluding # dev_dep_versions.yml itself. A count per file rather than just the set of files, because a site # that loses its version stops matching the search entirely rather than reporting a mismatch, and @@ -106,8 +103,9 @@ def _requirement_disagrees(actual: str, expected: str, version: str) -> str: "docker/MODULE.bazel.docker": 1, "docker/MODULE.bazel.ngc": 1, "justfile": 1, - # Two: the install command and the prose sentence below it. - "py/torch-tensorrt-executorch-runtime/README.md": 2, + # One: the fenced install command. The prose sentence below it is documentation, checked for + # pin agreement but not counted, so it cannot stand in for the command if that loses its pin. + "py/torch-tensorrt-executorch-runtime/README.md": 1, "py/torch-tensorrt-executorch-runtime/pyproject.toml": 1, "toolchains/ci_workspaces/MODULE.bazel.tmpl": 1, } @@ -162,23 +160,30 @@ def _versions() -> dict: return dict(re.findall(r'^(__\w+__): "([^"]+)"', text, re.MULTILINE)) -def _wants_range(path: str, number: int) -> bool: - # Scan upward past blanks and comment lines, so an explanatory line between the opt-out and - # the requirement neither reclassifies the site nor fails the build for a cosmetic reason. - # Only a comment carrying the token licenses a range; the first line of real content stops - # the scan, so the opt-out cannot leak onto an unrelated requirement further down. - lines = (REPO_ROOT / path).read_text().splitlines() +def _has_marker_above(path: str, number: int, marker: str) -> bool: + """Whether a comment carrying ``marker`` sits directly above line ``number``. + + Scans upward past blank and comment lines; the first line of real content stops the scan, so + the marker cannot leak from an unrelated command far above onto this one. + """ + lines = (REPO_ROOT / path).read_text(encoding="utf-8").splitlines() for line in reversed(lines[: number - 1]): stripped = line.strip() if not stripped: continue if not stripped.startswith("#"): return False - if USER_WORKFLOW_MARKER in stripped: + if marker in stripped: return True return False +def _wants_range(path: str, number: int) -> bool: + # Only a comment carrying the token licenses a range; the first line of real content stops the + # scan, so the opt-out cannot leak onto an unrelated requirement further down. + return _has_marker_above(path, number, USER_WORKFLOW_MARKER) + + def _release_line(version: str) -> tuple[str, str]: """Split a pin into its major and minor, for either a release or a nightly. @@ -195,17 +200,30 @@ def _expected(path: str, number: int, version: str) -> str: return f"executorch=={version}" major, minor = _release_line(version) - # Only the top-level setup.py carries the Linux marker. It is the site uv resolves for the - # win32 required-environment, where PyPI's candidates stop below this floor. - marker = "; platform_system == 'Linux'" if path == "setup.py" and number > 1 else "" - return f"executorch>={version},<{major}.{int(minor) + 1}{marker}" + return f"executorch>={version},<{major}.{int(minor) + 1}" + +# Nightly channels that actually carry the pinned ExecuTorch line. cu124 and cu128 exist on the +# index but are frozen at a CPU-only 0.5.0.dev build, so a recipe pointed at them resolves nothing +# the pin can use. Only these three serve the 1.5.0.dev wheels this change installs. +_PUBLISHED_NIGHTLY_CHANNELS = frozenset({"cu126", "cu130", "cu132"}) + +# Tracked files with no suffix that still carry install commands. justfile writes the nightly +# ExecuTorch install for local builds, so the printed-install walk has to read it by name. +_EXTENSIONLESS_INSTALL_FILES = frozenset({"justfile"}) + +# Index-URL variables whose value legitimately arrives from the CI environment and so has no +# assignment in the tree to resolve. An unresolved variable is accepted only if it is one of +# these; every other unresolved name, including a typo of a real one, fails the channel check +# rather than passing on sight. Empty today: every ExecuTorch install channels through either a +# literal nightly URL or ${EXECUTORCH_INDEX_URL}, which is assigned in executorch-build-linux.yml +# and therefore resolvable. Kept as the explicit seam a future environment-provided index goes +# through. +_ENVIRONMENT_INDEX_VARIABLES: frozenset[str] = frozenset() # The bazel repositories annotate their pinned commit with the wheel it corresponds to, in a # comment, because bazel fetches by commit and has no requirement string to carry. Those are the # only comment sites that count as pins, and the commit beside them is checked separately. -_PUBLISHED_NIGHTLY_CHANNELS = frozenset({"cu124", "cu126", "cu128", "cu130", "cu132"}) - _ANNOTATED_COMMIT_SITES = frozenset( { "MODULE.bazel", @@ -243,6 +261,60 @@ def _without_trailing_comment(path: str, text: str) -> str: return text +def _counts_toward_minimum(path: str, number: int) -> bool: + """Whether a requirement at this line counts toward the per-file minimum. + + In a prose file a requirement in running text is documentation, not a live pin. Gutting the + fenced install command to a bare ``executorch`` while a sentence below still spelled the pin + kept the per-file count satisfied and left the command unpinned, so only a requirement inside + a fenced code block counts for markdown. Every other file counts every live line; the + reStructuredText sites are guarded by the install-instruction test instead. + """ + if not path.endswith(".md"): + return True + fenced = False + for current, content in enumerate( + (REPO_ROOT / path).read_text(encoding="utf-8").splitlines(), start=1 + ): + if current == number: + return fenced + if content.lstrip().startswith(("```", "~~~")): + fenced = not fenced + return False + + +def _strip_whole_line_comments(text: str) -> str: + """Blank out whole-line ``#`` comments, preserving line count so DOTALL spans stay aligned. + + The Bazel commit match walks from ``name = "executorch"`` to the ``commit = "..."`` line with + DOTALL, so a commit commented out and replaced by a live ``branch = "main"`` still matched the + commented copy and the build floated to a branch while this test stayed green. + """ + return "\n".join( + "" if line.lstrip().startswith("#") else line for line in text.splitlines() + ) + + +def _resolve_shell_assignment(text: str, variable: str, before: int) -> str | None: + """The last literal ``VAR=...`` assignment of ``variable`` in ``text`` before offset ``before``. + + An install that channels through ``--extra-index-url "${VAR}"`` proves nothing on its own: the + value is whatever ``VAR`` was last set to. Repointing that assignment at PyPI, or dropping its + ``nightly/`` segment, left the install counted as channelled while it resolved nothing. Resolve + the assignment so the channel is checked where it is actually set. Returns ``None`` when no + assignment is found, meaning the value comes from the environment and cannot be resolved here. + """ + assignment = re.compile( + rf"""^\s*(?:export\s+)?{re.escape(variable)}=["']?([^"'\n]*)""", re.MULTILINE + ) + resolved = None + for match in assignment.finditer(text): + if match.start() >= before: + break + resolved = match.group(1) + return resolved + + def test_every_requirement_matches_the_pin() -> None: version = _versions()["__executorch_version__"] @@ -266,9 +338,11 @@ def test_every_requirement_matches_the_pin() -> None: # the same line kept the per-file count satisfied and left the install unpinned. The # annotated commit sites write their pin as a whole-line comment, which is handled above, # so nothing legitimate is lost here. + counts = _counts_toward_minimum(path, int(number)) for actual in REQUIREMENT.findall(_without_trailing_comment(path, text)): found += 1 - seen[path] += 1 + if counts: + seen[path] += 1 reason = _requirement_disagrees(actual, expected, version) if reason: wrong.append(f"{path}:{number} has {actual}, {reason}") @@ -415,6 +489,60 @@ def test_derived_requirements_match_the_pin(monkeypatch) -> None: ) +_RUNTIME_SETUP_PY = "py/torch-tensorrt-executorch-runtime/setup.py" + + +def _runtime_install_requires() -> dict[str, str]: + """The runtime wheel's ``install_requires`` entries, each mapped to its source text. + + Read as source, not imported: importing this setup.py runs a Bazel build. The values are + f-strings built at build time from the installed distributions, so the source segment is what + the check compares, not a resolved string. + """ + tree = ast.parse((REPO_ROOT / _RUNTIME_SETUP_PY).read_text(encoding="utf-8")) + source = (REPO_ROOT / _RUNTIME_SETUP_PY).read_text(encoding="utf-8") + call = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "setup" + ) + requires = next( + keyword.value for keyword in call.keywords if keyword.arg == "install_requires" + ) + entries = {} + for element in requires.elts: + text = ast.get_source_segment(source, element) + distribution = re.match(r'f?"([A-Za-z0-9_.-]+)', text) + if distribution: + entries[distribution.group(1)] = text + return entries + + +def test_the_runtime_wheel_pins_executorch_to_the_public_pin() -> None: + """The runtime wheel's own ExecuTorch requirement has to pin the pinned version, stripped. + + This is the one requirement whose native code is compiled against the ExecuTorch runtime, so a + wheel that requires a different ExecuTorch than it was built against loads a mismatched runtime. + The literal search above cannot see it: setup.py builds the string from the installed + distribution, so ``executorch==`` is followed by a brace, not a digit. Loosening it to a bare + ``executorch`` left every other test green. + """ + entries = _runtime_install_requires() + assert ( + "executorch" in entries + ), f"{_RUNTIME_SETUP_PY} install_requires no longer pins executorch: {sorted(entries)}" + # The value is built from installed_version("executorch"), the same source torch and + # torch-tensorrt use, and stripped of its local label the same way. Compare the source text so + # a bare name, a hardcoded version, or a different version source is rejected. + assert ( + entries["executorch"] == 'f"executorch=={public_version(executorch_version)}"' + ), ( + f"{_RUNTIME_SETUP_PY} must pin executorch to public_version(executorch_version), so the " + f"wheel requires the ExecuTorch it was compiled against, but it declares " + f"{entries['executorch']}" + ) + + def test_the_runner_follows_the_row_s_cuda_version(monkeypatch) -> None: """Call the runner and read the URL it builds, rather than matching its source. @@ -450,6 +578,81 @@ def channel_for(cu_version: str | None) -> str: assert channel_for(None).endswith("/nightly/cu130") +def _load_utils_channel_helpers(fake_cuda: str | None): + """Exec ``executorch_install_channel``/``executorch_install_command`` with a stub torch. + + ``py/torch_tensorrt/_utils.py`` imports ``tensorrt`` and the built ``torch_tensorrt``, neither + installed on the lint runner, so it cannot be imported here. Extract just the two functions and + exec them against a fake ``torch`` whose ``version.cuda`` is ``fake_cuda``, which is all they + read. This keeps the test on the source that ships rather than a copy of its logic. + """ + source = (REPO_ROOT / "py/torch_tensorrt/_utils.py").read_text(encoding="utf-8") + tree = ast.parse(source) + wanted = {"executorch_install_channel", "executorch_install_command"} + functions = [ + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name in wanted + ] + assert {f.name for f in functions} == wanted, ( + "py/torch_tensorrt/_utils.py must define executorch_install_channel and " + f"executorch_install_command; found {sorted(f.name for f in functions)}" + ) + + class _Version: + cuda = fake_cuda + + namespace: dict[str, object] = {"torch": type("torch", (), {"version": _Version})} + module = ast.Module(body=functions, type_ignores=[]) + exec(compile(module, "", "exec"), namespace) # noqa: S102 + return ( + namespace["executorch_install_channel"], + namespace["executorch_install_command"], + ) + + +def test_the_executorch_install_message_names_the_torch_channel() -> None: + """The three ExecuTorch install messages derive their channel from the running torch. + + ExecuTorch ships a distinct wheel per CUDA channel, so a message that hardcodes cu130 tells a + CUDA 13.2 user to install a CUDA 13.0 build. The messages route through + ``executorch_install_command`` so the channel is computed once from ``torch.version.cuda``. + A source-text assertion cannot see the value the format string produces, so exercise the helper + across both published 13.x channels and the no-CUDA fallback. + """ + channel, command = _load_utils_channel_helpers("13.2") + assert channel() == "cu132" + message = command() + assert "download.pytorch.org/whl/nightly/cu132" in message, message + # Raised from inside an installed torch_tensorrt, so pip treats the requirement as satisfied + # and exits 0 without the extra unless --upgrade forces a re-resolve. --pre selects the dev pin. + assert "--upgrade" in message and "--pre" in message, message + + channel_130, command_130 = _load_utils_channel_helpers("13.0") + assert channel_130() == "cu130" + assert "nightly/cu130" in command_130() + + # No CUDA build reports a placeholder rather than a channel the user cannot install from. + channel_none, command_none = _load_utils_channel_helpers(None) + assert channel_none() == "cuXYZ" + assert "nightly/cuXYZ" in command_none() + + # Every message site delegates to the shared command rather than spelling its own URL, so the + # derived channel and --upgrade cannot regress in one message while the test watches another. + for path in ( + "py/torch_tensorrt/_compile.py", + "py/torch_tensorrt/executorch/__init__.py", + ): + text = (REPO_ROOT / path).read_text(encoding="utf-8") + assert "executorch_install_command()" in text, ( + f"{path} must build its ExecuTorch install message with " + "executorch_install_command() so the channel derives from the running torch" + ) + assert ( + "nightly/cu130" not in text + ), f"{path} still hardcodes nightly/cu130 in a message instead of deriving the channel" + + def test_derived_requirements_roll_the_minor_over(tmp_path: Path) -> None: # The upper bound is a version, not a decimal: 1.9 has to become 1.10, not 1.1. # Spelled through a variable because the search above reads this file too, and a @@ -529,7 +732,8 @@ def test_every_source_commit_matches_the_pin() -> None: found = 0 seen: Counter[str] = Counter() for path in _git("grep", "-lI", "-E", 'name = "executorch"').split(): - for match in BAZEL_COMMIT.finditer((REPO_ROOT / path).read_text()): + source = _strip_whole_line_comments((REPO_ROOT / path).read_text()) + for match in BAZEL_COMMIT.finditer(source): found += 1 seen[path] += 1 if match.group(1) != commit: @@ -539,7 +743,12 @@ def test_every_source_commit_matches_the_pin() -> None: path, number, text = line.split(":", 2) if path == VERSIONS.name: continue - for actual in NAMED_COMMIT.findall(text): + # A commit in a comment is not a pin. Grepping raw lines let EXECUTORCH_REF float to a + # branch with the real SHA left behind in a "# ..." comment on the same file, which kept + # this scan green. The Bazel walk above already strips comments; do the same here. + if _is_commented_out(path, text): + continue + for actual in NAMED_COMMIT.findall(_without_trailing_comment(path, text)): found += 1 seen[path] += 1 if actual != commit: @@ -652,6 +861,64 @@ def test_the_lockfile_records_the_same_executorch_range_as_setup_py(): ) +_INSTALL_INVOCATION = re.compile( + r"(?:python[0-9.]*\s+-m\s+pip|uv\s+pip|\bpip)\s+(?:install|wheel)\b" +) + + +def _blank_comments_preserving_length(block: str) -> str: + """``block`` with comment text replaced by spaces, keeping every offset and newline in place. + + Used only to locate pip keywords without a ``# pip install ...`` in a comment starting a false + invocation. Length is preserved so an offset into the original block indexes the same character + here. Whole-line ``#`` comments blank entirely; a trailing `` #`` comment blanks from the hash, + but a ``#cu130`` URL fragment (no preceding space) is left intact. + """ + out = [] + for physical in block.splitlines(keepends=True): + newline = "\n" if physical.endswith("\n") else "" + body = physical[:-1] if newline else physical + if body.lstrip().startswith("#"): + body = " " * len(body) + else: + hash_at = re.search(r"(?:^|\s)#", body) + if hash_at: + cut = hash_at.start() + body = body[:cut] + " " * (len(body) - cut) + out.append(body + newline) + return "".join(out) + + +def _install_invocation_window(block: str, match_offset: int) -> str: + """The slice of ``block`` belonging to the one pip/uv invocation that owns the match. + + The channel and the requirement it channels have to belong to the *same* invocation. A window + bounded only at blank lines was too wide: it spanned every step of a contiguous YAML job and + every line of a shell if/else, so a ``--extra-index-url`` from a neighbouring ``pip install``, + an ``echo``, or prose satisfied the check for an install that carried none of its own. Two real + holes this closed: the win32 branch of ``install-torch-tensorrt.sh`` borrowed the else branch's + URL, and the ``.[executorch]`` step in ``docgen.yml`` borrowed the *Install base deps* step's. + + An invocation runs from its ``pip``/``uv pip`` keyword to the next such keyword in the block, or + to the block's end. That single boundary spans a backslash-continued shell command, a YAML + ``run:`` body and a Python error message built from adjacent string fragments alike, because + none of those start a second invocation between the keyword and the URL. ``match_offset`` is a + ``block``-relative offset into the original (un-stripped) text, so a requirement that appears + twice in one block resolves to its own invocation rather than the first copy's. When no + invocation keyword precedes the match the whole block is returned, leaving non-install prose + matches to the caller's other filters. + """ + scan = _blank_comments_preserving_length(block) + starts = [m.start() for m in _INSTALL_INVOCATION.finditer(scan)] + preceding = [s for s in starts if s <= match_offset] + if not preceding: + return block + begin = preceding[-1] + following = [s for s in starts if s > match_offset] + finish = following[0] if following else len(block) + return block[begin:finish] + + @pytest.mark.unit def test_every_printed_install_instruction_names_the_nightly_channel(): """Every ``[executorch]`` install instruction has to carry the nightly index. @@ -679,15 +946,40 @@ def test_every_printed_install_instruction_names_the_nightly_channel(): # The local-path spelling counts too. Matching only the named-distribution form left the four # sites that write "pip install .[executorch]" unguarded: the nightly index could be deleted # from all four with this test green. + # A fourth shape: a direct "executorch==" or "executorch>=" in a pip command. The + # runtime README build recipe installs ExecuTorch this way, and its nightly index could be + # deleted with this test green because none of the three shapes above match a bare + # distribution name. Gated on a pip context below so a requirement in pyproject or a comment + # is not mistaken for an install instruction. + # The built-wheel shape covers both the plain "torch_tensorrt*.whl" and the runtime wheel + # "torch_tensorrt_executorch_runtime-*.whl": the latter's install_requires names the same + # nightly ExecuTorch, so its documented install needs the channel too, and matching only the + # plain glob left the runtime README's Use command unscanned. It is gated on a real + # "pip install" without "--no-deps" below, so naming the file in an "ls" or a heredoc, or a + # "--no-deps" install that fetches nothing, is not mistaken for a dependency-resolving install. + # A fifth shape: a bare "pip install executorch" with no version operator. It resolves the + # stable 1.4.1 from PyPI, the version this change moves away from, and carries no operator so + # the shapes above miss it. Matched as a standalone distribution token and gated on a + # pip-install context below, so the word in a path, an import, a filename, or prose is not + # mistaken for an install. Like the named-distribution form it needs both the channel and + # --pre, since a bare name without --pre picks the stable release even off the nightly index. extra = re.compile( r"""torch[-_]tensorrt\[[^]]*executorch[^]]*\]""" r"""|(?=)""" + r"""|(?_-])""" ) missing = [] for name in tracked: - if not name or not name.endswith( - (".py", ".sh", ".md", ".yml", ".yaml", ".rst", ".txt") + base = name.rsplit("/", 1)[-1] + # Select by suffix, plus a few extensionless files that carry install commands. The + # justfile in particular writes a real "uv pip install ... executorch==" with its + # nightly index; filtering on suffix alone never read it, so both the index and the pin + # could be dropped from it with this test green. + if not base or not ( + name.endswith((".py", ".sh", ".md", ".yml", ".yaml", ".rst", ".txt")) + or base in _EXTENSIONLESS_INSTALL_FILES ): continue # This file states the rule; it is not itself an instruction. @@ -703,22 +995,130 @@ def test_every_printed_install_instruction_names_the_nightly_channel(): continue text = path.read_text(encoding="utf-8", errors="replace") for match in extra.finditer(text): - # The instruction is the pip invocation, so bound the window at the surrounding - # blank-line-separated block rather than guessing a fixed number of lines. - start = text.rfind("\n\n", 0, match.start()) + 1 - end = text.find("\n\n", match.end()) - block = text[start : end if end != -1 else len(text)] line = text.count("\n", 0, match.start()) + 1 + # Bound the window at the surrounding blank-line-separated block first, then narrow to + # the single pip invocation that owns the match. The block alone was too wide: a + # contiguous YAML job or a shell if/else is one block, so a --extra-index-url, --pre or + # channel from a neighbouring command satisfied an install that carried none itself. + block_start = text.rfind("\n\n", 0, match.start()) + 1 + block_end = text.find("\n\n", match.end()) + block = text[block_start : block_end if block_end != -1 else len(text)] + block = _install_invocation_window(block, match.start() - block_start) + # Strip comments after windowing: a shell comment naming the channel or --pre is prose, + # not an argument pip sees, so a gutted install line with a decoy comment beside it must + # not satisfy the check. Whole-line comments drop entirely; a trailing "#" comment is + # cut, but not the "#cu130" fragment of a URL, which carries no space. + block = "\n".join( + re.sub(r"(?:^|\s)#.*$", "", bl) + for bl in block.splitlines() + if not bl.lstrip().startswith("#") + ) + # A plain torch-tensorrt wheel install on a platform with no ExecuTorch dev wheel is + # exempt. win32 installs a glob that also matches the Linux-only runtime wheel, so the + # scan reaches it, but ExecuTorch publishes no win32 nightly. The marker has to sit + # directly above the invocation, and a separate test keeps it inside a win32 guard, so a + # gutted Linux install cannot claim it. + if _has_marker_above(name, line, NO_NIGHTLY_MARKER): + continue + is_direct_install = bool( + re.fullmatch(r"executorch(?:_[a-z]+)?\s*(?:==|>=)", match.group(0)) + ) + # A bare "executorch==" only counts as an instruction inside a pip command. Anywhere + # else it is a dependency declaration or prose, guarded by other tests. + if is_direct_install: + if not re.search(r"\bpip\s+(?:install|wheel)\b", block): + continue + # The version after the operator has to be a literal. "executorch==${OLD}" or + # "executorch==$(...)" clears the pip-context gate yet pins nothing: pip installs + # whatever the expansion yields, which floats off the pin. The one legitimate + # non-literal is docgen's, which reads __executorch_version__ out of + # dev_dep_versions.yml; a separate test proves that command equals the pin. + after = text[match.end() : match.end() + 80].lstrip() + if not after[:1].isdigit() and not after.startswith( + "$(python3 -c 'import yaml;" + ): + missing.append( + f"{name}:{line} installs executorch at a non-literal version, which pins " + "nothing: pip resolves whatever the expansion yields off the nightly index" + ) + continue + is_bare_name = match.group(0) == "executorch" + if is_bare_name: + # A bare distribution name is an install only inside a pip command; the same word + # in a path, an import, or prose is not. + if not re.search(r"\bpip\s+(?:install|wheel)\b", block): + continue + # The word also appears in prose that shares a block with a real + # "torch_tensorrt[executorch]" install, so require the bare token to sit on the pip + # command line itself before treating it as the install target. A bare target there + # pins nothing even with --pre and the channel: pip resolves the newest nightly, not + # this pin. Nothing in the tree installs executorch bare, so it is always a defect. + command_line = text.splitlines()[line - 1] + if re.search(r"\bpip\s+(?:install|wheel)\b", command_line): + missing.append( + f"{name}:{line} installs executorch by bare name, which pins nothing: pip " + "resolves the newest nightly rather than the pinned version" + ) + continue + is_built_wheel = bool( + re.fullmatch( + r"torch_tensorrt(?:_executorch_runtime-)?\*\.whl", match.group(0) + ) + ) + # This glob names a local file, so it pulls the nightly ExecuTorch dependency only in a + # "pip install" that resolves dependencies. Naming the file elsewhere (an "ls", a + # heredoc, "pip wheel", or a "--no-deps" install) fetches no ExecuTorch, so no channel + # applies. + if is_built_wheel and ( + not re.search(r"\bpip\s+install\b", block) + or re.search(r"(?:^|\s)--no-deps(?:\s|$)", block) + ): + continue channel = re.search( r"download\.pytorch\.org/whl/nightly(?:/(cu\d+))?", block ) + # CI passes the channel through a variable rather than a literal URL. Capture the + # variable name so its assignment can be resolved: accepting the reference on sight let + # the assignment be repointed at PyPI, or stripped of its nightly segment, with the + # install still counted as channelled. + variable_index = re.search( + r"--(?:extra-index-url|index-url)\s+\"?\$\{?([A-Za-z_][A-Za-z0-9_]*)", + block, + ) + if not channel and variable_index: + # Resolve the variable's last assignment before this install and check the channel + # there. An unresolved variable is accepted only when it is a known workflow input + # whose value arrives from the CI environment; every other unresolved name, a typo + # among them, fails rather than passing on sight. + assignment = _resolve_shell_assignment( + text, variable_index.group(1), match.start() + ) + if assignment is None: + if variable_index.group(1) in _ENVIRONMENT_INDEX_VARIABLES: + continue + missing.append( + f"{name}:{line} channels through ${{{variable_index.group(1)}}}, which has " + "no assignment in the tree and is not a known CI index input" + ) + continue + channel = re.search( + r"download\.pytorch\.org/whl/nightly(?:/(cu\d+))?", assignment + ) + if not channel: + missing.append( + f"{name}:{line} installs from ${{{variable_index.group(1)}}}, set to " + f"{assignment!r}, which is not the nightly channel" + ) + continue if not channel: missing.append(f"{name}:{line} names no nightly channel") continue # A substring proves a string sits nearby, not that it resolves anything. Rewriting # every channel in the tree to a nonexistent cu999 left this green. Not compared - # against __cuda_version__: five sites legitimately say cu130 while the pin says 13.2, - # and the index really does carry the pinned ExecuTorch under both. + # against __cuda_version__: the runtime error messages derive their channel from the + # user's torch build, and the documented recipes name a concrete published channel that + # carries the pinned ExecuTorch, so a literal cuXYZ here is checked only for being one + # the project publishes. suffix = channel.group(1) if suffix and suffix not in _PUBLISHED_NIGHTLY_CHANNELS: missing.append( @@ -745,6 +1145,58 @@ def test_every_printed_install_instruction_names_the_nightly_channel(): ) +@pytest.mark.unit +def test_the_no_nightly_marker_only_exempts_a_win32_install(): + """The ``no-nightly`` exemption is legitimate only where no ExecuTorch dev wheel exists. + + The channel scan skips an install carrying ``pin-check: no-nightly`` above it. That is correct + for win32, whose wheel glob also matches the Linux-only runtime wheel while ExecuTorch ships no + win32 nightly. Without this test the marker is a blanket silencer: strip the index from the + Linux install, paste the marker above it, and the channel scan stays green. Requiring the marker + to sit inside a ``win32`` platform guard keeps the exemption tied to the one case it describes. + """ + tracked = subprocess.run( + ["git", "ls-files", "-z"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.split("\0") + + misplaced = [] + for name in tracked: + if not name or name == "tests/py/dynamo/executorch/test_executorch_pin.py": + continue + path = REPO_ROOT / name + if not path.is_file(): + continue + text = path.read_text(encoding="utf-8", errors="replace") + if NO_NIGHTLY_MARKER not in text: + continue + lines = text.splitlines() + control = re.compile(r"^\s*(?:if\b|elif\b|else\b|fi\b)") + for index, content in enumerate(lines): + if NO_NIGHTLY_MARKER not in content: + continue + # The exempted install sits just below the marker, so the branch it lives in is the + # nearest control-flow keyword above it. Requiring that keyword to be the win32 guard + # ties the exemption to the one platform it describes: a marker pasted onto a Linux + # "else" install resolves to that "else", not to "if ... win32", and is rejected. + branch = next( + (lines[j] for j in range(index - 1, -1, -1) if control.match(lines[j])), + "", + ) + if "win32" not in branch: + misplaced.append( + f"{name}:{index + 1} carries {NO_NIGHTLY_MARKER!r} outside a win32 branch, " + "so it would exempt a Linux install that simply lost its index" + ) + + assert not misplaced, ( + "the no-nightly exemption is only valid inside a win32 branch: " f"{misplaced}" + ) + + @pytest.mark.unit def test_the_pin_check_runs_in_ci(): """This file has to be invoked by something, or its assertions never execute. @@ -772,10 +1224,21 @@ def test_the_pin_check_runs_in_ci(): f"linter.yml triggers on {sorted(map(str, trigger_names))}, not pull_request, so the pin " "check never runs when a pull request changes the pin" ) - # Match a live pytest invocation, not the filename anywhere in the script. Neutralising the - # command and leaving it in a shell comment satisfied a plain substring test. + # A paths filter on the trigger would keep the workflow from firing on a pin change outside + # those paths, leaving every assertion below green while nothing ran. + pull_request = triggers.get("pull_request") if isinstance(triggers, dict) else None + if isinstance(pull_request, dict): + for path_filter in ("paths", "paths-ignore"): + assert path_filter not in pull_request, ( + f"linter.yml narrows the pull_request trigger with {path_filter}, so a change to " + "the pin outside those paths would not run this check" + ) + # Match a live pytest invocation with pytest as the command word, not the filename anywhere + # in the script. A plain substring test was satisfied by a comment; an "anything before + # pytest" test was satisfied by "echo python3 -m pytest", which prints the command and runs + # nothing. invocation = re.compile( - rf"^\s*[^#\n]*\bpytest\b[^\n]*{re.escape(pathlib.Path(__file__).name)}", + rf"^\s*(?:python[0-9.]*\s+-m\s+)?pytest\b[^\n]*{re.escape(pathlib.Path(__file__).name)}", re.MULTILINE, ) owning = [ @@ -799,20 +1262,61 @@ def test_the_pin_check_runs_in_ci(): assert ( job_condition in live_conditions ), f"job {name} runs under {job_condition!r}, so the pin check may never dispatch" - # Tokenised, not substring-matched, and every way of neutralising the run counts. "--co" is - # pytest's own documented short form of "--collect-only" and slipped past a check for the long - # spelling. "|| true", "; true" and continue-on-error each discard the exit status, the last - # two at the step and at the job. - tokens = shlex.split(step["run"].replace("\\\n", " ")) - for flag in ("--collect-only", "--co", "--help", "-h"): - assert ( - flag not in tokens - ), f"the pin check in {name} passes {flag}, so no assertion executes" - for terminator in ("||", ";", "&"): - assert terminator not in tokens, ( - f"the pin check in {name} follows pytest with {terminator!r}, so its exit status does " - "not fail the step" - ) + + # Measure the effect rather than enumerating the spellings that defeat it. + # and shell shapes chased holes one at a time: -k, --co, --deselect, "; true", "set +e", a + # PYTEST_ADDOPTS in the env, an "if false" wrapper, an early "exit 0" each needed its own + # clause, and each clause missed a neighbour. Instead, run the step's own script against a + # stub test file and require its exit status to follow the test: non-zero when the stub fails, + # zero when it passes. Anything that discards the status, deselects the tests, or turns the + # run into a collect-only pass makes the failing case exit zero and fails this assertion. + def _run_ci_step_against(stub_body: str) -> int: + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + # The step invokes this file by its path; stand a stub with the same basename in a + # matching directory tree so the invocation resolves to the stub, not the real suite. + stub_dir = tmp_path / "tests/py/dynamo/executorch" + stub_dir.mkdir(parents=True) + (stub_dir / pathlib.Path(__file__).name).write_text( + stub_body, encoding="utf-8" + ) + # Run the exact script the step runs, under bash, with the step's own env overlaid. + # GITHUB_WORKSPACE points at the stub tree so "cd $GITHUB_WORKSPACE" lands there, and + # the interpreter is forced to this Python so the subprocess needs nothing installed. + env = {**os.environ, "GITHUB_WORKSPACE": str(tmp_path)} + for key, value in (step.get("env") or {}).items(): + env[str(key)] = str(value) + script = step["run"].replace( + "python3 -m pytest", f"{shlex.quote(sys.executable)} -m pytest" + ) + return subprocess.run( + ["bash", "-c", script], + env=env, + capture_output=True, + text=True, + ).returncode + + passing_stub = ( + "import pytest\n\n\n" + "@pytest.mark.unit\n" + "def test_stub_passes():\n" + " assert True\n" + ) + failing_stub = ( + "import pytest\n\n\n" + "@pytest.mark.unit\n" + "def test_stub_fails():\n" + " assert False, 'a pin inconsistency must fail the CI step'\n" + ) + assert _run_ci_step_against(passing_stub) == 0, ( + f"the pin check step in {name} does not exit zero when the pin tests pass, so it cannot " + "be trusted to gate on their result" + ) + assert _run_ci_step_against(failing_stub) != 0, ( + f"the pin check step in {name} exits zero even when a pin test fails, so a pin " + "inconsistency would not fail CI" + ) + assert not step.get( "continue-on-error" ), f"the pin check step in {name} is continue-on-error, so a failure cannot fail the job" @@ -824,7 +1328,14 @@ def test_the_pin_check_runs_in_ci(): # requirements.txt nor dependency-groups.lint carries them, and without them the step exits 1 # on "No module named pytest" before running any assertion. steps = job["steps"] - earlier = "\n".join(s.get("run") or "" for s in steps[: steps.index(step)]) + # Comment-stripped: a commented-out "uv pip install ... pytest pyyaml" still matched the raw + # text while installing nothing, so the step would die on a missing import at runtime. + earlier = "\n".join( + re.sub(r"(?:^|\s)#.*$", "", line) + for step_before in steps[: steps.index(step)] + for line in (step_before.get("run") or "").splitlines() + if not line.lstrip().startswith("#") + ) for package in ("pytest", "pyyaml"): assert re.search( rf"uv pip install --system[^\n]*\b{package}\b", earlier @@ -917,14 +1428,24 @@ def test_the_pairing_check_survives_the_gpu_lane_deselection() -> None: for job in workflow["jobs"].values() for step in job.get("steps", []) ] + [job.get("with", {}).get("script", "") for job in workflow["jobs"].values()] - invokes_tier = any( - re.search(r"^\s*trt_tier_executorch\b", script, re.MULTILINE) + # The call must not discard its own exit status. "trt_tier_executorch || true", a trailing + # ";", "&" or a pipe each let the tier fail while the lane stays green, so reject any status + # operator after the call on its line. + tier_calls = [ + match for script in scripts - ) - assert invokes_tier, ( + for match in re.finditer( + r"^\s*trt_tier_executorch\b([^\n]*)", script, re.MULTILINE + ) + ] + assert tier_calls, ( "executorch-test-linux.yml no longer calls trt_tier_executorch, so the pairing check " "never runs on the GPU lane even though its -k expression would select it" ) + assert any(not re.search(r"[|;&]", call.group(1)) for call in tier_calls), ( + "every trt_tier_executorch call in executorch-test-linux.yml discards its exit status " + "with a pipe, ';', '&' or '|| true', so a pairing failure cannot fail the lane" + ) # The manifest route: the executorch suite must exist and target a lane a runner requests. # A typo in its lane tuple silently drops it from every matrix, which the suite-name check diff --git a/tests/py/dynamo/executorch/test_update_executorch_pin.py b/tests/py/dynamo/executorch/test_update_executorch_pin.py new file mode 100644 index 0000000000..77595fe0c8 --- /dev/null +++ b/tests/py/dynamo/executorch/test_update_executorch_pin.py @@ -0,0 +1,255 @@ +"""The pin updater has to be trusted to run unattended and open a pull request, so its +parts are tested the way they fail in practice: version ordering that is not string order, +a wheel that forgot its provenance, and a rewrite that has to leave the tree in exactly the +state the pin guard demands. Every test runs the real function; none restates it. + +Text and metadata only, like test_executorch_pin.py: no network, no GPU, no ExecuTorch. The +two functions that reach the index (available_versions, wheel_git_version) are exercised +against captured output and a synthesized wheel, so this file runs on the lint runner. + +Every version here is a fake far-past date (year 2020) and every commit is an obvious +marker, never a real pin. The updater bumps the pin by replacing the old literal everywhere +it appears in the tree, so a real pin value living in this file would be rewritten by a +bump, quietly changing the fixtures. Synthetic values can never equal the live pin, so a +bump never touches this file. test_write_pins_updates_new_sites_but_never_its_own_source +holds that line. +""" + +from __future__ import annotations + +import importlib.util +import subprocess +import zipfile +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[4] +_SCRIPT = _REPO_ROOT / ".github" / "scripts" / "update_executorch_pin.py" +_SELF = "tests/py/dynamo/executorch/test_update_executorch_pin.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("update_executorch_pin", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +updater = _load() + + +# The shape of a `pip index versions executorch` run: dated nightlies of one line, out of +# order so a test that passes under string sorting still fails here. The dates are fake and +# far in the past so they can never equal the live pin. +_NIGHTLY_LIST = [ + "1.5.0.dev20200101+cu130", + "1.5.0.dev20200103+cu130", + "1.5.0.dev20200102+cu130", +] +# A stable channel carries finals and the occasional release candidate; the updater must +# take the newest final, not the newer-looking prerelease. The line is a fake 0.9 that +# ExecuTorch will never ship, so it cannot equal a real stable pin either. +_STABLE_LIST = ["0.9.1", "0.9.2", "0.9.3", "0.9.4rc1"] + + +def test_pick_target_nightly_takes_the_newest_date_not_the_longest_string() -> None: + # Distinct on purpose: if a tree-wide bump ever collapsed two of these into one, the + # list would stop testing ordering, so fail loudly the moment they are not unique. + assert len(set(_NIGHTLY_LIST)) == len(_NIGHTLY_LIST) + assert updater.pick_target(_NIGHTLY_LIST, "nightly") == "1.5.0.dev20200103" + + +def test_pick_target_strips_the_cuda_local_label() -> None: + # The pin serves every CUDA row, so the +cuXXX label the index carries must not survive + # into the pin. A label left on would fail the guard's exact-match on every site. + assert "+" not in updater.pick_target(_NIGHTLY_LIST, "nightly") + + +def test_pick_target_stable_ignores_dev_and_release_candidates() -> None: + assert updater.pick_target(_STABLE_LIST, "stable") == "0.9.3" + + +def test_pick_target_nightly_ignores_finals() -> None: + # A stable final on the nightly index is not a nightly; picking it would move the pin + # off the dev line the delegate is built against. + assert updater.pick_target(["0.9.3", "1.5.0.dev20200103"], "nightly") == ( + "1.5.0.dev20200103" + ) + + +def test_pick_target_nightly_ignores_release_candidates() -> None: + # An rc is a prerelease that sorts above every dev of the same line under PEP 440, so a + # filter that only excluded finals would let the first rc on the nightly index become the + # pin. A nightly is a dated dev build, and an rc is not one. + assert ( + updater.pick_target(["1.5.0.dev20200103+cu130", "1.5.0rc1+cu130"], "nightly") + == "1.5.0.dev20200103" + ) + + +def test_pick_target_raises_when_nothing_matches_the_track() -> None: + with pytest.raises(SystemExit): + updater.pick_target(["0.9.3", "0.9.2"], "nightly") + + +def test_available_versions_parses_the_pip_line(monkeypatch) -> None: + captured = ( + "executorch (1.5.0.dev20200103+cu130)\n" + "Available versions: 1.5.0.dev20200103+cu130, 1.5.0.dev20200102+cu130\n" + " INSTALLED: 1.1.0\n" + " LATEST: 1.5.0.dev20200103+cu130\n" + ) + monkeypatch.setattr(updater, "_run", lambda cmd: captured) + assert updater.available_versions([]) == [ + "1.5.0.dev20200103+cu130", + "1.5.0.dev20200102+cu130", + ] + + +def _synthesize_wheel(path: Path, body: str) -> None: + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("executorch/version.py", body) + + +def test_wheel_git_version_reads_the_recorded_commit(monkeypatch, tmp_path) -> None: + commit = "deadbeef" * 5 + wheel = tmp_path / "executorch-1.5.0.dev20200103-py3-none-any.whl" + _synthesize_wheel( + wheel, f'__version__ = "1.5.0.dev20200103"\ngit_version = "{commit}"\n' + ) + + def fake_download(cmd): + # The download call writes the wheel into the temp dir named right after --dest. + dest = Path(cmd[cmd.index("--dest") + 1]) + (dest / wheel.name).write_bytes(wheel.read_bytes()) + return "" + + monkeypatch.setattr(updater, "_run", fake_download) + assert updater.wheel_git_version("1.5.0.dev20200103", []) == commit + + +def test_wheel_git_version_rejects_a_wheel_without_provenance( + monkeypatch, tmp_path +) -> None: + # ExecuTorch writes git_version = None when built outside a git checkout. Such a wheel + # carries nothing the pins can be checked against, so it must not become a pin. + wheel = tmp_path / "executorch-1.5.0.dev20200103-py3-none-any.whl" + _synthesize_wheel(wheel, "__version__ = '1.5.0.dev20200103'\ngit_version = None\n") + + def fake_download(cmd): + dest = Path(cmd[cmd.index("--dest") + 1]) + (dest / wheel.name).write_bytes(wheel.read_bytes()) + return "" + + monkeypatch.setattr(updater, "_run", fake_download) + with pytest.raises(SystemExit): + updater.wheel_git_version("1.5.0.dev20200103", []) + + +def test_upper_bound_is_the_next_minor_of_the_line() -> None: + # A nightly and a final both belong to the release line their first two fields name, so + # the bound is the next minor either way. This mirrors the guard's own derivation, and + # the 0.9 case checks the minor rolls to a two-digit number rather than to "0.:". + assert updater._upper_bound("1.5.0.dev20200103") == "1.6" + assert updater._upper_bound("0.9.1") == "0.10" + assert updater._upper_bound("7.3.0") == "7.4" + + +def _worktree(tmp_path) -> Path: + """A throwaway checkout of the repo so write_pins edits a real tree, not a copy that + diverges from what git tracks. write_pins walks `git ls-files`, so the tree has to be a + real checkout.""" + work = tmp_path / "repo" + subprocess.run( + ["git", "clone", "--quiet", "--no-hardlinks", str(_REPO_ROOT), str(work)], + check=True, + ) + return work + + +# A synthetic bump target for the write_pins tests: a fake far-past nightly and an obvious +# marker commit, distinct from any live pin so the bump is a real change but never collides +# with a real value in the tree. +_FAKE_VERSION = "1.5.0.dev20200103" +_FAKE_COMMIT = "deadbeef" * 5 + + +def test_write_pins_leaves_a_tree_the_guard_accepts(tmp_path, monkeypatch) -> None: + # The one test that matters most: after a bump, the whole pin guard has to pass, because + # that guard is what the generated pull request will be judged by. A rewrite that the + # guard rejects would open a red pull request every night. + work = _worktree(tmp_path) + monkeypatch.setattr(updater, "_REPO_ROOT", work) + monkeypatch.setattr(updater, "_VERSIONS_FILE", work / "dev_dep_versions.yml") + + assert updater.write_pins(_FAKE_VERSION, _FAKE_COMMIT) is True + assert updater.read_pin("__executorch_version__") == _FAKE_VERSION + assert updater.read_pin("__executorch_commit__") == _FAKE_COMMIT + + import sys + + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "tests/py/dynamo/executorch/test_executorch_pin.py", + "-q", + "--no-header", + "-p", + "no:cacheprovider", + "--noconftest", + "-o", + "addopts=", + ], + cwd=work, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_write_pins_is_idempotent_when_already_current(tmp_path, monkeypatch) -> None: + # A day with no new nightly must rewrite nothing, so the run opens no pull request. The + # updater is safe to run every day and by hand. + work = _worktree(tmp_path) + monkeypatch.setattr(updater, "_REPO_ROOT", work) + monkeypatch.setattr(updater, "_VERSIONS_FILE", work / "dev_dep_versions.yml") + + current_version = updater.read_pin("__executorch_version__") + current_commit = updater.read_pin("__executorch_commit__") + assert updater.write_pins(current_version, current_commit) is False + + diff = subprocess.run( + ["git", "-C", str(work), "diff", "--quiet"], capture_output=True + ) + assert diff.returncode == 0, "write_pins changed the tree when the pin was current" + + +def test_write_pins_updates_new_sites_but_never_its_own_source( + tmp_path, monkeypatch +) -> None: + # The updater rewrites by content, not a fixed list, so a brand new pin site is caught + # with no code change. The flip side is that a real pin literal in the updater's own + # source or test would be rewritten too. Both properties are pinned here: a new site is + # updated, and the updater's two files come out byte for byte identical. + work = _worktree(tmp_path) + monkeypatch.setattr(updater, "_REPO_ROOT", work) + monkeypatch.setattr(updater, "_VERSIONS_FILE", work / "dev_dep_versions.yml") + + old_version = updater.read_pin("__executorch_version__") + new_site = work / "some_new_requirements.txt" + new_site.write_text(f"executorch=={old_version}\n") + subprocess.run(["git", "-C", str(work), "add", str(new_site)], check=True) + + script = work / ".github" / "scripts" / "update_executorch_pin.py" + test = work / _SELF + script_before = script.read_bytes() + test_before = test.read_bytes() + + assert updater.write_pins(_FAKE_VERSION, _FAKE_COMMIT) is True + + assert new_site.read_text() == f"executorch=={_FAKE_VERSION}\n" + assert script.read_bytes() == script_before + assert test.read_bytes() == test_before diff --git a/toolchains/ci_workspaces/MODULE.bazel.tmpl b/toolchains/ci_workspaces/MODULE.bazel.tmpl index 348ff0351b..124e03199d 100644 --- a/toolchains/ci_workspaces/MODULE.bazel.tmpl +++ b/toolchains/ci_workspaces/MODULE.bazel.tmpl @@ -214,8 +214,8 @@ new_local_repository( new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.5.0.dev20260822 - commit = "b575a5bc2eab6d671197bc7a920edb7fd0b8fbb7", + # executorch==1.5.0.dev20260825 + commit = "817929b7fb8d162d80eb9299d6630c35a3106979", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;",