diff --git a/.github/scripts/install-torch-tensorrt.sh b/.github/scripts/install-torch-tensorrt.sh index 67a3c9fe29..7141b31f3b 100755 --- a/.github/scripts/install-torch-tensorrt.sh +++ b/.github/scripts/install-torch-tensorrt.sh @@ -52,9 +52,24 @@ fi # Install Torch-TensorRT if [[ ${PLATFORM} == win32 ]]; then + # pin-check: no-nightly -- this glob also matches the Linux-only ExecuTorch runtime wheel, but + # ExecuTorch publishes no win32 nightly and the [executorch] extra is Linux-only, so this + # platform's plain torch-tensorrt install needs no nightly index. python -m pip install ${RUNNER_ARTIFACT_DIR}/torch_tensorrt*.whl else - python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl --use-deprecated=legacy-resolver + # The nightly channel is needed because this glob also matches the ExecuTorch runtime wheel, + # whose install_requires names an ExecuTorch dev build that is published only there. + # Hardcoded rather than ${CHANNEL} like the lines above: a .dev wheel exists on no other + # channel, so deriving it would break this install on exactly the test and release runs the + # index was added for. It is an extra index, not a replacement, and torch is already + # force-reinstalled from ${INDEX_URL} above, so the pinned torch is not at risk from it. + # || exit 1 because line 1's `set -exou pipefail` is commented out and linux-test.yml + # concatenates this file ahead of the user script, so a failure here would otherwise be + # discarded and the job would die later with an unrelated-looking ImportError. Scoped to the + # line this change is responsible for; re-enabling set -e for the whole file is a + # pre-existing hazard worth a separate change. + python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl --use-deprecated=legacy-resolver \ + --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" || exit 1 fi echo -e "Running test script"; diff --git a/.github/scripts/update_executorch_pin.py b/.github/scripts/update_executorch_pin.py new file mode 100644 index 0000000000..5f8dfbc5f8 --- /dev/null +++ b/.github/scripts/update_executorch_pin.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Move the ExecuTorch pin to the newest wheel published on an index. + +The pin is two coupled facts, spread across the tree but sourced from +``dev_dep_versions.yml``: ``__executorch_version__`` selects the wheel the delegate is +built to sit beside, and ``__executorch_commit__`` selects the tree it compiles from. +They must name one ExecuTorch, so this script never guesses the commit: it reads it from +the chosen wheel's own ``executorch/version.py``, which is the same provenance +``tests/py/dynamo/executorch/test_executorch_pin.py`` checks the pins against. + +The daily workflow runs this, then opens a pull request when the pin moved. The existing +pin guards and the executorch end-to-end lane run on that pull request, so "the newest +wheel that actually works" is decided by the same gate a human bump goes through, not +re-implemented here. A nightly that did not publish leaves the newest version unchanged, +so the run rewrites nothing and opens nothing. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path + +from packaging.version import InvalidVersion, Version + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_VERSIONS_FILE = _REPO_ROOT / "dev_dep_versions.yml" + +# The pin literal is rewritten everywhere it appears except this file, which carries the +# version only as a docstring example, not a pin. Every real site is checked by the guard, +# so an accidental new site fails that test rather than being silently missed here. +_SELF = "tests/py/dynamo/executorch/test_executorch_pin.py" + +# A wheel version on the PyTorch index carries a local label naming its CUDA build, for +# example ``1.5.0.devYYYYMMDD+cu130``. The pin omits it so one pin serves every CUDA row. +_LOCAL_LABEL = re.compile(r"\+.*$") + + +def _run(cmd: list[str]) -> str: + return subprocess.run(cmd, check=True, capture_output=True, text=True).stdout + + +def read_pin(field: str) -> str: + """Return a pinned value from ``dev_dep_versions.yml``.""" + text = _VERSIONS_FILE.read_text(encoding="utf-8") + match = re.search(rf'^{field}:\s*"?([^"\s]+)"?\s*$', text, re.MULTILINE) + if match is None: + raise SystemExit(f"{field} is not set in {_VERSIONS_FILE.name}") + return match.group(1) + + +def available_versions(index_args: list[str]) -> list[str]: + """Every ExecuTorch version the index offers. + + ``pip index versions`` prints one ``Available versions:`` line. Parsing that is stable + and needs no network code here; the workflow passes the index the same way every other + install in the tree does. + """ + out = _run( + [sys.executable, "-m", "pip", "index", "versions", "executorch", *index_args] + ) + match = re.search(r"^\s*Available versions:\s*(.+)$", out, re.MULTILINE) + if match is None: + raise SystemExit("pip index versions printed no Available versions line") + return [v.strip() for v in match.group(1).split(",") if v.strip()] + + +def pick_target(versions: list[str], track: str) -> str: + """The newest version on the wanted track. + + ``nightly`` takes the newest dated dev build; ``stable`` takes the newest final + release, ignoring dev builds and release candidates. Ordering is PEP 440, not string + order, so a newer dev date on the same line sorts above an older one correctly. + """ + parsed: list[tuple[Version, str]] = [] + for raw in versions: + try: + version = Version(raw) + except InvalidVersion: + continue + # A nightly is a dated dev build. is_prerelease is also true for release + # candidates, and an rc sorts above every dev of the same line under PEP 440, so + # filtering on it would let the first rc on the index silently become the nightly + # pin. Match the dev segment itself instead. + if track == "nightly" and version.dev is None: + continue + if track == "stable" and (version.is_prerelease or version.is_devrelease): + continue + parsed.append((version, raw)) + if not parsed: + raise SystemExit(f"no executorch version on the index matches track {track!r}") + newest = max(parsed, key=lambda pair: pair[0])[1] + return _LOCAL_LABEL.sub("", newest) + + +def wheel_git_version(version: str, index_args: list[str]) -> str: + """The source commit the chosen wheel records for itself. + + Every published wheel writes ``git_version`` into ``executorch/version.py``. Reading it + from the wheel is what keeps the two pins naming one ExecuTorch. A wheel built without + git provenance records ``None`` and must not become a pin, so that is an error, not a + guess. + """ + with tempfile.TemporaryDirectory() as tmp: + _run( + [ + sys.executable, + "-m", + "pip", + "download", + "--no-deps", + "--dest", + tmp, + f"executorch=={version}", + *index_args, + ] + ) + wheels = list(Path(tmp).glob("executorch-*.whl")) + if not wheels: + raise SystemExit( + f"pip download produced no wheel for executorch=={version}" + ) + with zipfile.ZipFile(wheels[0]) as archive: + source = archive.read("executorch/version.py").decode("utf-8") + match = re.search(r"""git_version[^=]*=\s*['"]([0-9a-f]{40})['"]""", source) + if match is None: + raise SystemExit( + f"executorch=={version} records no source commit, so the pin would name a " + "wheel whose provenance cannot be checked" + ) + return match.group(1) + + +def _upper_bound(version: str) -> str: + """The exclusive upper bound a range site pairs with the pin, next minor of its line. + + ``tests/py/dynamo/executorch/test_executorch_pin.py`` derives the same bound from the + same two fields, so a range this writes and the range the guard expects agree by the + same rule rather than by coincidence. + """ + major, minor = version.split(".")[:2] + return f"{major}.{int(minor) + 1}" + + +def _iter_tracked_files() -> list[Path]: + names = _run(["git", "-C", str(_REPO_ROOT), "ls-files"]).splitlines() + return [_REPO_ROOT / name for name in names if name != _SELF] + + +def write_pins(new_version: str, new_commit: str) -> bool: + """Rewrite every pin occurrence to the new version and commit. + + The old version and commit are unique, high-entropy tokens, so a literal replacement + cannot touch anything that is not already a pin. The range form is re-rendered after, + so its upper bound tracks the new line. Returns whether anything changed. + """ + old_version = read_pin("__executorch_version__") + old_commit = read_pin("__executorch_commit__") + if (new_version, new_commit) == (old_version, old_commit): + return False + + # Anchored on >= and ,< rather than on a package name, so this rewrites the range + # wherever it appears: the executorch>=X,={old_version},<{_upper_bound(old_version)}" + new_range = f">={new_version},<{_upper_bound(new_version)}" + + changed = False + for path in _iter_tracked_files(): + try: + text = path.read_text(encoding="utf-8") + except (UnicodeDecodeError, FileNotFoundError): + continue + if old_version not in text and old_commit not in text: + continue + # The range carries the old version too, so rewrite it first, then the remaining + # bare version occurrences, so the range's own version is not rewritten twice. + updated = text.replace(old_range, new_range) + updated = updated.replace(old_version, new_version) + updated = updated.replace(old_commit, new_commit) + if updated != text: + path.write_text(updated, encoding="utf-8") + changed = True + + if read_pin("__executorch_version__") != new_version: + raise SystemExit( + "the version pin did not take; dev_dep_versions.yml is unchanged" + ) + return changed + + +def _index_args(track: str, channel: str) -> list[str]: + if track == "nightly": + return [ + "--pre", + "--index-url", + f"https://download.pytorch.org/whl/nightly/{channel}", + ] + return [] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--track", choices=("nightly", "stable"), default="nightly") + parser.add_argument( + "--channel", + default="cu130", + help="nightly CUDA channel to read versions and provenance from", + ) + args = parser.parse_args(argv) + + index_args = _index_args(args.track, args.channel) + target = pick_target(available_versions(index_args), args.track) + current = read_pin("__executorch_version__") + if target == current: + print(f"executorch pin is already at the newest {args.track} version {current}") + return 0 + + commit = wheel_git_version(target, index_args) + if write_pins(target, commit): + print(f"moved executorch pin {current} -> {target} (commit {commit})") + else: + print("nothing to write") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/verify-executorch-reference-runner.sh b/.github/scripts/verify-executorch-reference-runner.sh index fe8e98d3de..d032662926 100755 --- a/.github/scripts/verify-executorch-reference-runner.sh +++ b/.github/scripts/verify-executorch-reference-runner.sh @@ -18,6 +18,11 @@ set +x # examples/torchtrt_executorch_example/export_kv_cache_decode.py). When given, # kv_cache_decode_check is built and run against it as well. # +# Optional third argument: path to a coalesced TensorRT + CUDA .pte (see +# examples/torchtrt_executorch_example/export_coalesced.py). When given, both +# runners are run against it and their output is compared to the eager +# reference that export script wrote next to the model. +# # Optional: # TensorRT_ROOT=/path/to/extracted/TensorRT # If unset, the script reuses Bazel's fetched TensorRT SDK when available @@ -33,8 +38,8 @@ set +x repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "${repo_root}" -if [[ $# -lt 1 || $# -gt 2 ]]; then - echo "Usage: $0 PATH_TO_MODEL.pte [PATH_TO_KV_CACHE_DECODE.pte]" >&2 +if [[ $# -lt 1 || $# -gt 3 ]]; then + echo "Usage: $0 PATH_TO_MODEL.pte [PATH_TO_KV_CACHE_DECODE.pte [PATH_TO_COALESCED.pte]]" >&2 exit 1 fi model_path="$1" @@ -47,6 +52,11 @@ if [[ -n "${kv_model_path}" && ! -f "${kv_model_path}" ]]; then echo "KV-cache decode model not found: ${kv_model_path}" >&2 exit 1 fi +coalesced_model_path="${3:-}" +if [[ -n "${coalesced_model_path}" && ! -f "${coalesced_model_path}" ]]; then + echo "Coalesced model not found: ${coalesced_model_path}" >&2 + exit 1 +fi python_executable="${PYTHON_EXECUTABLE:-}" if [[ -z "${python_executable}" ]]; then @@ -467,25 +477,35 @@ packaged_runner_log="${verify_root}/packaged_runner.log" --model_path="${model_path}" \ --num_runs=1 2>&1 | tee "${packaged_runner_log}" -# The sample model is x + 1 on a (2,3,4,4) input and both runners fill inputs with -# 1.0f, so the shape is exactly [2,3,4,4] and every printed value is exactly 2.0000. -# Assert both precisely. Matching only "shape=" accepts any shape, and matching one -# 2.0000 anywhere on the values line accepts a line of wrong numbers that happens to -# contain one right one, so neither catches a stream-ordering regression returning -# stale or partial output. Both lines come from fprintf in the runner, so these -# assertions hold whatever ET_LOG_ENABLED is set to. -for _log in "${runner_log}" "${packaged_runner_log}"; do - # A right answer produced entirely on the host would not prove much here: the - # program is delegated to TensorRT, so at least one planned buffer has to be - # served by a registered CUDA DeviceAllocator. Pin that, otherwise a model or - # a planning change could quietly turn this into a CPU-only test. +# Assert the printed shape, and that EVERY value on the "first N values:" line is +# the expected one. Matching only "shape=" accepts any shape, and matching one +# right value anywhere on the values line accepts a line of wrong numbers that +# happens to contain one, so neither on its own catches a stream-ordering +# regression returning stale or partial output. Both lines come from fprintf in +# the runner, so these assertions hold whatever ET_LOG_ENABLED is set to. The +# models used here are elementwise on an all-ones input, so one number describes +# the whole expected output. +assert_runner_output() { + local _log="$1" + local _shape="$2" + local _expected="$3" + local _tolerance="$4" + local _values + local _value + + # A right answer produced entirely on the host would not prove much: the + # programs here are delegated, so at least one planned buffer has to be served + # by a registered CUDA DeviceAllocator. Pin that, otherwise a model or a + # planning change could quietly turn this into a CPU-only test. if ! grep -q 'planned buffer\[[0-9]*\] = [0-9]* bytes on device_type 1' "${_log}"; then echo "No CUDA planned buffer was allocated in ${_log}:" >&2 grep 'planned buffer' "${_log}" >&2 || echo " no planned buffer line at all" >&2 exit 1 fi - if ! grep -q 'output\[0\] shape=\[2,3,4,4\]' "${_log}"; then + # -F: the shape is bracketed, and an unescaped [2,3,4,4] is a regex character + # class that would match any single one of those characters. + if ! grep -qF "output[0] shape=${_shape}" "${_log}"; then echo "Unexpected output shape in ${_log}:" >&2 grep 'output\[0\] shape=' "${_log}" >&2 || echo " no shape line at all" >&2 exit 1 @@ -498,11 +518,19 @@ for _log in "${runner_log}" "${packaged_runner_log}"; do exit 1 fi for _value in ${_values}; do - if [[ "${_value}" != "2.0000" ]]; then - echo "Unexpected output value '${_value}' in ${_log}: ${_values}" >&2 + if ! awk -v got="${_value}" -v want="${_expected}" -v tol="${_tolerance}" \ + 'BEGIN { d = got - want; if (d < 0) d = -d; exit !(d <= tol) }'; then + echo "Unexpected output value '${_value}' in ${_log}" \ + "(expected ${_expected} within ${_tolerance}): ${_values}" >&2 exit 1 fi done +} + +# The sample model is x + 1 on a (2,3,4,4) input, so every output value is exactly +# 2. That is exact in float32, hence a zero tolerance. +for _log in "${runner_log}" "${packaged_runner_log}"; do + assert_runner_output "${_log}" "[2,3,4,4]" "2.0000" 0 done if [[ -n "${kv_model_path}" ]]; then @@ -521,3 +549,37 @@ if [[ -n "${kv_model_path}" ]]; then "${kv_check_path}" --model_path="${kv_model_path}" 2>&1 | tee "${kv_check_log}" grep -q "PASS: decode at pos=1 observed the KV written at pos=0" "${kv_check_log}" fi + +if [[ -n "${coalesced_model_path}" ]]; then + # A coalesced program splits one graph across the TensorRT delegate and + # ExecuTorch's CUDA delegate, so a value produced by one delegate is consumed by + # the other on the device, inside one method. The checks above use a program with + # a single delegate, so they exercise neither the second backend nor the handover + # between the two. + coalesced_expected_path="${coalesced_model_path%.pte}.expected" + if [[ ! -f "${coalesced_expected_path}" ]]; then + echo "Coalesced reference output not found: ${coalesced_expected_path}" >&2 + echo "It is written by examples/torchtrt_executorch_example/export_coalesced.py" >&2 + exit 1 + fi + coalesced_shape="$(sed -n '1p' "${coalesced_expected_path}")" + coalesced_value="$(sed -n '2p' "${coalesced_expected_path}")" + if [[ -z "${coalesced_shape}" || -z "${coalesced_value}" ]]; then + echo "Malformed coalesced reference output in ${coalesced_expected_path}" >&2 + exit 1 + fi + + # Only the from-source runner runs the coalesced program. It links every + # ExecuTorch delegate, including the CUDA/AOTI backend that the CUDA partition + # of a coalesced .pte is handed to. The packaged runner in the wheel ships the + # TensorRT delegate alone, so it has no CudaBackend to run that partition and + # cannot execute this model. + coalesced_runner_log="${verify_root}/coalesced_my_runner.log" + "${runner_path}" \ + --model_path="${coalesced_model_path}" \ + --num_runs=1 2>&1 | tee "${coalesced_runner_log}" + + # TensorRT, AOTInductor and eager PyTorch compute the same math with different + # kernels, so compare within a tolerance instead of on the printed digits. + assert_runner_output "${coalesced_runner_log}" "${coalesced_shape}" "${coalesced_value}" 0.001 +fi diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index fb390df6ea..d31038762b 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -42,7 +42,12 @@ jobs: run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: Build Python Package run: | - python3 -m pip install --pre ".[executorch]" --extra-index-url https://download.pytorch.org/whl/nightly/cu130 + # The pin exactly, not the range the extra expands to: --pre plus a nightly + # channel that gains a member daily would otherwise install whichever dev build + # is newest that morning while the delegate compiles from the pinned commit. + python3 -m pip install --pre ".[executorch]" \ + "executorch==$(python3 -c 'import yaml;print(yaml.safe_load(open("dev_dep_versions.yml"))["__executorch_version__"])')" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 - name: Install uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/.github/workflows/executorch-build-linux.yml b/.github/workflows/executorch-build-linux.yml index f3eab9c237..8fbdba8804 100644 --- a/.github/workflows/executorch-build-linux.yml +++ b/.github/workflows/executorch-build-linux.yml @@ -79,7 +79,13 @@ jobs: export PATH="${RUNNER_TEMP}/bin:${PATH}" bazel --version - python -m pip install pyyaml "executorch==1.4.1" + # ExecuTorch's CUDA wheels are published only on the PyTorch nightly index, so the + # channel is needed. No --pre: a specifier naming a prerelease admits prereleases by + # itself, and --pre would apply to every other requirement in the same command too. + # CU_VERSION selects the row's own channel, which is what keeps the runtime the + # delegate links to the same CUDA build as the rest of the job. + EXECUTORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/${CU_VERSION}" + python -m pip install pyyaml --extra-index-url "${EXECUTORCH_INDEX_URL}" "executorch==1.5.0.dev20260825" export TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION="$(python -c 'import importlib.metadata; print(importlib.metadata.version("torch-tensorrt"))')" # The downloaded wheel has to carry the C++ runtime. A wheel built with @@ -125,8 +131,9 @@ jobs: executorch_cmake_location="$(bazel query @executorch//:executorch/CMakeLists.txt --output=location)" export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" - # this is to verify the end user's workflow - python -m pip install pyyaml "executorch>=1.4.1,<1.5" + # pin-check: range-ok -- this is to verify the end user's workflow, which resolves a + # range the way a user would rather than the exact artifact the delegate links. + python -m pip install pyyaml --extra-index-url "${EXECUTORCH_INDEX_URL}" "executorch>=1.5.0.dev20260825,<1.6" python examples/torchtrt_executorch_example/export_static_shape.py \ --model_path="${RUNNER_TEMP}/torchtrt-reference-runner.pte" .github/scripts/verify-executorch-reference-runner.sh \ diff --git a/.github/workflows/executorch-pin-update.yml b/.github/workflows/executorch-pin-update.yml new file mode 100644 index 0000000000..e4a87a5226 --- /dev/null +++ b/.github/workflows/executorch-pin-update.yml @@ -0,0 +1,105 @@ +name: Update ExecuTorch pin + +# Keep the ExecuTorch pin close to upstream without a human running the bump by hand. On +# main this tracks the nightly line every day; the newest wheel the index actually carries +# is by definition the newest one that built, so a failed nightly simply leaves the pin +# where it was. The bump lands as a pull request, not a direct push, so the existing pin +# guards and the executorch end-to-end lane decide whether the new wheel is usable before +# it reaches main. +# +# On a release branch the schedule is a no-op: a release pins a stable ExecuTorch and does +# not drift. Re-pinning there is a manual dispatch with track=stable, which is the only way +# the pin moves once a branch is cut. + +on: + schedule: + # A few hours after the nightly index publishes, so the freshest wheel is available. + - cron: "0 13 * * *" + workflow_dispatch: + inputs: + track: + description: "Which ExecuTorch line to pin to" + type: choice + default: nightly + options: + - nightly + - stable + +permissions: + contents: write + pull-requests: write + +jobs: + update-pin: + runs-on: ubuntu-latest + if: ${{ ! contains(github.actor, 'pytorchbot') }} + environment: pytorchbot-env + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + token: ${{ secrets.GH_PYTORCHBOT_TOKEN }} + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install packaging + run: python -m pip install packaging + + - name: Choose the track + id: track + run: | + set -euo pipefail + # A manual run pins whatever it asked for. The daily schedule pins the nightly + # line on main and does nothing on a release branch, so a release never drifts + # onto a nightly. + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "track=${{ inputs.track }}" >> "$GITHUB_OUTPUT" + elif [ "${{ github.ref }}" = "refs/heads/main" ]; then + echo "track=nightly" >> "$GITHUB_OUTPUT" + else + echo "no scheduled pin update on ${{ github.ref }}; release pins do not drift" + echo "track=" >> "$GITHUB_OUTPUT" + fi + + - name: Update the pin + id: update + if: steps.track.outputs.track != '' + run: | + set -euo pipefail + python .github/scripts/update_executorch_pin.py --track "${{ steps.track.outputs.track }}" + if git diff --quiet; then + echo "the pin is already at the newest ${{ steps.track.outputs.track }} wheel" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "version=$(sed -n 's/^__executorch_version__: "\(.*\)"$/\1/p' dev_dep_versions.yml)" >> "$GITHUB_OUTPUT" + fi + + - name: Open a pull request + if: steps.update.outputs.changed == 'true' + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ secrets.GH_PYTORCHBOT_TOKEN }} + branch: executorch-pin-update/${{ steps.track.outputs.track }} + delete-branch: true + commit-message: "Update ExecuTorch pin to ${{ steps.update.outputs.version }}" + title: "Update ExecuTorch pin to ${{ steps.update.outputs.version }}" + # The delegate is built from the tree the pinned wheel was built from, so both + # pins move together to the version and the commit that wheel records for itself. + # The pin guards and the executorch end-to-end lane gate this pull request. + body: | + Move the ExecuTorch pin to `${{ steps.update.outputs.version }}`, the newest + ${{ steps.track.outputs.track }} wheel on the index. The source commit is read + from that wheel, so the version pin and the source commit name one ExecuTorch. + + Opened automatically. The pin consistency checks and the ExecuTorch delegate + build and test lane run here and decide whether this wheel is usable. + committer: Torch-TensorRT Github Bot + author: Torch-TensorRT Github Bot + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true diff --git a/.github/workflows/executorch-test-linux.yml b/.github/workflows/executorch-test-linux.yml index 6d38fff9bb..eb6b6de824 100644 --- a/.github/workflows/executorch-test-linux.yml +++ b/.github/workflows/executorch-test-linux.yml @@ -64,7 +64,12 @@ jobs: chmod +x "${RUNNER_TEMP}/bin/bazel" export PATH="${RUNNER_TEMP}/bin:${PATH}" - python -m pip install pyyaml "executorch==1.4.1" + # ExecuTorch's CUDA wheels live only on the PyTorch nightly index, so the channel is + # needed. No --pre: a specifier naming a prerelease admits prereleases by itself, and + # --pre would apply to every other requirement in the same command too. + python -m pip install pyyaml \ + --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" \ + "executorch==1.5.0.dev20260825" # Run the check directly so its exit status is the step's exit status. # Wrapping it in `gdb --batch` reports gdb's own status, which is 0 # whatever the program does, so a SIGSEGV here was passing. @@ -95,8 +100,11 @@ jobs: --model_path="${RUNNER_TEMP}/torchtrt-python.pte" python examples/torchtrt_executorch_example/export_kv_cache_decode.py \ --model_path="${RUNNER_TEMP}/torchtrt-kv-cache-decode.pte" + python examples/torchtrt_executorch_example/export_coalesced.py \ + --model_path="${RUNNER_TEMP}/torchtrt-coalesced.pte" .github/scripts/verify-executorch-reference-runner.sh \ "${RUNNER_TEMP}/torchtrt-python.pte" \ - "${RUNNER_TEMP}/torchtrt-kv-cache-decode.pte" + "${RUNNER_TEMP}/torchtrt-kv-cache-decode.pte" \ + "${RUNNER_TEMP}/torchtrt-coalesced.pte" python examples/executorch_reference_runner/load_model.py \ --model_path="${RUNNER_TEMP}/torchtrt-python.pte" --num_runs=1 diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index af16185129..4fc43393c9 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -82,9 +82,34 @@ jobs: run: | uv pip install --system -r $GITHUB_WORKSPACE/.github/scripts/requirements.txt python3 -c "import tomllib, subprocess; deps = tomllib.load(open('pyproject.toml', 'rb'))['dependency-groups']['lint']; subprocess.run(['uv', 'pip', 'install', '--system'] + deps, check=True)" + # The pin check below runs under pytest and imports yaml to parse the workflows it + # asserts about. Neither pytest nor pyyaml is in requirements.txt or + # dependency-groups.lint, so the step exited 1 without running. + # test_update_executorch_pin imports packaging to order versions the way pip does. + uv pip install --system pytest pyyaml packaging - name: Lint Python run: | cd $GITHUB_WORKSPACE python3 $GITHUB_WORKSPACE/.github/scripts/run_py_linter.py env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + # Text and metadata only: no GPU, no ExecuTorch, no built wheel. It belongs on this runner + # rather than in the tests/ci manifest, where every suite becomes a CUDA-container job + # after `needs: build` -- one GPU row per python/CUDA combination for a check that takes + # seconds. --noconftest keeps it clear of tests/py/dynamo/conftest.py, which imports torch. + - name: Check the ExecuTorch pin is consistent + if: always() + run: | + cd $GITHUB_WORKSPACE + python3 -m pytest tests/py/dynamo/executorch/test_executorch_pin.py \ + -q --no-header -p no:cacheprovider --noconftest -o addopts="" + # The pin updater is pure repository tooling: it reads an index, rewrites the pin sites, + # and its tests clone the tree and shell out to pytest. No GPU or ExecuTorch, so it runs + # here beside the pin check rather than on a CUDA runner, and is excluded from the + # executorch e2e suite so it does not run there too. + - name: Test the ExecuTorch pin updater + if: always() + run: | + cd $GITHUB_WORKSPACE + python3 -m pytest tests/py/dynamo/executorch/test_update_executorch_pin.py \ + -q --no-header -p no:cacheprovider --noconftest -o addopts="" diff --git a/MODULE.bazel b/MODULE.bazel index 6ae9a88573..f6e6e2a6be 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -45,13 +45,16 @@ new_git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl" local_torch = use_repo_rule("//toolchains:local_torch.bzl", "local_torch") -# Keep this pin synchronized with the ExecuTorch release installed in -# py/torch-tensorrt-executorch-runtime/README.md. +# This commit must be the one the pinned ExecuTorch wheel was built from, because the delegate +# compiles headers from this tree and links the runtime out of that wheel. Every wheel records +# its source in executorch/version.py as git_version, and tests/py/dynamo/executorch/ +# test_executorch_pin.py checks the two agree wherever the pinned wheel is the one installed, +# so bump both pins together. new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.4.1 - commit = "e4d02f41f7909e8ed5bf4a14ffc520d733453d9f", + # executorch==1.5.0.dev20260825 + commit = "817929b7fb8d162d80eb9299d6630c35a3106979", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;", diff --git a/dev_dep_versions.yml b/dev_dep_versions.yml index 07bac57849..d32783a75c 100644 --- a/dev_dep_versions.yml +++ b/dev_dep_versions.yml @@ -2,5 +2,5 @@ __cuda_version__: "13.2" __tensorrt_version__: "11.2.1" __tensorrt_rtx_version__: "1.6.1" __tensorrt_llm_version__: "0.17.0.post1" -__executorch_version__: "1.4.1" -__executorch_commit__: "e4d02f41f7909e8ed5bf4a14ffc520d733453d9f" +__executorch_version__: "1.5.0.dev20260825" +__executorch_commit__: "817929b7fb8d162d80eb9299d6630c35a3106979" diff --git a/docker/MODULE.bazel.docker b/docker/MODULE.bazel.docker index f0c9b161bb..ce932f8f65 100644 --- a/docker/MODULE.bazel.docker +++ b/docker/MODULE.bazel.docker @@ -67,8 +67,8 @@ new_local_repository( new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.4.1 - commit = "e4d02f41f7909e8ed5bf4a14ffc520d733453d9f", + # executorch==1.5.0.dev20260825 + commit = "817929b7fb8d162d80eb9299d6630c35a3106979", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;", diff --git a/docker/MODULE.bazel.ngc b/docker/MODULE.bazel.ngc index fcf70cafa8..b01e9a6654 100644 --- a/docker/MODULE.bazel.ngc +++ b/docker/MODULE.bazel.ngc @@ -76,8 +76,8 @@ new_local_repository( new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.4.1 - commit = "e4d02f41f7909e8ed5bf4a14ffc520d733453d9f", + # executorch==1.5.0.dev20260825 + commit = "817929b7fb8d162d80eb9299d6630c35a3106979", recursive_init_submodules = True, patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", diff --git a/docsrc/user_guide/runtime_performance/saving_models.rst b/docsrc/user_guide/runtime_performance/saving_models.rst index 6470afd72e..76aec91564 100644 --- a/docsrc/user_guide/runtime_performance/saving_models.rst +++ b/docsrc/user_guide/runtime_performance/saving_models.rst @@ -227,8 +227,11 @@ c) ExecuTorch (.pte) The ``executorch`` output format lowers the compiled module to an ExecuTorch ``.pte`` program, delegating the TensorRT engines to the Torch-TensorRT ExecuTorch -backend. It requires the ``executorch`` package (``pip install -"torch_tensorrt[executorch]"``) and is Linux-only. +backend. It requires the ``executorch`` package, from the PyTorch nightly index +(``pip install --pre "torch_tensorrt[executorch]" --extra-index-url +https://download.pytorch.org/whl/nightly/cu130``), and is Linux-only. Add ``--upgrade`` +if a stable ``torch-tensorrt`` is already installed, or pip keeps it and reports that it +does not provide the ``executorch`` extra. There are two ways to produce a ``.pte``, and they suit different needs: diff --git a/examples/executorch_reference_runner/README.md b/examples/executorch_reference_runner/README.md index 1b1ccba454..ef3eaf8baa 100644 --- a/examples/executorch_reference_runner/README.md +++ b/examples/executorch_reference_runner/README.md @@ -44,7 +44,7 @@ torch_tensorrt/bin/example_executorch_runner ```bash # Get the ExecuTorch source snapshot this package is built against. Keep this in sync # with the executorch commit pinned in MODULE.bazel. -EXECUTORCH_REF="${EXECUTORCH_REF:-e4d02f41f7909e8ed5bf4a14ffc520d733453d9f}" +EXECUTORCH_REF="${EXECUTORCH_REF:-817929b7fb8d162d80eb9299d6630c35a3106979}" git clone --filter=blob:none --no-checkout \ https://github.com/pytorch/executorch.git executorch pushd executorch @@ -95,13 +95,26 @@ build-executorch-reference-runner/lib/libexecutorch_trt_backend.a ### Python -Install the complete prebuilt Python runtime and delegate: +Install the `executorch` authoring stack, which the `[executorch]` extra pulls in: ```bash -pip install "torch-tensorrt[executorch]" +pip install --pre "torch-tensorrt[executorch]" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 ``` -Load and run the model without an ExecuTorch checkout or native build: +The index is required, not optional: the extra's ExecuTorch floor names a dev build, and PyPI's +`executorch` stops below it, so without the nightly channel pip reports no matching distribution. +If a stable `torch-tensorrt` is already installed, add `--upgrade`, or pip keeps it and reports +that it does not provide the `executorch` extra. + +The extra installs `executorch` only. The delegate runtime, +`torch-tensorrt-executorch-runtime`, is not yet published to any index: its requirement in the +top-level `setup.py` is commented out for that reason. Build and install it from source following +`py/torch-tensorrt-executorch-runtime/README.md`. That wheel contains an ExecuTorch Python runtime +with `TensorRTBackend` linked into its backend registry, and loading a `.pte` through the delegate +needs it. + +Then load and run the model: ```bash python examples/executorch_reference_runner/load_model.py \ @@ -109,10 +122,6 @@ python examples/executorch_reference_runner/load_model.py \ --num_runs=1 ``` -The extra installs `executorch` and the matching -`torch-tensorrt-executorch-runtime` wheel. That wheel contains an ExecuTorch -Python runtime with `TensorRTBackend` linked into its backend registry. - ### C++ Run the reference runner against a Torch-TensorRT compiled ExecuTorch model: diff --git a/examples/torchtrt_executorch_example/export_coalesced.py b/examples/torchtrt_executorch_example/export_coalesced.py new file mode 100644 index 0000000000..ebbd4734e4 --- /dev/null +++ b/examples/torchtrt_executorch_example/export_coalesced.py @@ -0,0 +1,121 @@ +""" +.. _executorch_export_coalesced: + +Exporting a Coalesced TensorRT + CUDA Model to ExecuTorch (.pte) +================================================================ + +This example exports one graph split across two ExecuTorch backends. TensorRT +takes the operators it can convert, and everything left over goes to +ExecuTorch's own CUDA backend, which compiles it with AOTInductor. Both +delegates end up inside a single ``.pte`` and run in the same method. + +The model is ``cos(erfinv(tanh(x)))``. TensorRT has no converter for +``erfinv``, so that operator is the one the CUDA backend has to claim. That +makes the split real rather than incidental. + +A value produced by one delegate is consumed by the other on the device, inside +a single method, so this is what proves the two backends can complete each +other rather than only work on their own. + +The CUDA backend also writes an ``aoti_cuda_blob.ptd`` next to the ``.pte`` for +its external weights. This model has no weights, so that file is empty of +tensors and the reference runner does not need it. + +Besides the ``.pte`` this writes ``.expected``, holding the output +shape and the eager reference value for an all-ones input. The reference runner +gate reads that file instead of hard-coding a number, so the expected value +cannot drift away from the model. + +Prerequisites +------------- +Install Torch-TensorRT with the ExecuTorch extra before running this example:: + + pip install -e ".[executorch]" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 + +ExecuTorch's CUDA backend also needs a CUDA toolkit (``nvcc``) at export time, +for the AOTInductor compile. +""" + +import argparse +import sys +from pathlib import Path + +import torch +import torch_tensorrt +from executorch.backends.cuda.cuda_backend import CudaBackend +from executorch.backends.cuda.cuda_partitioner import CudaPartitioner +from executorch.exir._serialize._program import deserialize_pte_binary + +SHAPE = (64, 64) + + +class CoalescedModel(torch.nn.Module): + def forward(self, x): + return torch.cos(torch.erfinv(torch.tanh(x))) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_path", default="coalesced.pte", help="Path to save the .pte" + ) + args = parser.parse_args() + model_path = Path(args.model_path) + + with torch.no_grad(): + model = CoalescedModel().eval().cuda() + example_input = (torch.randn(SHAPE).cuda(),) + + exported_program = torch.export.export(model, example_input) + trt_gm = torch_tensorrt.dynamo.compile( + exported_program, + arg_inputs=example_input, + min_block_size=1, + truncate_double=True, + ) + torch_tensorrt.save( + trt_gm, + str(model_path), + output_format="executorch", + arg_inputs=example_input, + retrace=False, + # Catch-all, so every operator TensorRT rejected goes to the CUDA + # backend instead of falling back to a portable CPU kernel. + partitioners=[ + CudaPartitioner( + [CudaBackend.generate_method_name_compile_spec("forward")] + ) + ], + ) + + # Both delegates must really be in the file. A partitioner change that + # quietly routed the whole graph to TensorRT would otherwise leave a + # green job that no longer tests the coalesced path at all. + program = deserialize_pte_binary(model_path.read_bytes()).program + delegates = [d.id for plan in program.execution_plan for d in plan.delegates] + missing = [ + name for name in ("TensorRTBackend", "CudaBackend") if name not in delegates + ] + if missing: + sys.exit( + f"{model_path} is not coalesced: missing {missing}, found {delegates}" + ) + + # The reference runners fill every input element with 1.0, and this model + # is elementwise, so a single number describes the whole expected output. + reference = model(torch.ones(SHAPE).cuda()) + expected_path = model_path.with_suffix(".expected") + expected_path.write_text( + "[{}]\n{:.4f}\n".format( + ",".join(str(dim) for dim in reference.shape), + reference.flatten()[0].item(), + ) + ) + + print(f"Saved {model_path} with delegates {delegates}.") + print(f"Saved {expected_path} with the eager reference output.") + + +if __name__ == "__main__": + main() diff --git a/examples/torchtrt_executorch_example/export_dynamic_shape.py b/examples/torchtrt_executorch_example/export_dynamic_shape.py index 28115f696c..64847c3c50 100644 --- a/examples/torchtrt_executorch_example/export_dynamic_shape.py +++ b/examples/torchtrt_executorch_example/export_dynamic_shape.py @@ -16,7 +16,8 @@ ------------- Install Torch-TensorRT with the ExecuTorch extra before running this example:: - pip install -e ".[executorch]" + pip install -e ".[executorch]" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 See https://pytorch.org/executorch/stable/getting-started-setup.html for details. """ diff --git a/examples/torchtrt_executorch_example/export_kv_cache_decode.py b/examples/torchtrt_executorch_example/export_kv_cache_decode.py index c8590d98b0..5dcf3476fa 100644 --- a/examples/torchtrt_executorch_example/export_kv_cache_decode.py +++ b/examples/torchtrt_executorch_example/export_kv_cache_decode.py @@ -18,7 +18,8 @@ ------------- Install Torch-TensorRT with the ExecuTorch extra before running this example:: - pip install -e ".[executorch]" + pip install -e ".[executorch]" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 """ import argparse diff --git a/examples/torchtrt_executorch_example/export_static_shape.py b/examples/torchtrt_executorch_example/export_static_shape.py index ed8bb218da..eadc36f0d4 100644 --- a/examples/torchtrt_executorch_example/export_static_shape.py +++ b/examples/torchtrt_executorch_example/export_static_shape.py @@ -12,7 +12,8 @@ ------------- Install Torch-TensorRT with the ExecuTorch extra before running this example:: - pip install -e ".[executorch]" + pip install -e ".[executorch]" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 See https://pytorch.org/executorch/stable/getting-started-setup.html for details. """ diff --git a/justfile b/justfile index 06f1574b02..cb91342dc1 100644 --- a/justfile +++ b/justfile @@ -85,7 +85,16 @@ summary *args: # Install optional test deps so model/kernels/quantization/executorch suites run install-test-ext: uv pip install --group test-ext --group kernels --group quantization - uv pip install pyyaml "executorch>=1.4.1,<1.5" + # ExecuTorch's CUDA wheels are only on the PyTorch nightly index, so the channel is needed. + # No --pre: a specifier naming a prerelease admits prereleases by itself, and --pre would + # apply to pyyaml here too. cu130 matches the torch index this project resolves against by + # default. + # + # Exact, not a range: the nightly channel gains a member every day, and the delegate is + # compiled from the commit this version pairs with. + uv pip install pyyaml \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 \ + "executorch==1.5.0.dev20260825" # ── Linting ─────────────────────────────────────────────────────────────────── diff --git a/py/torch-tensorrt-executorch-runtime/README.md b/py/torch-tensorrt-executorch-runtime/README.md index d6d627a7b7..d724b9a1eb 100644 --- a/py/torch-tensorrt-executorch-runtime/README.md +++ b/py/torch-tensorrt-executorch-runtime/README.md @@ -39,7 +39,9 @@ of the wheel runtime contract. ```bash export TensorRT_ROOT=/path/to/TensorRT -python -m pip install pyyaml "executorch==1.4.1" +python -m pip install pyyaml \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 \ + "executorch==1.5.0.dev20260825" python -m pip wheel --no-build-isolation --no-deps \ --wheel-dir dist py/torch-tensorrt-executorch-runtime ``` @@ -47,7 +49,7 @@ python -m pip wheel --no-build-isolation --no-deps \ The native build obtains the ExecuTorch source through Bazel; no separate source checkout or `EXECUTORCH_SOURCE_DIR` setting is required. The source commit pinned in `MODULE.bazel` is the revision recorded by the -`executorch==1.4.1` wheel. +`executorch==1.5.0.dev20260825` wheel. The static ExecuTorch and delegate archives are intermediate build inputs; users receive the final native Python module and do not compile anything. @@ -73,8 +75,14 @@ GPU should use the ExecuTorch C++ runner. ## Use +The wheel's dependencies (`executorch`, `torch-tensorrt`, and the CUDA +runtime) resolve from the PyTorch nightly index, so install it with the same +channel the build recipe used. `--pre` lets pip select the pinned ExecuTorch +dev build: + ```bash -python -m pip install torch-tensorrt-executorch-runtime +python -m pip install --pre dist/torch_tensorrt_executorch_runtime-*.whl \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 ``` ```python diff --git a/py/torch-tensorrt-executorch-runtime/pyproject.toml b/py/torch-tensorrt-executorch-runtime/pyproject.toml index 78e33164c4..5a036cdbc1 100644 --- a/py/torch-tensorrt-executorch-runtime/pyproject.toml +++ b/py/torch-tensorrt-executorch-runtime/pyproject.toml @@ -6,6 +6,6 @@ requires = [ # environment. # Builds must use --no-build-isolation; see README.md. "torch", - "executorch==1.4.1", + "executorch==1.5.0.dev20260825", ] build-backend = "setuptools.build_meta" diff --git a/py/torch-tensorrt-executorch-runtime/setup.py b/py/torch-tensorrt-executorch-runtime/setup.py index dc3d853556..bb6e1a1078 100644 --- a/py/torch-tensorrt-executorch-runtime/setup.py +++ b/py/torch-tensorrt-executorch-runtime/setup.py @@ -157,7 +157,7 @@ def build_extension(self, ext: Extension) -> None: install_requires=[ f"torch=={public_version(torch.__version__)}", f"executorch=={public_version(executorch_version)}", - f"torch-tensorrt=={torchtrt_version()}", + f"torch-tensorrt=={public_version(torchtrt_version())}", f"{TENSORRT_DISTRIBUTION}=={tensorrt_version}", f"{CUDA_RUNTIME_DISTRIBUTION}=={cuda_runtime_version}", ], diff --git a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py index 27010326d7..90cf4f98f6 100644 --- a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py +++ b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py @@ -10,13 +10,12 @@ def _get_runtime() -> _Runtime: - try: - from torch_tensorrt_executorch_runtime import get_runtime - except ImportError as error: - raise ImportError( - "ExecuTorch Python inference requires the prebuilt delegate. " - 'Install it with: pip install "torch-tensorrt[executorch]"' - ) from error + # get_runtime is defined at module scope in this package's __init__, from stdlib and local + # imports, so importing the name here cannot fail once this submodule is importable. Calling it + # can still fail: it imports the ExecuTorch runtime, which raises ModuleNotFoundError when + # ExecuTorch is absent, and DelegateCompatibilityError when the delegate is not registered. + from torch_tensorrt_executorch_runtime import get_runtime + return get_runtime() diff --git a/py/torch_tensorrt/_compile.py b/py/torch_tensorrt/_compile.py index fa70e13c46..6a0fb9930d 100644 --- a/py/torch_tensorrt/_compile.py +++ b/py/torch_tensorrt/_compile.py @@ -26,6 +26,7 @@ from torch_tensorrt._enums import dtype from torch_tensorrt._features import ENABLED_FEATURES, needs_cross_compile from torch_tensorrt._Input import Input +from torch_tensorrt._utils import executorch_install_command from torch_tensorrt.dynamo.runtime._CudaGraphsTorchTensorRTModule import ( CudaGraphsTorchTensorRTModule, ) @@ -630,9 +631,10 @@ def load( if format == "executorch": if not _has_executorch_runtime(): raise ImportError( - "Loading an ExecuTorch program requires the prebuilt " - "Torch-TensorRT ExecuTorch delegate. Install it with: " - "pip install torch-tensorrt-executorch-runtime" + "Loading an ExecuTorch program requires the Torch-TensorRT " + "ExecuTorch delegate runtime (torch_tensorrt_executorch_runtime), " + "which is not yet published to any package index. Build and install " + "it from source following py/torch-tensorrt-executorch-runtime/README.md." ) from torch_tensorrt_executorch_runtime.runtime import load as load_executorch @@ -856,9 +858,9 @@ def save( ) if output_format == "executorch" and not _has_executorch_exir(): raise ImportError( - "Saving in ExecuTorch format requires the executorch package " - "with executorch.exir. Install with: pip install " - "\"torch_tensorrt[executorch]\" to use output_format='executorch'." + "Saving in ExecuTorch format requires the executorch package with " + "executorch.exir, published for Linux only, to use " + "output_format='executorch'. Install with: " + executorch_install_command() ) if output_format == "executorch": # Every executorch option is popped above, so a leftover kwarg is a typo. Fail @@ -1405,8 +1407,8 @@ def _save_as_executorch(exp_program: Any, file_path: str, **kwargs: Any) -> None from torch_tensorrt.executorch import export except ImportError: raise ImportError( - "ExecuTorch is not installed. Install with: pip install " - "\"torch_tensorrt[executorch]\" to use output_format='executorch'." + "ExecuTorch is not installed, and is published for Linux only, to use " + "output_format='executorch'. Install with: " + executorch_install_command() ) import torch_tensorrt.dynamo.runtime.meta_ops.register_meta_ops # noqa: F401 diff --git a/py/torch_tensorrt/_utils.py b/py/torch_tensorrt/_utils.py index 5d43db0e57..565fae87de 100644 --- a/py/torch_tensorrt/_utils.py +++ b/py/torch_tensorrt/_utils.py @@ -26,6 +26,37 @@ def sanitized_torch_version() -> Any: ) +def executorch_install_channel() -> str: + """The PyTorch nightly channel that carries the ExecuTorch build matching this torch. + + ExecuTorch publishes a distinct wheel per CUDA channel (``+cu130`` and ``+cu132`` are separate + builds), so an install instruction has to name the channel that matches the user's torch, or a + CUDA 13.2 user installs a CUDA 13.0 ExecuTorch. Derived from ``torch.version.cuda`` rather than + hardcoded for that reason. Falls back to the literal placeholder ``cuXYZ`` when torch reports no + CUDA build, so the message stays honest instead of naming a channel the user cannot use. + """ + cuda_version = torch.version.cuda + if not cuda_version: + return "cuXYZ" + major, _, minor = cuda_version.partition(".") + return f"cu{major}{minor or '0'}" + + +def executorch_install_command() -> str: + """The exact ``pip install`` line for the ExecuTorch authoring stack, channel included. + + Shared by every runtime error message that tells a user how to install ExecuTorch, so the + channel is derived once from the running torch and the three messages cannot drift from each + other or from the pin. ``--upgrade`` because the message is raised from inside an already + installed ``torch_tensorrt``: without it pip treats the requirement as satisfied and exits 0 + without adding the ``executorch`` extra. + """ + return ( + 'pip install --pre --upgrade "torch_tensorrt[executorch]" ' + f"--extra-index-url https://download.pytorch.org/whl/nightly/{executorch_install_channel()}" + ) + + def check_cross_compile_trt_win_lib() -> bool: # cross compile feature is only available on linux # build engine on linux and run on windows diff --git a/py/torch_tensorrt/executorch/__init__.py b/py/torch_tensorrt/executorch/__init__.py index fef0943ce7..28becd4df8 100644 --- a/py/torch_tensorrt/executorch/__init__.py +++ b/py/torch_tensorrt/executorch/__init__.py @@ -22,10 +22,12 @@ def _has_executorch_exir() -> bool: if not _has_executorch_exir(): def __getattr__(name: str) -> NoReturn: + from torch_tensorrt._utils import executorch_install_command + raise ImportError( f"Cannot access torch_tensorrt.executorch.{name}: " - "ExecuTorch with executorch.exir is required. " - 'Install with: pip install "torch_tensorrt[executorch]"' + "ExecuTorch with executorch.exir is required, and is published for " + "Linux only. Install with: " + executorch_install_command() ) __all__ = [ diff --git a/setup.py b/setup.py index 1c994f6f39..1a5372aa4c 100644 --- a/setup.py +++ b/setup.py @@ -206,12 +206,24 @@ def load_dep_info(): # The delegate is compiled from the ExecuTorch source revision pinned in MODULE.bazel, so the # installed wheel should agree with it. The upper bound is the load-bearing half: ExecuTorch's C++ # runtime API is not stable across minor releases, and an unbounded floor would resolve a future -# minor against a backend built for this one. Patch releases stay allowed because they come off the -# same release branch; the exact pin belongs in the runtime package, which does derive it. +# minor against a backend built for this one. Everything below that ceiling resolves: later 1.5 +# nightlies, a 1.5 release candidate, and 1.5 patch releases alike, since the exact pin belongs in +# the runtime package, which does derive it. +# The floor currently names a dev build, because the runtime split the delegate needs does not +# exist in any ExecuTorch release yet: 1.4.1's executorch/lib carries no standalone linkable +# runtime, and no CUDA wheel at all. +# That also makes this range prefer a release as soon as one exists, since 1.5.0 sorts above +# every 1.5.0.devN, so nothing here changes on the day it ships. _executorch_major, _executorch_minor = __executorch_version__.split(".")[:2] +# Linux-only, and not incidentally: the delegate is a Linux shared object, ExecuTorch publishes +# CUDA wheels for no other platform, and the feature is documented Linux-only. Without the marker +# the extra also has to resolve for the win32 entry in pyproject.toml's uv required-environments, +# where the only candidates are PyPI's, which stop at 1.4.1 -- so raising this floor above that +# makes `uv lock` fail outright rather than pick something older. EXECUTORCH_REQUIREMENT = ( f"executorch>={__executorch_version__}," - f"<{_executorch_major}.{int(_executorch_minor) + 1}" + f"<{_executorch_major}.{int(_executorch_minor) + 1}; " + "platform_system == 'Linux'" ) # TODO: Enable this once the runtime wheel is published to the PyTorch index. # EXECUTORCH_RUNTIME_REQUIREMENT = ( diff --git a/tests/ci/runner.py b/tests/ci/runner.py index fe98ab3423..2d904c5777 100644 --- a/tests/ci/runner.py +++ b/tests/ci/runner.py @@ -27,12 +27,15 @@ def _executorch_requirement() -> str: # Read the pin the way the drift test does, so this file is not a second # place to edit when it moves. Regex rather than yaml: the runner declares # no runtime dependencies of its own and importing it should not add one. + # + # Exact, not a range: the nightly channel gains a member every day, so a + # range would install whatever is newest while the delegate is compiled + # from the pinned commit. Pairing them is the point. text = (REPO_ROOT / "dev_dep_versions.yml").read_text() version = dict(re.findall(r'^(__\w+__): "([^"]+)"', text, re.MULTILINE))[ "__executorch_version__" ] - major, minor = version.split(".")[:2] - return f"executorch>={version},<{major}.{int(minor) + 1}" + return f"executorch=={version}" # Known transient cudagraph/TRT-driver flake signatures. Expand ONLY with @@ -131,10 +134,25 @@ def _setup_commands(step: str) -> list[tuple[list[str], Path]]: if step == "hub": return [(launcher + ["hub.py"], REPO_ROOT / "tests/modules")] if step == "executorch": + # ExecuTorch's CUDA wheels are published only on the PyTorch nightly index, so the + # channel is needed here. Derived from CU_VERSION rather than fixed, because the + # executorch suite is nightly-only and the nightly matrix runs cu132 rows as well as + # cu130 ones; a fixed channel would install a CUDA 13.0 runtime into a 13.2 job. The + # cu130 default is for a local run with no CU_VERSION set, and matches the torch index + # pyproject.toml resolves against by default. + cuda = os.environ.get("CU_VERSION", "cu130") return [ ( launcher - + ["-m", "pip", "install", "pyyaml", _executorch_requirement()], + + [ + "-m", + "pip", + "install", + "pyyaml", + "--extra-index-url", + f"https://download.pytorch.org/whl/nightly/{cuda}", + _executorch_requirement(), + ], REPO_ROOT, ) ] @@ -202,7 +220,20 @@ def run_suite( print(f"==> setup[{step}]: {shlex.join(argv)}", flush=True) rc = subprocess.run(argv, cwd=scwd, env=env).returncode if rc != 0: - print(f"::warning::setup step {step!r} exited {rc}", flush=True) + # Fail rather than warn and continue, for every setup step and not just the + # executorch one: a suite whose dependencies did not install cannot test what it + # was asked to test, whichever step failed. Most of the executorch suite gates on + # pytest.importorskip, so a failed install skips those files, leaves the rest + # passing, and reports success with a populated junit xml -- the run looks green + # precisely when the thing it exists to test is absent. This matters more now + # that the ExecuTorch pin names a nightly build, which is pruned from the + # channel eventually; when that happens this has to be loud. + print( + f"::error::setup step {step!r} exited {rc}, so the suite cannot test what " + "it was asked to test", + flush=True, + ) + return rc print(f"==> {suite.name} [{variant}]: {shlex.join(pytest_cmd)}", flush=True) rc = subprocess.run(pytest_cmd, cwd=cwd, env=env).returncode diff --git a/tests/ci/suites.py b/tests/ci/suites.py index 5e5fa2bf4b..d3cd6b8e5f 100644 --- a/tests/ci/suites.py +++ b/tests/ci/suites.py @@ -28,7 +28,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Literal +from typing import Any, Literal, get_args Tier = Literal["l0", "l1", "l2"] # python-only validates the PYTHON_ONLY=1 wheel (no C++ runtime) against the @@ -108,6 +108,26 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: base.update(self.overrides.get(variant, {})) return base + def __post_init__(self) -> None: + # Validate the enum-like fields at construction. They are typed as Literal, but nothing + # enforces that at runtime, so a typo like lanes=("nightl",) used to define a suite that + # every lane filter silently skipped, dropping it from CI with no error anywhere. Checking + # here turns that typo into an immediate, located failure when this module is imported. + for field_name, allowed in ( + ("tier", get_args(Tier)), + ("lanes", get_args(Lane)), + ("variants", get_args(Variant)), + ("platforms", get_args(Platform)), + ): + value = getattr(self, field_name) + values = (value,) if isinstance(value, str) else value + unknown = [v for v in values if v not in allowed] + if unknown: + raise ValueError( + f"suite {self.name!r} has unknown {field_name} {unknown}; " + f"expected a subset of {list(allowed)}" + ) + # ── L0 — smoke / fast lane ──────────────────────────────────────────────────── _L0: list[Suite] = [ @@ -260,6 +280,15 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: tier="l2", lanes=("nightly",), paths=("executorch/",), + keyword=( + # The pairing test is the one check here that needs a real ExecuTorch installed, so + # it has to survive this deselection. Everything else in that file is a + # source-consistency check the lint job already covers. The pin updater's own tests + # are repository tooling with no GPU or ExecuTorch need, covered by the lint job, so + # they are deselected here rather than run in a CUDA container. + "(not test_executorch_pin or test_the_pinned_commit_is_the_pinned_wheels_own_source)" + " and not test_update_executorch_pin" + ), setup=("executorch",), jobs="auto", variants=("standard",), diff --git a/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py b/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py index f3ef0d47e5..9bdc065e18 100644 --- a/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py +++ b/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py @@ -89,11 +89,10 @@ def _delegate_ids(pte_path): # NOTE: these tests are COMPOSITION-ONLY. They assert the serialized .pte carries # both a TensorRTBackend and a CudaBackend delegate (and, below, that external -# CUDA weights are persisted as a .ptd). They do NOT load or run the program: a -# coalesced ATen-mode .pte cannot be loaded yet because memory-planned CUDA -# buffers get a CPU data pointer, so Method::init fails the CUDA backend's device -# check (tensor_parser_aten hardcodes CPU). A load-run-allclose test should be -# added once that runtime fix lands (follow-up). +# CUDA weights are persisted as a .ptd). They do NOT load or run the program. +# Execution of a coalesced program is covered by the reference runner gate, see +# examples/torchtrt_executorch_example/export_coalesced.py and the third argument +# of .github/scripts/verify-executorch-reference-runner.sh. def test_erfinv_routes_to_cuda_backend(tmp_path): diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index ecb6458f5d..d1f0b46e83 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -8,19 +8,64 @@ right spelling per role is the point: installable metadata that pins exactly would reject a compatible patch release, and a build input that takes a range could resolve an ExecuTorch the artifact was not compiled against. + +Agreeing with the file is necessary but not sufficient, so +test_the_pinned_commit_is_the_pinned_wheels_own_source closes the gap the others leave: they +only prove the repository is self-consistent, which it would be even if the wheel and the +commit named two different ExecuTorch trees. """ import ast import os +import pathlib import re +import shlex import subprocess import sys +import tempfile +from collections import Counter from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[4] VERSIONS = REPO_ROOT / "dev_dep_versions.yml" -REQUIREMENT = re.compile(r"executorch(?:==|>=)[0-9][^\"'\s,`]*(?:,<[0-9.]+)?") +# Every PEP 440 operator, not just the two this repository happens to use, and an optional +# space before it. A site added with a compatible-release or bare-inequality operator is a site +# that drifted from the pin, and it should be visible to the search rather than silently +# exempt. Operators are named through the pattern rather than spelled out in prose here, +# because the search below reads this file too and an example would read as such a site. +# The whole specifier set, not just its first clause. Capturing up to the first comma compared +# equal on "executorch==PIN,!=PIN", a specifier that excludes the very version it appears to pin, +# and rejected the legal PEP 508 spelling with spaces around the operator. +REQUIREMENT = re.compile( + r"executorch\s*(?:===|==|>=|<=|~=|!=|<|>)\s*[^\"'\s`,]+" + r"(?:\s*,\s*(?:===|==|>=|<=|~=|!=|<|>)\s*[^\"'\s`,]+)*" +) + + +def _requirement_disagrees(actual: str, expected: str, version: str) -> str: + """Why ``actual`` is not the pinned requirement, or an empty string if it is. + + Compares parsed specifier sets rather than matched text. Raw text equality could not see a + clause the pattern did not capture, and treated whitespace the specification allows as drift. + """ + from packaging.requirements import InvalidRequirement, Requirement + + try: + parsed = Requirement(actual) + wanted = Requirement(expected) + except InvalidRequirement as error: + return f"which is not a valid requirement ({error})" + if parsed.name != wanted.name: + return f"which names {parsed.name}, not {wanted.name}" + if not parsed.specifier.contains(version, prereleases=True): + return f"whose specifier excludes the pinned {version}" + if set(parsed.specifier) != set(wanted.specifier): + return f"expected {expected}" + return "" + # The bazel repository puts the commit on its own line, so this one has to run against file # contents rather than a git grep line. @@ -31,22 +76,76 @@ # Anywhere else the commit appears it is named on the same line, as a shell default or prose. NAMED_COMMIT = re.compile(r"executorch[_a-z]*[^0-9a-f]*([0-9a-f]{40})", re.IGNORECASE) -# Requirements that end up in metadata someone resolves at install time. A patch release off -# the same branch has to stay installable, so these take the range. Everything else pins -# exactly: a build input compiled against one wheel, or a comment labelling a source sha. -# Only literal requirements land here. setup.py and tests/ci/runner.py derive theirs from -# dev_dep_versions.yml, so the search below no longer sees them and -# test_derived_requirements_match_the_pin covers them instead. -RANGE_SITES = frozenset( - { - "justfile", - } -) +PAIRING_TEST = "test_the_pinned_commit_is_the_pinned_wheels_own_source" +# A requirement takes the install-time range only when a comment carrying this exact token sits +# above it, so a site that reproduces a user's workflow can stay installable across a patch +# release. An explicit opt-out token rather than prose: "verify the end user's workflow" is a +# sentence someone can paste above a requirement without meaning to license a range there. +USER_WORKFLOW_MARKER = "pin-check: range-ok" + +# A pip install of the plain torch-tensorrt wheel on a platform that has no ExecuTorch dev wheel +# carries this token. win32 is the case: the glob it installs also matches the Linux-only runtime +# wheel, so the channel scan reaches it, but ExecuTorch publishes no win32 nightly and the +# [executorch] extra is Linux-only. An explicit token rather than inference, so the exemption is +# deliberate and cannot be granted by accident to a Linux install that simply lost its index. +NO_NIGHTLY_MARKER = "pin-check: no-nightly" + +# The files expected to pin ExecuTorch, mapped to how many sites each must carry, excluding +# dev_dep_versions.yml itself. A count per file rather than just the set of files, because a site +# that loses its version stops matching the search entirely rather than reporting a mismatch, and +# two of these files carry more than one site, so a set of paths let either quietly drop one. A +# minimum rather than an exact count, since the stacked runtime-wheel change removes one README +# site and an exact count could not hold on both branches. +_EXPECTED_REQUIREMENT_SITES = { + ".github/workflows/executorch-build-linux.yml": 2, + ".github/workflows/executorch-test-linux.yml": 1, + "MODULE.bazel": 1, + "docker/MODULE.bazel.docker": 1, + "docker/MODULE.bazel.ngc": 1, + "justfile": 1, + # One: the fenced install command. The prose sentence below it is documentation, checked for + # pin agreement but not counted, so it cannot stand in for the command if that loses its pin. + "py/torch-tensorrt-executorch-runtime/README.md": 1, + "py/torch-tensorrt-executorch-runtime/pyproject.toml": 1, + "toolchains/ci_workspaces/MODULE.bazel.tmpl": 1, +} -# A step that exists to reproduce what a user runs belongs to the range group even inside a -# file that otherwise pins build inputs, so the marker travels with the line rather than -# with the path. -USER_WORKFLOW_MARKER = "verify the end user's workflow" +# Same idea for the source commit the delegate compiles from. Five sites, not four: the +# reference-runner README names the ref as a shell default, which the old nonzero check could +# not distinguish from the four MODULE.bazel files. +_EXPECTED_COMMIT_SITES = { + "MODULE.bazel": 1, + "docker/MODULE.bazel.docker": 1, + "docker/MODULE.bazel.ngc": 1, + "toolchains/ci_workspaces/MODULE.bazel.tmpl": 1, + "examples/executorch_reference_runner/README.md": 1, +} + + +def _assert_every_site_present( + seen: "Counter[str]", expected: dict[str, int], what: str +) -> None: + """Require every expected file to still carry at least its expected number of sites. + + A site that drops its version or commit stops matching the search rather than reporting a + mismatch, so counting is the only way to notice it left. + """ + short = { + path: (count, seen.get(path, 0)) + for path, count in expected.items() + if seen.get(path, 0) < count + } + unexpected = sorted(set(seen) - set(expected)) + assert not short and not unexpected, ( + f"the set of sites {what} changed.\n" + + "".join( + f" {path} carries {actual} of {want} expected sites\n" + for path, (want, actual) in sorted(short.items()) + ) + + "".join(f" {path} is new and unaccounted for\n" for path in unexpected) + + "A site that lost its pin does not appear in the search at all, so look for one that " + "now names a bare reference before updating the expected counts." + ) def _git(*arguments: str) -> str: @@ -61,45 +160,195 @@ def _versions() -> dict: return dict(re.findall(r'^(__\w+__): "([^"]+)"', text, re.MULTILINE)) -def _wants_range(path: str, number: int) -> bool: - if path in RANGE_SITES: - return True +def _has_marker_above(path: str, number: int, marker: str) -> bool: + """Whether a comment carrying ``marker`` sits directly above line ``number``. - # Scan upward rather than reading one fixed line, so reformatting the workflow cannot - # silently reclassify the requirement and fail the build for an unrelated reason. - lines = (REPO_ROOT / path).read_text().splitlines() + Scans upward past blank and comment lines; the first line of real content stops the scan, so + the marker cannot leak from an unrelated command far above onto this one. + """ + lines = (REPO_ROOT / path).read_text(encoding="utf-8").splitlines() for line in reversed(lines[: number - 1]): - if not line.strip(): + stripped = line.strip() + if not stripped: continue - return USER_WORKFLOW_MARKER in line + if not stripped.startswith("#"): + return False + if marker in stripped: + return True return False +def _wants_range(path: str, number: int) -> bool: + # Only a comment carrying the token licenses a range; the first line of real content stops the + # scan, so the opt-out cannot leak onto an unrelated requirement further down. + return _has_marker_above(path, number, USER_WORKFLOW_MARKER) + + +def _release_line(version: str) -> tuple[str, str]: + """Split a pin into its major and minor, for either a release or a nightly. + + ``1.4.1`` and ``1.5.0.dev20260822`` both belong to the release line their first two + fields name, so the range a site gets is derived from those and nothing else. Splitting + on every dot instead assumes three fields and raises on the nightly form. + """ + major, minor = version.split(".")[:2] + return major, minor + + def _expected(path: str, number: int, version: str) -> str: if not _wants_range(path, number): return f"executorch=={version}" - # Assumes X.Y.Z, which is what ExecuTorch releases and what this file records. - major, minor, _ = version.split(".") + major, minor = _release_line(version) return f"executorch>={version},<{major}.{int(minor) + 1}" +# Nightly channels that actually carry the pinned ExecuTorch line. cu124 and cu128 exist on the +# index but are frozen at a CPU-only 0.5.0.dev build, so a recipe pointed at them resolves nothing +# the pin can use. Only these three serve the 1.5.0.dev wheels this change installs. +_PUBLISHED_NIGHTLY_CHANNELS = frozenset({"cu126", "cu130", "cu132"}) + +# Tracked files with no suffix that still carry install commands. justfile writes the nightly +# ExecuTorch install for local builds, so the printed-install walk has to read it by name. +_EXTENSIONLESS_INSTALL_FILES = frozenset({"justfile"}) + +# Index-URL variables whose value legitimately arrives from the CI environment and so has no +# assignment in the tree to resolve. An unresolved variable is accepted only if it is one of +# these; every other unresolved name, including a typo of a real one, fails the channel check +# rather than passing on sight. Empty today: every ExecuTorch install channels through either a +# literal nightly URL or ${EXECUTORCH_INDEX_URL}, which is assigned in executorch-build-linux.yml +# and therefore resolvable. Kept as the explicit seam a future environment-provided index goes +# through. +_ENVIRONMENT_INDEX_VARIABLES: frozenset[str] = frozenset() + +# The bazel repositories annotate their pinned commit with the wheel it corresponds to, in a +# comment, because bazel fetches by commit and has no requirement string to carry. Those are the +# only comment sites that count as pins, and the commit beside them is checked separately. +_ANNOTATED_COMMIT_SITES = frozenset( + { + "MODULE.bazel", + "docker/MODULE.bazel.docker", + "docker/MODULE.bazel.ngc", + "toolchains/ci_workspaces/MODULE.bazel.tmpl", + } +) + + +def _is_commented_out(path: str, text: str) -> bool: + """Whether this requirement sits in a comment rather than in live configuration. + + A comment is not a pin: a site could be gutted to a bare ``executorch`` while the exact pin + lived on in a comment in the same file, which kept the per-file minimum satisfied and left the + real requirement unpinned. + """ + if path in _ANNOTATED_COMMIT_SITES: + return False + stripped = text.strip() + # No exemption for prose. Returning False for .md/.rst/.txt defeated the threat named above, + # because it made a comment count as a pin in exactly the files where install commands live: a + # README install line gutted to a bare "executorch" passed as long as a decoy "# executorch==" + # sat beside it. A "#" inside a fenced shell block is a shell comment, the same as anywhere else. + return stripped.startswith(("#", "//", "/*", "*")) + + +def _without_trailing_comment(path: str, text: str) -> str: + """``text`` up to a trailing ``#`` or ``//`` comment, unless the site annotates its pin there.""" + if path in _ANNOTATED_COMMIT_SITES: + return text + for marker in ("#", "//"): + if marker in text: + text = text.split(marker, 1)[0] + return text + + +def _counts_toward_minimum(path: str, number: int) -> bool: + """Whether a requirement at this line counts toward the per-file minimum. + + In a prose file a requirement in running text is documentation, not a live pin. Gutting the + fenced install command to a bare ``executorch`` while a sentence below still spelled the pin + kept the per-file count satisfied and left the command unpinned, so only a requirement inside + a fenced code block counts for markdown. Every other file counts every live line; the + reStructuredText sites are guarded by the install-instruction test instead. + """ + if not path.endswith(".md"): + return True + fenced = False + for current, content in enumerate( + (REPO_ROOT / path).read_text(encoding="utf-8").splitlines(), start=1 + ): + if current == number: + return fenced + if content.lstrip().startswith(("```", "~~~")): + fenced = not fenced + return False + + +def _strip_whole_line_comments(text: str) -> str: + """Blank out whole-line ``#`` comments, preserving line count so DOTALL spans stay aligned. + + The Bazel commit match walks from ``name = "executorch"`` to the ``commit = "..."`` line with + DOTALL, so a commit commented out and replaced by a live ``branch = "main"`` still matched the + commented copy and the build floated to a branch while this test stayed green. + """ + return "\n".join( + "" if line.lstrip().startswith("#") else line for line in text.splitlines() + ) + + +def _resolve_shell_assignment(text: str, variable: str, before: int) -> str | None: + """The last literal ``VAR=...`` assignment of ``variable`` in ``text`` before offset ``before``. + + An install that channels through ``--extra-index-url "${VAR}"`` proves nothing on its own: the + value is whatever ``VAR`` was last set to. Repointing that assignment at PyPI, or dropping its + ``nightly/`` segment, left the install counted as channelled while it resolved nothing. Resolve + the assignment so the channel is checked where it is actually set. Returns ``None`` when no + assignment is found, meaning the value comes from the environment and cannot be resolved here. + """ + assignment = re.compile( + rf"""^\s*(?:export\s+)?{re.escape(variable)}=["']?([^"'\n]*)""", re.MULTILINE + ) + resolved = None + for match in assignment.finditer(text): + if match.start() >= before: + break + resolved = match.group(1) + return resolved + + def test_every_requirement_matches_the_pin() -> None: version = _versions()["__executorch_version__"] wrong = [] found = 0 - for line in _git("grep", "-nI", "-E", r"executorch(==|>=)[0-9]").splitlines(): + seen: Counter[str] = Counter() + for line in _git( + "grep", "-nI", "-E", r"executorch ?(===|==|>=|<=|~=|!=|<|>) ?[0-9]" + ).splitlines(): path, number, text = line.split(":", 2) if path == VERSIONS.name: continue + if _is_commented_out(path, text): + # A comment is not a pin. Counting raw matches meant a site could be gutted to a bare + # "executorch" while the exact pin lived on in a comment in the same file, keeping the + # per-file minimum satisfied. + continue expected = _expected(path, int(number), version) - for actual in REQUIREMENT.findall(text): + # A trailing comment is not a pin either. Skipping whole-line comments was not enough: a + # live install gutted to a bare "executorch" with a decoy "# executorch==" after it on + # the same line kept the per-file count satisfied and left the install unpinned. The + # annotated commit sites write their pin as a whole-line comment, which is handled above, + # so nothing legitimate is lost here. + counts = _counts_toward_minimum(path, int(number)) + for actual in REQUIREMENT.findall(_without_trailing_comment(path, text)): found += 1 - if actual != expected: - wrong.append(f"{path}:{number} has {actual}, expected {expected}") + if counts: + seen[path] += 1 + reason = _requirement_disagrees(actual, expected, version) + if reason: + wrong.append(f"{path}:{number} has {actual}, {reason}") assert found, "no ExecuTorch requirement found, so this test is not looking" + _assert_every_site_present(seen, _EXPECTED_REQUIREMENT_SITES, "pinning ExecuTorch") assert not wrong, "\n ".join(["", *wrong]) @@ -144,15 +393,264 @@ def _runner_requirement(root: Path) -> str: ).stdout.strip() -def test_derived_requirements_match_the_pin() -> None: +def test_derived_requirements_match_the_pin(monkeypatch) -> None: # setup.py and tests/ci/runner.py build their requirement from the pin, so the search # above cannot see them. Check the strings they produce instead. + # + # They want different shapes. setup.py declares what users may install, so it is a range + # over the release line. runner.py installs the wheel CI tests the delegate against, and + # the delegate is compiled from the commit the pin names, so it has to be exact: the + # nightly channel gains a member every day and a range there silently unpairs the two. version = _versions()["__executorch_version__"] - major, minor, _ = version.split(".") - expected = f"executorch>={version},<{major}.{int(minor) + 1}" + major, minor = _release_line(version) + + # The Linux marker is part of the requirement: the extra has to resolve for the win32 entry + # in pyproject.toml's uv required-environments, where the only candidates are PyPI's and they + # stop below this floor, so without it `uv lock` fails outright. + assert _setup_py_requirement(version) == ( + f"executorch>={version},<{major}.{int(minor) + 1}; platform_system == 'Linux'" + ) + assert _runner_requirement(REPO_ROOT) == f"executorch=={version}" + + # The argument list CI actually runs, not just the string the helper derives. Dropping the + # requirement from the setup command left every one of these tests green: the step still + # succeeds, having installed no ExecuTorch, and the suite then skips on importorskip. + monkeypatch.syspath_prepend(str(REPO_ROOT / "tests")) + from ci.runner import _executorch_requirement, _setup_commands + + argv = [arg for command, _cwd in _setup_commands("executorch") for arg in command] + assert argv.count(_executorch_requirement()) == 1, ( + "the executorch setup step does not install the pinned ExecuTorch exactly once: " + f"{argv}" + ) + + # And the extra every documented `pip install "torch-tensorrt[executorch]"` relies on. + # Emptying it left these tests green too. Read as source rather than imported, because + # importing the top-level setup.py executes it. + setup_tree = ast.parse((REPO_ROOT / "setup.py").read_text(encoding="utf-8")) + extras = next( + node.value + for node in ast.walk(setup_tree) + if isinstance(node, ast.Assign) + and any(getattr(t, "id", None) == "EXTRAS_REQUIRE" for t in node.targets) + ) + # The published extras have to exist, or the loop below iterates nothing and deleting both + # keys is indistinguishable from them being correct. Only these two: an unrelated future extra + # has no reason to name ExecuTorch, and requiring it of every key made this test the one that + # turns red when someone adds "debug". + published = {"executorch", "all"} + present = {key.value for key in extras.keys if isinstance(key, ast.Constant)} + assert published <= present, ( + f"setup.py must publish the {sorted(published)} extras, but EXTRAS_REQUIRE has " + f"{sorted(present)}. Every documented install command names one of them." + ) + for key, value in zip(extras.keys, extras.values): + if getattr(key, "value", None) not in published: + continue + named = [ + element.id + for element in getattr(value, "elts", []) + if isinstance(element, ast.Name) + ] + assert named.count("EXECUTORCH_REQUIREMENT") == 1, ( + f"extra {getattr(key, 'value', key)!r} does not reference " + f"EXECUTORCH_REQUIREMENT exactly once: {named}" + ) + + # docgen builds its overlay with a shell substitution, so neither the literal search nor the + # two helpers above can see it: `$(` is not a digit. Run the command it embeds and compare + # what it prints, which fails if the line is deleted or the key is renamed. + workflow = (REPO_ROOT / ".github/workflows/docgen.yml").read_text(encoding="utf-8") + # Anchor to a live line: leading whitespace only, no "#". A commented-out install still + # carries the pattern, so a plain search stayed green when the whole step was disabled. + embedded = re.search( + r'^[ \t]*"executorch==\$\((python3 -c \'[^\']+\')\)"', + workflow, + re.MULTILINE, + ) + assert embedded, ( + ".github/workflows/docgen.yml no longer pins ExecuTorch alongside the extra on a live " + "line. It installs with --pre from the nightly channel, so without the pin it resolves " + "through the range and takes whichever dev build is newest that day." + ) + # Compared as text, not executed. Running it meant whatever that line said got executed on + # every pull request: rewriting the one-liner to write a file left the test green and the file + # written. It has to read __executorch_version__ out of dev_dep_versions.yml and print nothing + # else, which is the property that makes the shell substitution equal the pin. + command = embedded.group(1) + reads_the_pin = re.fullmatch( + r"""python3 -c 'import yaml;print\(yaml\.safe_load\(open\("dev_dep_versions\.yml"\)\)""" + r"""\["__executorch_version__"\]\)'""", + command, + ) + assert reads_the_pin, ( + "the docgen pin no longer reads __executorch_version__ out of dev_dep_versions.yml, so " + f"what it installs is no longer the pin: {command}" + ) - assert _setup_py_requirement(version) == expected - assert _runner_requirement(REPO_ROOT) == expected + +_RUNTIME_SETUP_PY = "py/torch-tensorrt-executorch-runtime/setup.py" + + +def _runtime_install_requires() -> dict[str, str]: + """The runtime wheel's ``install_requires`` entries, each mapped to its source text. + + Read as source, not imported: importing this setup.py runs a Bazel build. The values are + f-strings built at build time from the installed distributions, so the source segment is what + the check compares, not a resolved string. + """ + tree = ast.parse((REPO_ROOT / _RUNTIME_SETUP_PY).read_text(encoding="utf-8")) + source = (REPO_ROOT / _RUNTIME_SETUP_PY).read_text(encoding="utf-8") + call = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "setup" + ) + requires = next( + keyword.value for keyword in call.keywords if keyword.arg == "install_requires" + ) + entries = {} + for element in requires.elts: + text = ast.get_source_segment(source, element) + distribution = re.match(r'f?"([A-Za-z0-9_.-]+)', text) + if distribution: + entries[distribution.group(1)] = text + return entries + + +def test_the_runtime_wheel_pins_executorch_to_the_public_pin() -> None: + """The runtime wheel's own ExecuTorch requirement has to pin the pinned version, stripped. + + This is the one requirement whose native code is compiled against the ExecuTorch runtime, so a + wheel that requires a different ExecuTorch than it was built against loads a mismatched runtime. + The literal search above cannot see it: setup.py builds the string from the installed + distribution, so ``executorch==`` is followed by a brace, not a digit. Loosening it to a bare + ``executorch`` left every other test green. + """ + entries = _runtime_install_requires() + assert ( + "executorch" in entries + ), f"{_RUNTIME_SETUP_PY} install_requires no longer pins executorch: {sorted(entries)}" + # The value is built from installed_version("executorch"), the same source torch and + # torch-tensorrt use, and stripped of its local label the same way. Compare the source text so + # a bare name, a hardcoded version, or a different version source is rejected. + assert ( + entries["executorch"] == 'f"executorch=={public_version(executorch_version)}"' + ), ( + f"{_RUNTIME_SETUP_PY} must pin executorch to public_version(executorch_version), so the " + f"wheel requires the ExecuTorch it was compiled against, but it declares " + f"{entries['executorch']}" + ) + + +def test_the_runner_follows_the_row_s_cuda_version(monkeypatch) -> None: + """Call the runner and read the URL it builds, rather than matching its source. + + The executorch suite is nightly-only, and the nightly matrix runs cu132 rows as well as cu130 + ones, so a fixed channel would install a CUDA 13.0 ExecuTorch into a CUDA 13.2 job. PRs pin to + cu130, which is why watching PR CI cannot catch it. A source-text assertion could not catch it + either: keeping the ``os.environ.get`` line while hardcoding the URL passes one. + """ + monkeypatch.syspath_prepend(str(REPO_ROOT / "tests")) + try: + from ci import runner + finally: + sys.path.pop(0) + + def channel_for(cu_version: str | None) -> str: + if cu_version is None: + monkeypatch.delenv("CU_VERSION", raising=False) + else: + monkeypatch.setenv("CU_VERSION", cu_version) + commands = runner._setup_commands("executorch") + urls = [ + argument + for command, _ in commands + for argument in command + if "download.pytorch.org" in argument + ] + assert len(urls) == 1, f"expected one index URL, got {urls}" + return urls[0] + + assert channel_for("cu132").endswith("/nightly/cu132") + assert channel_for("cu130").endswith("/nightly/cu130") + # Unset is a local run, and matches the index pyproject.toml resolves against by default. + assert channel_for(None).endswith("/nightly/cu130") + + +def _load_utils_channel_helpers(fake_cuda: str | None): + """Exec ``executorch_install_channel``/``executorch_install_command`` with a stub torch. + + ``py/torch_tensorrt/_utils.py`` imports ``tensorrt`` and the built ``torch_tensorrt``, neither + installed on the lint runner, so it cannot be imported here. Extract just the two functions and + exec them against a fake ``torch`` whose ``version.cuda`` is ``fake_cuda``, which is all they + read. This keeps the test on the source that ships rather than a copy of its logic. + """ + source = (REPO_ROOT / "py/torch_tensorrt/_utils.py").read_text(encoding="utf-8") + tree = ast.parse(source) + wanted = {"executorch_install_channel", "executorch_install_command"} + functions = [ + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name in wanted + ] + assert {f.name for f in functions} == wanted, ( + "py/torch_tensorrt/_utils.py must define executorch_install_channel and " + f"executorch_install_command; found {sorted(f.name for f in functions)}" + ) + + class _Version: + cuda = fake_cuda + + namespace: dict[str, object] = {"torch": type("torch", (), {"version": _Version})} + module = ast.Module(body=functions, type_ignores=[]) + exec(compile(module, "", "exec"), namespace) # noqa: S102 + return ( + namespace["executorch_install_channel"], + namespace["executorch_install_command"], + ) + + +def test_the_executorch_install_message_names_the_torch_channel() -> None: + """The three ExecuTorch install messages derive their channel from the running torch. + + ExecuTorch ships a distinct wheel per CUDA channel, so a message that hardcodes cu130 tells a + CUDA 13.2 user to install a CUDA 13.0 build. The messages route through + ``executorch_install_command`` so the channel is computed once from ``torch.version.cuda``. + A source-text assertion cannot see the value the format string produces, so exercise the helper + across both published 13.x channels and the no-CUDA fallback. + """ + channel, command = _load_utils_channel_helpers("13.2") + assert channel() == "cu132" + message = command() + assert "download.pytorch.org/whl/nightly/cu132" in message, message + # Raised from inside an installed torch_tensorrt, so pip treats the requirement as satisfied + # and exits 0 without the extra unless --upgrade forces a re-resolve. --pre selects the dev pin. + assert "--upgrade" in message and "--pre" in message, message + + channel_130, command_130 = _load_utils_channel_helpers("13.0") + assert channel_130() == "cu130" + assert "nightly/cu130" in command_130() + + # No CUDA build reports a placeholder rather than a channel the user cannot install from. + channel_none, command_none = _load_utils_channel_helpers(None) + assert channel_none() == "cuXYZ" + assert "nightly/cuXYZ" in command_none() + + # Every message site delegates to the shared command rather than spelling its own URL, so the + # derived channel and --upgrade cannot regress in one message while the test watches another. + for path in ( + "py/torch_tensorrt/_compile.py", + "py/torch_tensorrt/executorch/__init__.py", + ): + text = (REPO_ROOT / path).read_text(encoding="utf-8") + assert "executorch_install_command()" in text, ( + f"{path} must build its ExecuTorch install message with " + "executorch_install_command() so the channel derives from the running torch" + ) + assert ( + "nightly/cu130" not in text + ), f"{path} still hardcodes nightly/cu130 in a message instead of deriving the channel" def test_derived_requirements_roll_the_minor_over(tmp_path: Path) -> None: @@ -160,13 +658,71 @@ def test_derived_requirements_roll_the_minor_over(tmp_path: Path) -> None: # Spelled through a variable because the search above reads this file too, and a # written-out requirement here would read as a site that drifted from the pin. version = "1.9.0" - expected = f"executorch>={version},<1.10" (tmp_path / "dev_dep_versions.yml").write_text( f'__executorch_version__: "{version}"\n' ) - assert _setup_py_requirement(version) == expected - assert _runner_requirement(tmp_path) == expected + assert ( + _setup_py_requirement(version) + == f"executorch>={version},<1.10; platform_system == 'Linux'" + ) + # No upper bound to roll over, but it must still track the pin it is given. + assert _runner_requirement(tmp_path) == f"executorch=={version}" + + +def test_the_pinned_commit_is_the_pinned_wheels_own_source() -> None: + """The two pins must name one ExecuTorch, not two that happen to be close. + + ``__executorch_version__`` selects the wheel the delegate is built to sit alongside, and + ``__executorch_commit__`` selects the tree it compiles from. Nothing about the two strings + forces them to agree, and a mismatch is invisible: both pins look plausible, the build + succeeds, and the delegate is compiled from one ExecuTorch while running against another. + Every published wheel records the commit it was built from, so the pairing is checkable + rather than a convention. + + Skipped rather than failed whenever the installed wheel is not the one the pin names, not + installed at all, a different member of a floating range, or built without git provenance. + None of those say anything about whether the two pins agree, and this file stays readable + offline. + """ + versions = _versions() + expected_commit = versions["__executorch_commit__"] + expected_version = versions["__executorch_version__"] + + try: + from executorch.version import __version__ as installed_version + from executorch.version import git_version as installed_commit + except ImportError: + pytest.skip("executorch is not installed, so the pinned wheel cannot be read") + + if installed_commit is None: + # ExecuTorch records this as Optional[str] and writes None when it is built outside a + # git checkout. Such a wheel carries no provenance to compare, which is not the pins + # disagreeing. + pytest.skip( + f"the installed ExecuTorch {installed_version} records no source commit, " + "so the pairing cannot be checked against it" + ) + + # The wheel carries a local version label naming its CUDA build (`+cu132`), which the pin + # deliberately omits so one pin serves every CUDA row. Compare the part they share. + if installed_version.split("+")[0] != expected_version: + # No evidence either way rather than a mismatch to report: only the wheel the pin names + # carries the commit the pin should agree with. Every CI install path that builds or + # tests the delegate requests the pin exactly -- the one deliberate range is the + # end-user install rehearsal in executorch-build-linux.yml -- so arriving here usually + # means the environment was built some other way, and that wheel's commit says nothing + # about whether the two pins agree. + pytest.skip( + f"the installed ExecuTorch is {installed_version}, not the pinned " + f"{expected_version}, so its commit says nothing about whether the pins agree" + ) + + assert installed_commit == expected_commit, ( + f"ExecuTorch {installed_version} was built from {installed_commit}, but " + f"__executorch_commit__ pins {expected_commit}. The delegate would compile against " + "one ExecuTorch and link another." + ) def test_every_source_commit_matches_the_pin() -> None: @@ -174,9 +730,12 @@ def test_every_source_commit_matches_the_pin() -> None: wrong = [] found = 0 + seen: Counter[str] = Counter() for path in _git("grep", "-lI", "-E", 'name = "executorch"').split(): - for match in BAZEL_COMMIT.finditer((REPO_ROOT / path).read_text()): + source = _strip_whole_line_comments((REPO_ROOT / path).read_text()) + for match in BAZEL_COMMIT.finditer(source): found += 1 + seen[path] += 1 if match.group(1) != commit: wrong.append(f"{path} compiles {match.group(1)}") @@ -184,10 +743,722 @@ def test_every_source_commit_matches_the_pin() -> None: path, number, text = line.split(":", 2) if path == VERSIONS.name: continue - for actual in NAMED_COMMIT.findall(text): + # A commit in a comment is not a pin. Grepping raw lines let EXECUTORCH_REF float to a + # branch with the real SHA left behind in a "# ..." comment on the same file, which kept + # this scan green. The Bazel walk above already strips comments; do the same here. + if _is_commented_out(path, text): + continue + for actual in NAMED_COMMIT.findall(_without_trailing_comment(path, text)): found += 1 + seen[path] += 1 if actual != commit: wrong.append(f"{path}:{number} uses {actual}") assert found, "no ExecuTorch source commit found, so this test is not looking" + _assert_every_site_present(seen, _EXPECTED_COMMIT_SITES, "naming the source commit") assert not wrong, f"pin says {commit}:\n " + "\n ".join(wrong) + + +@pytest.mark.unit +@pytest.mark.parametrize("setup_rc,expected", [(0, 0), (7, 7)]) +def test_a_failed_setup_step_stops_the_suite(monkeypatch, setup_rc, expected): + """A setup step that fails must fail the run, not warn and continue into pytest. + + Most of the ExecuTorch suite gates on ``pytest.importorskip``, so an install that fails makes + those files skip while everything else passes: the run reports success precisely when the + thing it exists to test is absent. That matters here because the pin names a nightly build, + which the channel eventually prunes. Replacing the ``return rc`` with ``continue`` kept every + other test in this file green, so assert on ``run_suite`` itself. + """ + monkeypatch.syspath_prepend(str(REPO_ROOT / "tests")) + from ci import runner + + calls: list[list[str]] = [] + + class Completed: + def __init__(self, argv): + # The setup step is the pip install; anything else is pytest, which must not run + # at all once setup has failed. + self.returncode = setup_rc if "pip" in argv else 0 + + def record(argv, **kwargs): + calls.append(argv) + return Completed(argv) + + monkeypatch.setattr(runner.subprocess, "run", record) + suite = next(s for s in runner.SUITES if s.name == "executorch") + rc = runner.run_suite(suite, "standard") + + assert rc == expected, f"run_suite returned {rc}, expected {expected}" + ran_pytest = any("pytest" in " ".join(argv) for argv in calls) + assert ran_pytest is (setup_rc == 0), ( + "pytest ran even though a setup step failed" + if ran_pytest + else "pytest never ran even though every setup step succeeded" + ) + + +@pytest.mark.unit +def test_the_lockfile_records_the_same_executorch_range_as_setup_py(): + """``uv.lock`` caches what ``setup.py`` declares, so it drifts when the pin moves. + + A stale lock breaks nothing here, because only ``uv-update.yml`` runs ``uv sync --locked``, + and its resolved hashes come from a resolver run against the nightly index that cannot be + faked in an editor. So this accepts two states: the range the pin derives, and a range that + predates the pin. + + It used to be a strict xfail, which meant the moment anyone refreshed the lock the assertion + passed and pytest reported that pass as a failure. The lint step runs this file on every pull + request, so that would have turned the lint job red repo-wide for a file none of those pull + requests touched. Nor is the lock only machine-generated: it was hand-refreshed twice inside + ordinary version-bump changes on 2026-08-23. + + The literal pin search cannot see this file: it writes ``specifier = ">=1.4.1,<1.5"``, with no + ``executorch==`` for the grep to match. + """ + lock = REPO_ROOT / "uv.lock" + if not lock.is_file(): + pytest.skip("no uv.lock in this checkout") + + recorded = set( + re.findall( + r'\{ name = "executorch", marker = "[^"]*", specifier = "([^"]+)" \}', + lock.read_text(encoding="utf-8"), + ) + ) + if not recorded: + pytest.skip("uv.lock records no executorch requirement") + + version = _versions()["__executorch_version__"] + major, minor = _release_line(version) + expected = f">={version},<{major}.{int(minor) + 1}" + if recorded == {expected}: + return + + # Behind the pin is the expected resting state until the lock is regenerated. Ahead of it is + # not: that means the lock names an ExecuTorch this repository does not pin. + from packaging.specifiers import SpecifierSet + from packaging.version import Version + + # Ahead means the range's own lower bound is above the pin. Probing the specifier with sample + # versions was fragile in both directions: an upper-bound test missed an open-ended ">=1.7", + # and a low sentinel called the ordinary behind-the-pin state a failure. + pinned = Version(version) + ahead = [ + entry + for entry in sorted(recorded) + if any( + clause.operator in {">=", ">", "==", "~=", "==="} + and Version(clause.version.rstrip(".*") or "0") > pinned + for clause in SpecifierSet(entry) + ) + ] + assert not ahead, ( + f"uv.lock records executorch {ahead}, which is ahead of the pinned {version}. The pin " + "derives " + + repr(expected) + + ", so run `uv lock --refresh` and commit the result." + ) + + +_INSTALL_INVOCATION = re.compile( + r"(?:python[0-9.]*\s+-m\s+pip|uv\s+pip|\bpip)\s+(?:install|wheel)\b" +) + + +def _blank_comments_preserving_length(block: str) -> str: + """``block`` with comment text replaced by spaces, keeping every offset and newline in place. + + Used only to locate pip keywords without a ``# pip install ...`` in a comment starting a false + invocation. Length is preserved so an offset into the original block indexes the same character + here. Whole-line ``#`` comments blank entirely; a trailing `` #`` comment blanks from the hash, + but a ``#cu130`` URL fragment (no preceding space) is left intact. + """ + out = [] + for physical in block.splitlines(keepends=True): + newline = "\n" if physical.endswith("\n") else "" + body = physical[:-1] if newline else physical + if body.lstrip().startswith("#"): + body = " " * len(body) + else: + hash_at = re.search(r"(?:^|\s)#", body) + if hash_at: + cut = hash_at.start() + body = body[:cut] + " " * (len(body) - cut) + out.append(body + newline) + return "".join(out) + + +def _install_invocation_window(block: str, match_offset: int) -> str: + """The slice of ``block`` belonging to the one pip/uv invocation that owns the match. + + The channel and the requirement it channels have to belong to the *same* invocation. A window + bounded only at blank lines was too wide: it spanned every step of a contiguous YAML job and + every line of a shell if/else, so a ``--extra-index-url`` from a neighbouring ``pip install``, + an ``echo``, or prose satisfied the check for an install that carried none of its own. Two real + holes this closed: the win32 branch of ``install-torch-tensorrt.sh`` borrowed the else branch's + URL, and the ``.[executorch]`` step in ``docgen.yml`` borrowed the *Install base deps* step's. + + An invocation runs from its ``pip``/``uv pip`` keyword to the next such keyword in the block, or + to the block's end. That single boundary spans a backslash-continued shell command, a YAML + ``run:`` body and a Python error message built from adjacent string fragments alike, because + none of those start a second invocation between the keyword and the URL. ``match_offset`` is a + ``block``-relative offset into the original (un-stripped) text, so a requirement that appears + twice in one block resolves to its own invocation rather than the first copy's. When no + invocation keyword precedes the match the whole block is returned, leaving non-install prose + matches to the caller's other filters. + """ + scan = _blank_comments_preserving_length(block) + starts = [m.start() for m in _INSTALL_INVOCATION.finditer(scan)] + preceding = [s for s in starts if s <= match_offset] + if not preceding: + return block + begin = preceding[-1] + following = [s for s in starts if s > match_offset] + finish = following[0] if following else len(block) + return block[begin:finish] + + +@pytest.mark.unit +def test_every_printed_install_instruction_names_the_nightly_channel(): + """Every ``[executorch]`` install instruction has to carry the nightly index. + + ExecuTorch is published only to the nightly CUDA channel, so an instruction without + ``--extra-index-url`` resolves nothing and the user gets a bare "no matching distribution". + The property had regressed and been re-fixed three times across this change with nothing + asserting it, which is the signature of a property no test covers. + + Whole files rather than single lines: every one of these instructions wraps, so the extra and + the index land on different lines and a line-oriented check sees neither together. Tracked + files only, so a stale build directory cannot fail this. + """ + tracked = subprocess.run( + ["git", "ls-files", "-z"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.split("\0") + + # Two shapes need the channel: an instruction naming the [executorch] extra, and the CI + # install of a locally built torch-tensorrt wheel, whose ExecuTorch dependency resolves from + # the same index. The second is the site that regressed most often and carries no extra. + # The local-path spelling counts too. Matching only the named-distribution form left the four + # sites that write "pip install .[executorch]" unguarded: the nightly index could be deleted + # from all four with this test green. + # A fourth shape: a direct "executorch==" or "executorch>=" in a pip command. The + # runtime README build recipe installs ExecuTorch this way, and its nightly index could be + # deleted with this test green because none of the three shapes above match a bare + # distribution name. Gated on a pip context below so a requirement in pyproject or a comment + # is not mistaken for an install instruction. + # The built-wheel shape covers both the plain "torch_tensorrt*.whl" and the runtime wheel + # "torch_tensorrt_executorch_runtime-*.whl": the latter's install_requires names the same + # nightly ExecuTorch, so its documented install needs the channel too, and matching only the + # plain glob left the runtime README's Use command unscanned. It is gated on a real + # "pip install" without "--no-deps" below, so naming the file in an "ls" or a heredoc, or a + # "--no-deps" install that fetches nothing, is not mistaken for a dependency-resolving install. + # A fifth shape: a bare "pip install executorch" with no version operator. It resolves the + # stable 1.4.1 from PyPI, the version this change moves away from, and carries no operator so + # the shapes above miss it. Matched as a standalone distribution token and gated on a + # pip-install context below, so the word in a path, an import, a filename, or prose is not + # mistaken for an install. Like the named-distribution form it needs both the channel and + # --pre, since a bare name without --pre picks the stable release even off the nightly index. + extra = re.compile( + r"""torch[-_]tensorrt\[[^]]*executorch[^]]*\]""" + r"""|(?=)""" + r"""|(?_-])""" + ) + missing = [] + for name in tracked: + base = name.rsplit("/", 1)[-1] + # Select by suffix, plus a few extensionless files that carry install commands. The + # justfile in particular writes a real "uv pip install ... executorch==" with its + # nightly index; filtering on suffix alone never read it, so both the index and the pin + # could be dropped from it with this test green. + if not base or not ( + name.endswith((".py", ".sh", ".md", ".yml", ".yaml", ".rst", ".txt")) + or base in _EXTENSIONLESS_INSTALL_FILES + ): + continue + # This file states the rule; it is not itself an instruction. + if name == "tests/py/dynamo/executorch/test_executorch_pin.py": + continue + # docs/ is Sphinx output committed to the tree. Its sources live in docsrc/, which is + # where a correction has to go, so flagging the generated copy sends the fix to a file + # the next docs build overwrites. + if name.startswith("docs/"): + continue + path = REPO_ROOT / name + if not path.is_file(): + continue + text = path.read_text(encoding="utf-8", errors="replace") + for match in extra.finditer(text): + line = text.count("\n", 0, match.start()) + 1 + # Bound the window at the surrounding blank-line-separated block first, then narrow to + # the single pip invocation that owns the match. The block alone was too wide: a + # contiguous YAML job or a shell if/else is one block, so a --extra-index-url, --pre or + # channel from a neighbouring command satisfied an install that carried none itself. + block_start = text.rfind("\n\n", 0, match.start()) + 1 + block_end = text.find("\n\n", match.end()) + block = text[block_start : block_end if block_end != -1 else len(text)] + block = _install_invocation_window(block, match.start() - block_start) + # Strip comments after windowing: a shell comment naming the channel or --pre is prose, + # not an argument pip sees, so a gutted install line with a decoy comment beside it must + # not satisfy the check. Whole-line comments drop entirely; a trailing "#" comment is + # cut, but not the "#cu130" fragment of a URL, which carries no space. + block = "\n".join( + re.sub(r"(?:^|\s)#.*$", "", bl) + for bl in block.splitlines() + if not bl.lstrip().startswith("#") + ) + # A plain torch-tensorrt wheel install on a platform with no ExecuTorch dev wheel is + # exempt. win32 installs a glob that also matches the Linux-only runtime wheel, so the + # scan reaches it, but ExecuTorch publishes no win32 nightly. The marker has to sit + # directly above the invocation, and a separate test keeps it inside a win32 guard, so a + # gutted Linux install cannot claim it. + if _has_marker_above(name, line, NO_NIGHTLY_MARKER): + continue + is_direct_install = bool( + re.fullmatch(r"executorch(?:_[a-z]+)?\s*(?:==|>=)", match.group(0)) + ) + # A bare "executorch==" only counts as an instruction inside a pip command. Anywhere + # else it is a dependency declaration or prose, guarded by other tests. + if is_direct_install: + if not re.search(r"\bpip\s+(?:install|wheel)\b", block): + continue + # The version after the operator has to be a literal. "executorch==${OLD}" or + # "executorch==$(...)" clears the pip-context gate yet pins nothing: pip installs + # whatever the expansion yields, which floats off the pin. The one legitimate + # non-literal is docgen's, which reads __executorch_version__ out of + # dev_dep_versions.yml; a separate test proves that command equals the pin. + after = text[match.end() : match.end() + 80].lstrip() + if not after[:1].isdigit() and not after.startswith( + "$(python3 -c 'import yaml;" + ): + missing.append( + f"{name}:{line} installs executorch at a non-literal version, which pins " + "nothing: pip resolves whatever the expansion yields off the nightly index" + ) + continue + is_bare_name = match.group(0) == "executorch" + if is_bare_name: + # A bare distribution name is an install only inside a pip command; the same word + # in a path, an import, or prose is not. + if not re.search(r"\bpip\s+(?:install|wheel)\b", block): + continue + # The word also appears in prose that shares a block with a real + # "torch_tensorrt[executorch]" install, so require the bare token to sit on the pip + # command line itself before treating it as the install target. A bare target there + # pins nothing even with --pre and the channel: pip resolves the newest nightly, not + # this pin. Nothing in the tree installs executorch bare, so it is always a defect. + command_line = text.splitlines()[line - 1] + if re.search(r"\bpip\s+(?:install|wheel)\b", command_line): + missing.append( + f"{name}:{line} installs executorch by bare name, which pins nothing: pip " + "resolves the newest nightly rather than the pinned version" + ) + continue + is_built_wheel = bool( + re.fullmatch( + r"torch_tensorrt(?:_executorch_runtime-)?\*\.whl", match.group(0) + ) + ) + # This glob names a local file, so it pulls the nightly ExecuTorch dependency only in a + # "pip install" that resolves dependencies. Naming the file elsewhere (an "ls", a + # heredoc, "pip wheel", or a "--no-deps" install) fetches no ExecuTorch, so no channel + # applies. + if is_built_wheel and ( + not re.search(r"\bpip\s+install\b", block) + or re.search(r"(?:^|\s)--no-deps(?:\s|$)", block) + ): + continue + channel = re.search( + r"download\.pytorch\.org/whl/nightly(?:/(cu\d+))?", block + ) + # CI passes the channel through a variable rather than a literal URL. Capture the + # variable name so its assignment can be resolved: accepting the reference on sight let + # the assignment be repointed at PyPI, or stripped of its nightly segment, with the + # install still counted as channelled. + variable_index = re.search( + r"--(?:extra-index-url|index-url)\s+\"?\$\{?([A-Za-z_][A-Za-z0-9_]*)", + block, + ) + if not channel and variable_index: + # Resolve the variable's last assignment before this install and check the channel + # there. An unresolved variable is accepted only when it is a known workflow input + # whose value arrives from the CI environment; every other unresolved name, a typo + # among them, fails rather than passing on sight. + assignment = _resolve_shell_assignment( + text, variable_index.group(1), match.start() + ) + if assignment is None: + if variable_index.group(1) in _ENVIRONMENT_INDEX_VARIABLES: + continue + missing.append( + f"{name}:{line} channels through ${{{variable_index.group(1)}}}, which has " + "no assignment in the tree and is not a known CI index input" + ) + continue + channel = re.search( + r"download\.pytorch\.org/whl/nightly(?:/(cu\d+))?", assignment + ) + if not channel: + missing.append( + f"{name}:{line} installs from ${{{variable_index.group(1)}}}, set to " + f"{assignment!r}, which is not the nightly channel" + ) + continue + if not channel: + missing.append(f"{name}:{line} names no nightly channel") + continue + # A substring proves a string sits nearby, not that it resolves anything. Rewriting + # every channel in the tree to a nonexistent cu999 left this green. Not compared + # against __cuda_version__: the runtime error messages derive their channel from the + # user's torch build, and the documented recipes name a concrete published channel that + # carries the pinned ExecuTorch, so a literal cuXYZ here is checked only for being one + # the project publishes. + suffix = channel.group(1) + if suffix and suffix not in _PUBLISHED_NIGHTLY_CHANNELS: + missing.append( + f"{name}:{line} installs from nightly/{suffix}, which the project does not " + f"publish for; expected one of {sorted(_PUBLISHED_NIGHTLY_CHANNELS)}" + ) + # The named-distribution form needs --pre. "torch-tensorrt[executorch]" with no + # version pin resolves to the stable PyPI wheel, which carries no executorch extra at + # all, so the command exits 0 with a warning and installs nothing the feature needs. + # The ".[executorch]" and built-wheel forms already pin executorch to a dev version, + # which enables prerelease selection on their own, so they do not need it. + named_distribution = re.fullmatch( + r"torch[-_]tensorrt\[[^]]*executorch[^]]*\]", match.group(0) + ) + if named_distribution and not re.search(r"(?:^|\s)--pre(?:\s|$)", block): + missing.append( + f"{name}:{line} installs {match.group(0)} without --pre, so pip resolves the " + "stable release with no executorch extra rather than the nightly prerelease" + ) + + assert not missing, ( + "these ExecuTorch install instructions do not name the nightly channel, so they " + f"resolve no ExecuTorch at all: {missing}" + ) + + +@pytest.mark.unit +def test_the_no_nightly_marker_only_exempts_a_win32_install(): + """The ``no-nightly`` exemption is legitimate only where no ExecuTorch dev wheel exists. + + The channel scan skips an install carrying ``pin-check: no-nightly`` above it. That is correct + for win32, whose wheel glob also matches the Linux-only runtime wheel while ExecuTorch ships no + win32 nightly. Without this test the marker is a blanket silencer: strip the index from the + Linux install, paste the marker above it, and the channel scan stays green. Requiring the marker + to sit inside a ``win32`` platform guard keeps the exemption tied to the one case it describes. + """ + tracked = subprocess.run( + ["git", "ls-files", "-z"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.split("\0") + + misplaced = [] + for name in tracked: + if not name or name == "tests/py/dynamo/executorch/test_executorch_pin.py": + continue + path = REPO_ROOT / name + if not path.is_file(): + continue + text = path.read_text(encoding="utf-8", errors="replace") + if NO_NIGHTLY_MARKER not in text: + continue + lines = text.splitlines() + control = re.compile(r"^\s*(?:if\b|elif\b|else\b|fi\b)") + for index, content in enumerate(lines): + if NO_NIGHTLY_MARKER not in content: + continue + # The exempted install sits just below the marker, so the branch it lives in is the + # nearest control-flow keyword above it. Requiring that keyword to be the win32 guard + # ties the exemption to the one platform it describes: a marker pasted onto a Linux + # "else" install resolves to that "else", not to "if ... win32", and is rejected. + branch = next( + (lines[j] for j in range(index - 1, -1, -1) if control.match(lines[j])), + "", + ) + if "win32" not in branch: + misplaced.append( + f"{name}:{index + 1} carries {NO_NIGHTLY_MARKER!r} outside a win32 branch, " + "so it would exempt a Linux install that simply lost its index" + ) + + assert not misplaced, ( + "the no-nightly exemption is only valid inside a win32 branch: " f"{misplaced}" + ) + + +@pytest.mark.unit +def test_the_pin_check_runs_in_ci(): + """This file has to be invoked by something, or its assertions never execute. + + Two suites deselect it by name so it does not need an installed ExecuTorch on a GPU runner, + which leaves the lint job as the only path that runs it. Deleting that step is invisible + otherwise: every test here still passes locally while nothing runs them in CI. + """ + # Parse the workflow and assert inside the owning job. Searching the file as one blob could + # not tell which job it was reading, so an identical install line in a sibling job that has no + # pin check satisfied it, and deleting the real one stayed green. A commented-out step also + # vanishes from the parse, where a text search still finds it. + import yaml + + workflow = yaml.safe_load( + (REPO_ROOT / ".github/workflows/linter.yml").read_text(encoding="utf-8") + ) + # The workflow must actually run on pull requests. PyYAML reads the unquoted "on" key as the + # boolean True (the YAML 1.1 "Norway problem"), so accept either spelling, then require a + # pull_request trigger. Reducing "on:" to workflow_dispatch left every string in place while + # the workflow never fired on a pull request. + triggers = workflow.get("on", workflow.get(True)) + trigger_names = set(triggers) if isinstance(triggers, (dict, list)) else {triggers} + assert "pull_request" in trigger_names, ( + f"linter.yml triggers on {sorted(map(str, trigger_names))}, not pull_request, so the pin " + "check never runs when a pull request changes the pin" + ) + # A paths filter on the trigger would keep the workflow from firing on a pin change outside + # those paths, leaving every assertion below green while nothing ran. + pull_request = triggers.get("pull_request") if isinstance(triggers, dict) else None + if isinstance(pull_request, dict): + for path_filter in ("paths", "paths-ignore"): + assert path_filter not in pull_request, ( + f"linter.yml narrows the pull_request trigger with {path_filter}, so a change to " + "the pin outside those paths would not run this check" + ) + # Match a live pytest invocation with pytest as the command word, not the filename anywhere + # in the script. A plain substring test was satisfied by a comment; an "anything before + # pytest" test was satisfied by "echo python3 -m pytest", which prints the command and runs + # nothing. + invocation = re.compile( + rf"^\s*(?:python[0-9.]*\s+-m\s+)?pytest\b[^\n]*{re.escape(pathlib.Path(__file__).name)}", + re.MULTILINE, + ) + owning = [ + (name, job, step) + for name, job in workflow["jobs"].items() + for step in job.get("steps", []) + if invocation.search(step.get("run") or "") + ] + assert owning, "no CI job invokes this file, so nothing here runs on a pull request" + name, job, step = owning[0] + + # A falsy condition disables the step or the whole job while leaving every string in place, so + # check both. GitHub treats a bare "false", "${{ false }}" and any always-false expression the + # same way, so restrict each to the small set of conditions that can actually be true. + live_conditions = {"always()", "success()", "success() || failure()"} + step_condition = str(step.get("if", "always()")) + assert ( + step_condition in live_conditions + ), f"the pin check step in {name} runs under {step_condition!r}, which may never be true" + job_condition = str(job.get("if", "always()")) + assert ( + job_condition in live_conditions + ), f"job {name} runs under {job_condition!r}, so the pin check may never dispatch" + + # Measure the effect rather than enumerating the spellings that defeat it. + # and shell shapes chased holes one at a time: -k, --co, --deselect, "; true", "set +e", a + # PYTEST_ADDOPTS in the env, an "if false" wrapper, an early "exit 0" each needed its own + # clause, and each clause missed a neighbour. Instead, run the step's own script against a + # stub test file and require its exit status to follow the test: non-zero when the stub fails, + # zero when it passes. Anything that discards the status, deselects the tests, or turns the + # run into a collect-only pass makes the failing case exit zero and fails this assertion. + def _run_ci_step_against(stub_body: str) -> int: + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + # The step invokes this file by its path; stand a stub with the same basename in a + # matching directory tree so the invocation resolves to the stub, not the real suite. + stub_dir = tmp_path / "tests/py/dynamo/executorch" + stub_dir.mkdir(parents=True) + (stub_dir / pathlib.Path(__file__).name).write_text( + stub_body, encoding="utf-8" + ) + # Run the exact script the step runs, under bash, with the step's own env overlaid. + # GITHUB_WORKSPACE points at the stub tree so "cd $GITHUB_WORKSPACE" lands there, and + # the interpreter is forced to this Python so the subprocess needs nothing installed. + env = {**os.environ, "GITHUB_WORKSPACE": str(tmp_path)} + for key, value in (step.get("env") or {}).items(): + env[str(key)] = str(value) + script = step["run"].replace( + "python3 -m pytest", f"{shlex.quote(sys.executable)} -m pytest" + ) + return subprocess.run( + ["bash", "-c", script], + env=env, + capture_output=True, + text=True, + ).returncode + + passing_stub = ( + "import pytest\n\n\n" + "@pytest.mark.unit\n" + "def test_stub_passes():\n" + " assert True\n" + ) + failing_stub = ( + "import pytest\n\n\n" + "@pytest.mark.unit\n" + "def test_stub_fails():\n" + " assert False, 'a pin inconsistency must fail the CI step'\n" + ) + assert _run_ci_step_against(passing_stub) == 0, ( + f"the pin check step in {name} does not exit zero when the pin tests pass, so it cannot " + "be trusted to gate on their result" + ) + assert _run_ci_step_against(failing_stub) != 0, ( + f"the pin check step in {name} exits zero even when a pin test fails, so a pin " + "inconsistency would not fail CI" + ) + + assert not step.get( + "continue-on-error" + ), f"the pin check step in {name} is continue-on-error, so a failure cannot fail the job" + assert not job.get( + "continue-on-error" + ), f"job {name} is continue-on-error, so a failed pin check cannot fail the workflow" + + # pytest and pyyaml must be installed by an earlier step of the SAME job: neither + # requirements.txt nor dependency-groups.lint carries them, and without them the step exits 1 + # on "No module named pytest" before running any assertion. + steps = job["steps"] + # Comment-stripped: a commented-out "uv pip install ... pytest pyyaml" still matched the raw + # text while installing nothing, so the step would die on a missing import at runtime. + earlier = "\n".join( + re.sub(r"(?:^|\s)#.*$", "", line) + for step_before in steps[: steps.index(step)] + for line in (step_before.get("run") or "").splitlines() + if not line.lstrip().startswith("#") + ) + for package in ("pytest", "pyyaml"): + assert re.search( + rf"uv pip install --system[^\n]*\b{package}\b", earlier + ), f"job {name} does not install {package} before the pin check, so the step cannot run" + + +@pytest.mark.unit +def test_the_pairing_check_survives_the_gpu_lane_deselection() -> None: + """The one test here that needs a real ExecuTorch must not be deselected with the rest. + + Every other test in this file is a source-consistency check, so the executorch tier deselects + the whole module by name to avoid paying for them twice. ``-k`` matches the module name in the + test id, so a bare ``not test_executorch_pin`` drops the pairing check too, and that check + only means anything where ExecuTorch is installed, which is nowhere the lint job runs. + + Two routes reach this tier: the nightly manifest suite in ``tests/ci/suites.py``, and + ``executorch-test-linux.yml``, which installs the pinned wheel and runs on pull requests once + the runtime build succeeds. Both go through one of the two keyword expressions checked here. + """ + # Run pytest's own collection under each expression rather than grepping for the name. A + # string test passes on "and" in place of "or", which collects nothing at all, and on the + # name surviving only in a comment. Both leave the pairing check unreachable. + module = pathlib.Path(__file__).name + for path, pattern in ( + ("tests/ci/suites.py", r'keyword=\(\s*(?:#[^\n]*\n\s*)*"([^"]+)"'), + # Anchored on the executorch junitxml name, because the file passes -k in several + # functions and the first match belongs to a different tier. + ( + "tests/py/utils/ci_helpers.sh", + r'executorch_tests_results[^\n]*?-k "([^"]+)"', + ), + ): + text = (REPO_ROOT / path).read_text(encoding="utf-8") + found = re.search(pattern, text) + assert ( + found + ), f"{path} no longer passes a single -k expression this test can read" + keyword = found.group(1) + assert "not test_executorch_pin" in keyword, ( + f"{path} no longer deselects this module, so the source-consistency checks here " + "would run twice" + ) + selected = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + str(pathlib.Path(__file__).parent), + "--collect-only", + "-q", + "--noconftest", + "-p", + "no:cacheprovider", + "-o", + "addopts=", + "-k", + keyword, + ], + capture_output=True, + text=True, + cwd=REPO_ROOT, + ).stdout + assert f"{module}::{PAIRING_TEST}" in selected, ( + f"{path} runs pytest with -k {keyword!r}, which does not select {PAIRING_TEST}, so " + "the only check that needs a real ExecuTorch installed runs nowhere" + ) + others = [ + line + for line in selected.splitlines() + if module in line and PAIRING_TEST not in line + ] + assert not others, ( + f"{path} selects {len(others)} other tests from this module, which the lane " + f"deselects deliberately: {others[:2]}" + ) + + # Proving the -k expression selects the pairing test says nothing about whether either route + # is actually wired to run it. Replacing the workflow's `trt_tier_executorch` call with `echo + # skipped`, or pointing the executorch suite at a lane name no runner requests, both leave the + # checks above green while the test runs nowhere. So assert each route reaches the tier. + import yaml + + workflow = yaml.safe_load( + (REPO_ROOT / ".github/workflows/executorch-test-linux.yml").read_text( + encoding="utf-8" + ) + ) + scripts = [ + step.get("with", {}).get("script", "") + for job in workflow["jobs"].values() + for step in job.get("steps", []) + ] + [job.get("with", {}).get("script", "") for job in workflow["jobs"].values()] + # The call must not discard its own exit status. "trt_tier_executorch || true", a trailing + # ";", "&" or a pipe each let the tier fail while the lane stays green, so reject any status + # operator after the call on its line. + tier_calls = [ + match + for script in scripts + for match in re.finditer( + r"^\s*trt_tier_executorch\b([^\n]*)", script, re.MULTILINE + ) + ] + assert tier_calls, ( + "executorch-test-linux.yml no longer calls trt_tier_executorch, so the pairing check " + "never runs on the GPU lane even though its -k expression would select it" + ) + assert any(not re.search(r"[|;&]", call.group(1)) for call in tier_calls), ( + "every trt_tier_executorch call in executorch-test-linux.yml discards its exit status " + "with a pipe, ';', '&' or '|| true', so a pairing failure cannot fail the lane" + ) + + # The manifest route: the executorch suite must exist and target a lane a runner requests. + # A typo in its lane tuple silently drops it from every matrix, which the suite-name check + # above cannot see. + import importlib + + suites = importlib.import_module("tests.ci.suites") + executorch_suite = next((s for s in suites.SUITES if s.name == "executorch"), None) + assert executorch_suite is not None, ( + "tests/ci/suites.py no longer defines an 'executorch' suite, so the manifest route to the " + "pairing check is gone" + ) + assert "nightly" in executorch_suite.lanes, ( + f"the executorch suite runs on lanes {executorch_suite.lanes!r}, none of which is the " + "nightly lane the GPU tier requests, so the pairing check runs nowhere" + ) diff --git a/tests/py/dynamo/executorch/test_update_executorch_pin.py b/tests/py/dynamo/executorch/test_update_executorch_pin.py new file mode 100644 index 0000000000..77595fe0c8 --- /dev/null +++ b/tests/py/dynamo/executorch/test_update_executorch_pin.py @@ -0,0 +1,255 @@ +"""The pin updater has to be trusted to run unattended and open a pull request, so its +parts are tested the way they fail in practice: version ordering that is not string order, +a wheel that forgot its provenance, and a rewrite that has to leave the tree in exactly the +state the pin guard demands. Every test runs the real function; none restates it. + +Text and metadata only, like test_executorch_pin.py: no network, no GPU, no ExecuTorch. The +two functions that reach the index (available_versions, wheel_git_version) are exercised +against captured output and a synthesized wheel, so this file runs on the lint runner. + +Every version here is a fake far-past date (year 2020) and every commit is an obvious +marker, never a real pin. The updater bumps the pin by replacing the old literal everywhere +it appears in the tree, so a real pin value living in this file would be rewritten by a +bump, quietly changing the fixtures. Synthetic values can never equal the live pin, so a +bump never touches this file. test_write_pins_updates_new_sites_but_never_its_own_source +holds that line. +""" + +from __future__ import annotations + +import importlib.util +import subprocess +import zipfile +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[4] +_SCRIPT = _REPO_ROOT / ".github" / "scripts" / "update_executorch_pin.py" +_SELF = "tests/py/dynamo/executorch/test_update_executorch_pin.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("update_executorch_pin", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +updater = _load() + + +# The shape of a `pip index versions executorch` run: dated nightlies of one line, out of +# order so a test that passes under string sorting still fails here. The dates are fake and +# far in the past so they can never equal the live pin. +_NIGHTLY_LIST = [ + "1.5.0.dev20200101+cu130", + "1.5.0.dev20200103+cu130", + "1.5.0.dev20200102+cu130", +] +# A stable channel carries finals and the occasional release candidate; the updater must +# take the newest final, not the newer-looking prerelease. The line is a fake 0.9 that +# ExecuTorch will never ship, so it cannot equal a real stable pin either. +_STABLE_LIST = ["0.9.1", "0.9.2", "0.9.3", "0.9.4rc1"] + + +def test_pick_target_nightly_takes_the_newest_date_not_the_longest_string() -> None: + # Distinct on purpose: if a tree-wide bump ever collapsed two of these into one, the + # list would stop testing ordering, so fail loudly the moment they are not unique. + assert len(set(_NIGHTLY_LIST)) == len(_NIGHTLY_LIST) + assert updater.pick_target(_NIGHTLY_LIST, "nightly") == "1.5.0.dev20200103" + + +def test_pick_target_strips_the_cuda_local_label() -> None: + # The pin serves every CUDA row, so the +cuXXX label the index carries must not survive + # into the pin. A label left on would fail the guard's exact-match on every site. + assert "+" not in updater.pick_target(_NIGHTLY_LIST, "nightly") + + +def test_pick_target_stable_ignores_dev_and_release_candidates() -> None: + assert updater.pick_target(_STABLE_LIST, "stable") == "0.9.3" + + +def test_pick_target_nightly_ignores_finals() -> None: + # A stable final on the nightly index is not a nightly; picking it would move the pin + # off the dev line the delegate is built against. + assert updater.pick_target(["0.9.3", "1.5.0.dev20200103"], "nightly") == ( + "1.5.0.dev20200103" + ) + + +def test_pick_target_nightly_ignores_release_candidates() -> None: + # An rc is a prerelease that sorts above every dev of the same line under PEP 440, so a + # filter that only excluded finals would let the first rc on the nightly index become the + # pin. A nightly is a dated dev build, and an rc is not one. + assert ( + updater.pick_target(["1.5.0.dev20200103+cu130", "1.5.0rc1+cu130"], "nightly") + == "1.5.0.dev20200103" + ) + + +def test_pick_target_raises_when_nothing_matches_the_track() -> None: + with pytest.raises(SystemExit): + updater.pick_target(["0.9.3", "0.9.2"], "nightly") + + +def test_available_versions_parses_the_pip_line(monkeypatch) -> None: + captured = ( + "executorch (1.5.0.dev20200103+cu130)\n" + "Available versions: 1.5.0.dev20200103+cu130, 1.5.0.dev20200102+cu130\n" + " INSTALLED: 1.1.0\n" + " LATEST: 1.5.0.dev20200103+cu130\n" + ) + monkeypatch.setattr(updater, "_run", lambda cmd: captured) + assert updater.available_versions([]) == [ + "1.5.0.dev20200103+cu130", + "1.5.0.dev20200102+cu130", + ] + + +def _synthesize_wheel(path: Path, body: str) -> None: + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("executorch/version.py", body) + + +def test_wheel_git_version_reads_the_recorded_commit(monkeypatch, tmp_path) -> None: + commit = "deadbeef" * 5 + wheel = tmp_path / "executorch-1.5.0.dev20200103-py3-none-any.whl" + _synthesize_wheel( + wheel, f'__version__ = "1.5.0.dev20200103"\ngit_version = "{commit}"\n' + ) + + def fake_download(cmd): + # The download call writes the wheel into the temp dir named right after --dest. + dest = Path(cmd[cmd.index("--dest") + 1]) + (dest / wheel.name).write_bytes(wheel.read_bytes()) + return "" + + monkeypatch.setattr(updater, "_run", fake_download) + assert updater.wheel_git_version("1.5.0.dev20200103", []) == commit + + +def test_wheel_git_version_rejects_a_wheel_without_provenance( + monkeypatch, tmp_path +) -> None: + # ExecuTorch writes git_version = None when built outside a git checkout. Such a wheel + # carries nothing the pins can be checked against, so it must not become a pin. + wheel = tmp_path / "executorch-1.5.0.dev20200103-py3-none-any.whl" + _synthesize_wheel(wheel, "__version__ = '1.5.0.dev20200103'\ngit_version = None\n") + + def fake_download(cmd): + dest = Path(cmd[cmd.index("--dest") + 1]) + (dest / wheel.name).write_bytes(wheel.read_bytes()) + return "" + + monkeypatch.setattr(updater, "_run", fake_download) + with pytest.raises(SystemExit): + updater.wheel_git_version("1.5.0.dev20200103", []) + + +def test_upper_bound_is_the_next_minor_of_the_line() -> None: + # A nightly and a final both belong to the release line their first two fields name, so + # the bound is the next minor either way. This mirrors the guard's own derivation, and + # the 0.9 case checks the minor rolls to a two-digit number rather than to "0.:". + assert updater._upper_bound("1.5.0.dev20200103") == "1.6" + assert updater._upper_bound("0.9.1") == "0.10" + assert updater._upper_bound("7.3.0") == "7.4" + + +def _worktree(tmp_path) -> Path: + """A throwaway checkout of the repo so write_pins edits a real tree, not a copy that + diverges from what git tracks. write_pins walks `git ls-files`, so the tree has to be a + real checkout.""" + work = tmp_path / "repo" + subprocess.run( + ["git", "clone", "--quiet", "--no-hardlinks", str(_REPO_ROOT), str(work)], + check=True, + ) + return work + + +# A synthetic bump target for the write_pins tests: a fake far-past nightly and an obvious +# marker commit, distinct from any live pin so the bump is a real change but never collides +# with a real value in the tree. +_FAKE_VERSION = "1.5.0.dev20200103" +_FAKE_COMMIT = "deadbeef" * 5 + + +def test_write_pins_leaves_a_tree_the_guard_accepts(tmp_path, monkeypatch) -> None: + # The one test that matters most: after a bump, the whole pin guard has to pass, because + # that guard is what the generated pull request will be judged by. A rewrite that the + # guard rejects would open a red pull request every night. + work = _worktree(tmp_path) + monkeypatch.setattr(updater, "_REPO_ROOT", work) + monkeypatch.setattr(updater, "_VERSIONS_FILE", work / "dev_dep_versions.yml") + + assert updater.write_pins(_FAKE_VERSION, _FAKE_COMMIT) is True + assert updater.read_pin("__executorch_version__") == _FAKE_VERSION + assert updater.read_pin("__executorch_commit__") == _FAKE_COMMIT + + import sys + + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "tests/py/dynamo/executorch/test_executorch_pin.py", + "-q", + "--no-header", + "-p", + "no:cacheprovider", + "--noconftest", + "-o", + "addopts=", + ], + cwd=work, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_write_pins_is_idempotent_when_already_current(tmp_path, monkeypatch) -> None: + # A day with no new nightly must rewrite nothing, so the run opens no pull request. The + # updater is safe to run every day and by hand. + work = _worktree(tmp_path) + monkeypatch.setattr(updater, "_REPO_ROOT", work) + monkeypatch.setattr(updater, "_VERSIONS_FILE", work / "dev_dep_versions.yml") + + current_version = updater.read_pin("__executorch_version__") + current_commit = updater.read_pin("__executorch_commit__") + assert updater.write_pins(current_version, current_commit) is False + + diff = subprocess.run( + ["git", "-C", str(work), "diff", "--quiet"], capture_output=True + ) + assert diff.returncode == 0, "write_pins changed the tree when the pin was current" + + +def test_write_pins_updates_new_sites_but_never_its_own_source( + tmp_path, monkeypatch +) -> None: + # The updater rewrites by content, not a fixed list, so a brand new pin site is caught + # with no code change. The flip side is that a real pin literal in the updater's own + # source or test would be rewritten too. Both properties are pinned here: a new site is + # updated, and the updater's two files come out byte for byte identical. + work = _worktree(tmp_path) + monkeypatch.setattr(updater, "_REPO_ROOT", work) + monkeypatch.setattr(updater, "_VERSIONS_FILE", work / "dev_dep_versions.yml") + + old_version = updater.read_pin("__executorch_version__") + new_site = work / "some_new_requirements.txt" + new_site.write_text(f"executorch=={old_version}\n") + subprocess.run(["git", "-C", str(work), "add", str(new_site)], check=True) + + script = work / ".github" / "scripts" / "update_executorch_pin.py" + test = work / _SELF + script_before = script.read_bytes() + test_before = test.read_bytes() + + assert updater.write_pins(_FAKE_VERSION, _FAKE_COMMIT) is True + + assert new_site.read_text() == f"executorch=={_FAKE_VERSION}\n" + assert script.read_bytes() == script_before + assert test.read_bytes() == test_before diff --git a/tests/py/utils/ci_helpers.sh b/tests/py/utils/ci_helpers.sh index 7f3b4e5861..ff24cbf2e7 100755 --- a/tests/py/utils/ci_helpers.sh +++ b/tests/py/utils/ci_helpers.sh @@ -159,8 +159,12 @@ trt_tier_l2_dynamo_core() { } trt_tier_executorch() { + # The pin checks are excluded here because the lint workflow already runs them on a CPU + # runner; this tier needs a GPU and a built wheel, which they do not. The one exception is + # kept by name: it compares the pinned commit against the installed wheel's own recorded + # source, so it needs an ExecuTorch the lint runner does not have and skips everywhere else. ( cd "${TRT_REPO_ROOT}/tests/py/dynamo" - _trt_py -m pytest -ra $(_trt_nproc auto) --junitxml="$(_trt_xml executorch_tests_results)" executorch/ "$@" ) + _trt_py -m pytest -ra $(_trt_nproc auto) --junitxml="$(_trt_xml executorch_tests_results)" -k "not test_executorch_pin or test_the_pinned_commit_is_the_pinned_wheels_own_source" executorch/ "$@" ) } trt_tier_l2_plugin() { diff --git a/toolchains/ci_workspaces/MODULE.bazel.tmpl b/toolchains/ci_workspaces/MODULE.bazel.tmpl index 796f714375..124e03199d 100644 --- a/toolchains/ci_workspaces/MODULE.bazel.tmpl +++ b/toolchains/ci_workspaces/MODULE.bazel.tmpl @@ -214,8 +214,8 @@ new_local_repository( new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # executorch==1.4.1 - commit = "e4d02f41f7909e8ed5bf4a14ffc520d733453d9f", + # executorch==1.5.0.dev20260825 + commit = "817929b7fb8d162d80eb9299d6630c35a3106979", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;",