Track the ExecuTorch pin and move it to new nightlies automatically

.github/scripts/install-torch-tensorrt.sh

@@ -52,9 +52,24 @@ fi
 
 # Install Torch-TensorRT
 if [[ ${PLATFORM} == win32 ]]; then
+    # pin-check: no-nightly -- this glob also matches the Linux-only ExecuTorch runtime wheel, but
+    # ExecuTorch publishes no win32 nightly and the [executorch] extra is Linux-only, so this
+    # platform's plain torch-tensorrt install needs no nightly index.
     python -m pip install ${RUNNER_ARTIFACT_DIR}/torch_tensorrt*.whl
 else
-    python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl --use-deprecated=legacy-resolver
+    # The nightly channel is needed because this glob also matches the ExecuTorch runtime wheel,
+    # whose install_requires names an ExecuTorch dev build that is published only there.
+    # Hardcoded rather than ${CHANNEL} like the lines above: a .dev wheel exists on no other
+    # channel, so deriving it would break this install on exactly the test and release runs the
+    # index was added for. It is an extra index, not a replacement, and torch is already
+    # force-reinstalled from ${INDEX_URL} above, so the pinned torch is not at risk from it.
+    # || exit 1 because line 1's `set -exou pipefail` is commented out and linux-test.yml
+    # concatenates this file ahead of the user script, so a failure here would otherwise be
+    # discarded and the job would die later with an unrelated-looking ImportError. Scoped to the
+    # line this change is responsible for; re-enabling set -e for the whole file is a
+    # pre-existing hazard worth a separate change.
+    python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl --use-deprecated=legacy-resolver \
+        --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" || exit 1
 fi
 
 echo -e "Running test script";

.github/scripts/update_executorch_pin.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""Move the ExecuTorch pin to the newest wheel published on an index.
+
+The pin is two coupled facts, spread across the tree but sourced from
+``dev_dep_versions.yml``: ``__executorch_version__`` selects the wheel the delegate is
+built to sit beside, and ``__executorch_commit__`` selects the tree it compiles from.
+They must name one ExecuTorch, so this script never guesses the commit: it reads it from
+the chosen wheel's own ``executorch/version.py``, which is the same provenance
+``tests/py/dynamo/executorch/test_executorch_pin.py`` checks the pins against.
+
+The daily workflow runs this, then opens a pull request when the pin moved. The existing
+pin guards and the executorch end-to-end lane run on that pull request, so "the newest
+wheel that actually works" is decided by the same gate a human bump goes through, not
+re-implemented here. A nightly that did not publish leaves the newest version unchanged,
+so the run rewrites nothing and opens nothing.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import subprocess
+import sys
+import tempfile
+import zipfile
+from pathlib import Path
+
+from packaging.version import InvalidVersion, Version
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+_VERSIONS_FILE = _REPO_ROOT / "dev_dep_versions.yml"
+
+# The pin literal is rewritten everywhere it appears except this file, which carries the
+# version only as a docstring example, not a pin. Every real site is checked by the guard,
+# so an accidental new site fails that test rather than being silently missed here.
+_SELF = "tests/py/dynamo/executorch/test_executorch_pin.py"
+
+# A wheel version on the PyTorch index carries a local label naming its CUDA build, for
+# example ``1.5.0.devYYYYMMDD+cu130``. The pin omits it so one pin serves every CUDA row.
+_LOCAL_LABEL = re.compile(r"\+.*$")
+
+
+def _run(cmd: list[str]) -> str:
+    return subprocess.run(cmd, check=True, capture_output=True, text=True).stdout
+
+
+def read_pin(field: str) -> str:
+    """Return a pinned value from ``dev_dep_versions.yml``."""
+    text = _VERSIONS_FILE.read_text(encoding="utf-8")
+    match = re.search(rf'^{field}:\s*"?([^"\s]+)"?\s*$', text, re.MULTILINE)
+    if match is None:
+        raise SystemExit(f"{field} is not set in {_VERSIONS_FILE.name}")
+    return match.group(1)
+
+
+def available_versions(index_args: list[str]) -> list[str]:
+    """Every ExecuTorch version the index offers.
+
+    ``pip index versions`` prints one ``Available versions:`` line. Parsing that is stable
+    and needs no network code here; the workflow passes the index the same way every other
+    install in the tree does.
+    """
+    out = _run(
+        [sys.executable, "-m", "pip", "index", "versions", "executorch", *index_args]
+    )
+    match = re.search(r"^\s*Available versions:\s*(.+)$", out, re.MULTILINE)
+    if match is None:
+        raise SystemExit("pip index versions printed no Available versions line")
+    return [v.strip() for v in match.group(1).split(",") if v.strip()]
+
+
+def pick_target(versions: list[str], track: str) -> str:
+    """The newest version on the wanted track.
+
+    ``nightly`` takes the newest dated dev build; ``stable`` takes the newest final
+    release, ignoring dev builds and release candidates. Ordering is PEP 440, not string
+    order, so a newer dev date on the same line sorts above an older one correctly.
+    """
+    parsed: list[tuple[Version, str]] = []
+    for raw in versions:
+        try:
+            version = Version(raw)
+        except InvalidVersion:
+            continue
+        # A nightly is a dated dev build. is_prerelease is also true for release
+        # candidates, and an rc sorts above every dev of the same line under PEP 440, so
+        # filtering on it would let the first rc on the index silently become the nightly
+        # pin. Match the dev segment itself instead.
+        if track == "nightly" and version.dev is None:
+            continue
+        if track == "stable" and (version.is_prerelease or version.is_devrelease):
+            continue
+        parsed.append((version, raw))
+    if not parsed:
+        raise SystemExit(f"no executorch version on the index matches track {track!r}")
+    newest = max(parsed, key=lambda pair: pair[0])[1]
+    return _LOCAL_LABEL.sub("", newest)
+
+
+def wheel_git_version(version: str, index_args: list[str]) -> str:
+    """The source commit the chosen wheel records for itself.
+
+    Every published wheel writes ``git_version`` into ``executorch/version.py``. Reading it
+    from the wheel is what keeps the two pins naming one ExecuTorch. A wheel built without
+    git provenance records ``None`` and must not become a pin, so that is an error, not a
+    guess.
+    """
+    with tempfile.TemporaryDirectory() as tmp:
+        _run(
+            [
+                sys.executable,
+                "-m",
+                "pip",
+                "download",
+                "--no-deps",
+                "--dest",
+                tmp,
+                f"executorch=={version}",
+                *index_args,
+            ]
+        )
+        wheels = list(Path(tmp).glob("executorch-*.whl"))
+        if not wheels:
+            raise SystemExit(
+                f"pip download produced no wheel for executorch=={version}"
+            )
+        with zipfile.ZipFile(wheels[0]) as archive:
+            source = archive.read("executorch/version.py").decode("utf-8")
+    match = re.search(r"""git_version[^=]*=\s*['"]([0-9a-f]{40})['"]""", source)
+    if match is None:
+        raise SystemExit(
+            f"executorch=={version} records no source commit, so the pin would name a "
+            "wheel whose provenance cannot be checked"
+        )
+    return match.group(1)
+
+
+def _upper_bound(version: str) -> str:
+    """The exclusive upper bound a range site pairs with the pin, next minor of its line.
+
+    ``tests/py/dynamo/executorch/test_executorch_pin.py`` derives the same bound from the
+    same two fields, so a range this writes and the range the guard expects agree by the
+    same rule rather than by coincidence.
+    """
+    major, minor = version.split(".")[:2]
+    return f"{major}.{int(minor) + 1}"
+
+
+def _iter_tracked_files() -> list[Path]:
+    names = _run(["git", "-C", str(_REPO_ROOT), "ls-files"]).splitlines()
+    return [_REPO_ROOT / name for name in names if name != _SELF]
+
+
+def write_pins(new_version: str, new_commit: str) -> bool:
+    """Rewrite every pin occurrence to the new version and commit.
+
+    The old version and commit are unique, high-entropy tokens, so a literal replacement
+    cannot touch anything that is not already a pin. The range form is re-rendered after,
+    so its upper bound tracks the new line. Returns whether anything changed.
+    """
+    old_version = read_pin("__executorch_version__")
+    old_commit = read_pin("__executorch_commit__")
+    if (new_version, new_commit) == (old_version, old_commit):
+        return False
+
+    # Anchored on >= and ,< rather than on a package name, so this rewrites the range
+    # wherever it appears: the executorch>=X,<Y in setup.py and the prefixless
+    # specifier = ">=X,<Y" uv.lock records for the same requirement. A name-prefixed anchor
+    # would miss the lock, and the bare-version pass below would then move the lower bound
+    # while leaving the upper bound behind, which breaks the moment the pin crosses a minor.
+    old_range = f">={old_version},<{_upper_bound(old_version)}"
+    new_range = f">={new_version},<{_upper_bound(new_version)}"
+
+    changed = False
+    for path in _iter_tracked_files():
+        try:
+            text = path.read_text(encoding="utf-8")
+        except (UnicodeDecodeError, FileNotFoundError):
+            continue
+        if old_version not in text and old_commit not in text:
+            continue
+        # The range carries the old version too, so rewrite it first, then the remaining
+        # bare version occurrences, so the range's own version is not rewritten twice.
+        updated = text.replace(old_range, new_range)
+        updated = updated.replace(old_version, new_version)
+        updated = updated.replace(old_commit, new_commit)
+        if updated != text:
+            path.write_text(updated, encoding="utf-8")
+            changed = True
+
+    if read_pin("__executorch_version__") != new_version:
+        raise SystemExit(
+            "the version pin did not take; dev_dep_versions.yml is unchanged"
+        )
+    return changed
+
+
+def _index_args(track: str, channel: str) -> list[str]:
+    if track == "nightly":
+        return [
+            "--pre",
+            "--index-url",
+            f"https://download.pytorch.org/whl/nightly/{channel}",
+        ]
+    return []
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--track", choices=("nightly", "stable"), default="nightly")
+    parser.add_argument(
+        "--channel",
+        default="cu130",
+        help="nightly CUDA channel to read versions and provenance from",
+    )
+    args = parser.parse_args(argv)
+
+    index_args = _index_args(args.track, args.channel)
+    target = pick_target(available_versions(index_args), args.track)
+    current = read_pin("__executorch_version__")
+    if target == current:
+        print(f"executorch pin is already at the newest {args.track} version {current}")
+        return 0
+
+    commit = wheel_git_version(target, index_args)
+    if write_pins(target, commit):
+        print(f"moved executorch pin {current} -> {target} (commit {commit})")
+    else:
+        print("nothing to write")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())