From a8bd0b603e3ea31e2dcb6fc5925b841b4ec045c9 Mon Sep 17 00:00:00 2001 From: Zingo Andersen Date: Thu, 22 Oct 2026 16:45:04 +0200 Subject: [PATCH 1/2] ci: consolidate pinned PyTorch installation Share the pinned PyTorch installer between Docker and CI setup. Reuse matching builds and preserve the macOS wheel cache. Resolve named refs without cloning and install domain packages even when the pinned torch build is reused. Verify the exact PyTorch pin and LAPACK support after Linux and macOS dependency installation. Add mocked installer tests. Authored with Codex. Signed-off-by: Zingo Andersen Change-Id: Ia35463d87c8757b001c51da72839a3c4e20c77eb --- .ci/docker/README.md | 38 ++ .ci/docker/common/install_pinned_pytorch.sh | 194 ++++++++++ .ci/docker/common/install_pytorch.sh | 87 +---- .ci/docker/ubuntu/Dockerfile | 3 +- .ci/scripts/install_pinned_pytorch.sh | 10 + .ci/scripts/setup-linux.sh | 3 + .ci/scripts/setup-macos.sh | 7 +- .../tests/test_install_pinned_pytorch.py | 346 ++++++++++++++++++ .ci/scripts/utils.sh | 162 -------- 9 files changed, 606 insertions(+), 244 deletions(-) create mode 100755 .ci/docker/common/install_pinned_pytorch.sh create mode 100755 .ci/scripts/install_pinned_pytorch.sh create mode 100644 .ci/scripts/tests/test_install_pinned_pytorch.py diff --git a/.ci/docker/README.md b/.ci/docker/README.md index 267eff4b0b1..d2353c31bdf 100644 --- a/.ci/docker/README.md +++ b/.ci/docker/README.md @@ -32,3 +32,41 @@ torch identity and schema-1.0 reporter after installing the domain libraries. These pins participate in the existing global Linux image hash. A pin change can require other Linux images to rebuild, even though only clang12 reads it. Validate the rebuilt image before enabling test-report uploads. + +## Reusing the pinned PyTorch installer + +From an ExecuTorch checkout, activate a venv or conda environment and run: + +```bash +bash .ci/scripts/install_pinned_pytorch.sh +bash .ci/scripts/install_pinned_pytorch.sh --verify-only +``` + +The installer reads `.ci/docker/ci_commit_pins/pytorch.txt`, resolves named refs, +and reuses an installed build with the same commit. Set `PYTHON_EXECUTABLE` to +select another interpreter, or `TORCH_VERSION` to override the source ref as the +Docker build does. A mismatched build is replaced, including when downgrading. +Install mode also installs the domain packages when the torch build is reused. +`--verify-only` checks the installed PyTorch commit and LAPACK support, returning +nonzero on a mismatch or failed check. It does not install packages or change the +active environment. Named refs are resolved with `git ls-remote`, without cloning. +This is not a check of every ExecuTorch build or test dependency. + +Use it inside a CI container to check that its prebuilt PyTorch matches the pin +in the current checkout, or locally to check the selected venv/conda environment +before running tests: + +```bash +PYTHON_EXECUTABLE="$PWD/env/bin/python" \ + bash .ci/scripts/install_pinned_pytorch.sh --verify-only +``` + +It can also run after dependency installation to detect replacement of the +pinned build. Linux and macOS CI verify after installing ExecuTorch, before +building the test runner. On Linux this also checks that the Docker image +provides the commit required by the checkout. + +The shared implementation lives in `common/install_pinned_pytorch.sh` so it is +also available in the Docker build context. Docker and macOS CI use this same +implementation. System build dependencies (including BLAS/LAPACK) must already +be installed. Windows is not supported. diff --git a/.ci/docker/common/install_pinned_pytorch.sh b/.ci/docker/common/install_pinned_pytorch.sh new file mode 100755 index 00000000000..3231bd85894 --- /dev/null +++ b/.ci/docker/common/install_pinned_pytorch.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Called directly or through .ci/scripts/install_pinned_pytorch.sh. +# Activate a venv/conda environment first, or set PYTHON_EXECUTABLE. +set -euo pipefail + +dedupe_macos_loader_path_rpaths() { + if [[ "$(uname)" != "Darwin" ]]; then + return + fi + + local torch_lib_dir + torch_lib_dir=$(cd / && "${python_executable}" -c "import importlib.util; print(importlib.util.find_spec('torch').submodule_search_locations[0])")/lib + + if [[ -z "${torch_lib_dir}" || ! -d "${torch_lib_dir}" ]]; then + return + fi + + local torch_libs=( + "libtorch_cpu.dylib" + "libtorch.dylib" + "libc10.dylib" + ) + + for lib_name in "${torch_libs[@]}"; do + local lib_path="${torch_lib_dir}/${lib_name}" + if [[ ! -f "${lib_path}" ]]; then + continue + fi + + local removed=0 + # Repeatedly remove the @loader_path rpath entries until none remain. + while install_name_tool -delete_rpath @loader_path "${lib_path}" 2>/dev/null; do + removed=1 + done + + if [[ "${removed}" == "1" ]]; then + install_name_tool -add_rpath @loader_path "${lib_path}" || true + fi + done +} + +verify_only=false +if [[ "${1:-}" == "--verify-only" ]]; then + verify_only=true + shift +fi +if [[ $# -gt 1 || "${1:-}" == -* ]]; then + echo "Usage: $0 [--verify-only] [EXECUTORCH_DIRECTORY]" >&2 + exit 2 +fi +executorch_dir=$(cd "${1:-.}" && pwd) +python_executable=${PYTHON_EXECUTABLE:-python} +# Resolve the interpreter before entering the temporary source directory. +python_executable=$("${python_executable}" -c 'import sys; print(sys.executable)') +system_name=$(uname) +case "${system_name}" in + Linux | Darwin) ;; + *) echo "Unsupported system: ${system_name}" >&2; exit 1 ;; +esac + +torch_ref=${TORCH_VERSION:-$(cat "${executorch_dir}/.ci/docker/ci_commit_pins/pytorch.txt")} +torch_ref=$(printf '%s' "${torch_ref}" | tr -d '[:space:]') +git check-ref-format --branch "${torch_ref}" >/dev/null +installed_pin=$(cd / && "${python_executable}" -c \ + 'import torch; print(torch.version.git_version or "")' 2>/dev/null || true) +expected_pin=${torch_ref} + +verify_pin() { + if [[ "${installed_pin}" != "${expected_pin}" ]]; then + echo "PyTorch pin mismatch: expected ${expected_pin}, found ${installed_pin:-none}" >&2 + return 1 + fi + (cd / && "${python_executable}" -c ' +import torch +assert torch._C.has_lapack, "PyTorch was built without LAPACK" +torch.linalg.qr(torch.randn(4, 4)) +') +} + +install_domains() { + "${python_executable}" -m pip install numpy pillow + "${python_executable}" -m pip install --no-build-isolation --no-deps \ + "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION:-release/2.11}" \ + "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION:-release/0.29}" +} + +if [[ ! "${torch_ref}" =~ ^[0-9a-f]{40}$ ]]; then + expected_pin=$(git ls-remote --exit-code https://github.com/pytorch/pytorch.git \ + "${torch_ref}" "${torch_ref}^{}" | awk ' + { sub(/\^\{\}$/, "", $2); pins[$2] = $1 } + END { for (ref in pins) print pins[ref] } + ' | sort -u) || expected_pin="" + if [[ ! "${expected_pin}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Cannot resolve PyTorch ref to one commit: ${torch_ref}" >&2 + exit 1 + fi +fi + +if [[ "${installed_pin}" == "${expected_pin}" || "${verify_only}" == true ]]; then + verify_pin + echo "PyTorch pin ${expected_pin} is already installed" + if [[ "${verify_only}" == false ]]; then + dedupe_macos_loader_path_rpaths + install_domains + fi + exit 0 +fi + +build_root=$(mktemp -d "${TMPDIR:-/tmp}/executorch-pytorch.XXXXXXXX") +trap ' + status=$? + if [[ ${status} -ne 0 ]]; then + echo "Failed to prepare the replacement PyTorch wheel." >&2 + echo "Existing torch and domain packages have not been removed." >&2 + fi + rm -rf "${build_root}" + exit "${status}" +' EXIT +git clone --filter=blob:none --no-checkout https://github.com/pytorch/pytorch.git "${build_root}/pytorch" +cd "${build_root}/pytorch" +git fetch --depth=1 origin "${expected_pin}" +git checkout --detach FETCH_HEAD + +cached=false +if [[ "${system_name}" == Darwin ]]; then + platform=$("${python_executable}" -c 'import sysconfig; import platform; v=platform.mac_ver()[0].split(".")[0]; platform=sysconfig.get_platform().split("-"); platform[1]=f"{v}_0"; print("_".join(platform))') + python_version=$("${python_executable}" -c 'import platform; v=platform.python_version_tuple(); print(f"{v[0]}{v[1]}")') + torch_release=$(cat version.txt) + torch_short_hash=$(git rev-parse --short=7 HEAD) + torch_wheel_path="cached_artifacts/pytorch/executorch/pytorch_wheels/${system_name}/${python_version}" + torch_wheel_name="torch-${torch_release}+git${torch_short_hash}-cp${python_version}-cp${python_version}-${platform}.whl" + cached_torch_wheel="https://gha-artifacts.s3.us-east-1.amazonaws.com/${torch_wheel_path}/${torch_wheel_name//+/%2B}" + if "${python_executable}" -m pip download --no-deps --dest dist "${cached_torch_wheel}"; then + cached=true + fi +fi + +if [[ "${cached}" == false ]]; then + echo "Building PyTorch at ${expected_pin}" + git submodule update --init --recursive + if [[ "$(uname -m)" == aarch64 ]]; then + export BUILD_IGNORE_SVE_UNAVAILABLE=1 + fi + if [[ "${system_name}" == Linux ]]; then + export _GLIBCXX_USE_CXX11_ABI=1 + fi + if [[ -n "${PYTORCH_BUILD_MAX_JOBS:-}" ]]; then + export MAX_JOBS="${PYTORCH_BUILD_MAX_JOBS}" + fi + build_venv="${build_root}/build-env" + "${python_executable}" -m venv --system-site-packages "${build_venv}" + # Keep in sync with pytorch/pyproject.toml [build-system].requires. + # Do not install pip cmake: it hides the image's MKL/libomp search paths. + "${build_venv}/bin/python" -m pip install build "scikit-build-core>=1.0" \ + ninja "packaging>=24.2" "typing-extensions>=4.10.0" pyyaml six numpy + CMAKE_CXX_SCAN_FOR_MODULES=OFF USE_DISTRIBUTED=1 \ + "${build_venv}/bin/python" -m build --wheel --no-isolation +fi + +shopt -s nullglob +wheels=(dist/*.whl) +shopt -u nullglob +if [[ ${#wheels[@]} -ne 1 ]]; then + echo "Expected one PyTorch wheel, found ${#wheels[@]}" >&2 + exit 1 +fi +if [[ "${system_name}" == Darwin ]]; then + if [[ "$(basename "${wheels[0]}")" != "${torch_wheel_name}" ]]; then + echo "Built PyTorch wheel does not match cache key: ${torch_wheel_name}" >&2 + exit 1 + fi +fi + +trap 'rm -rf "${build_root}"' EXIT +# Remove the old builds explicitly so equal package versions also replace +# different source commits, and domain dependencies cannot replace the pin. +"${python_executable}" -m pip uninstall -y torch torchaudio torchvision +"${python_executable}" -m pip install "${wheels[0]}" + +dedupe_macos_loader_path_rpaths +install_domains +installed_pin=$(cd / && "${python_executable}" -c 'import torch; print(torch.version.git_version or "")') +verify_pin + +# Preserve the existing macOS wheel cache on AWS CI runners. +if [[ "${system_name}" == Darwin && "${cached}" == false && "${GITHUB_ACTIONS:-}" == true && -z "${GITHUB_RUNNER:-}" ]] && command -v aws >/dev/null; then + aws s3 cp "${wheels[0]}" "s3://gha-artifacts/${torch_wheel_path}/${torch_wheel_name}" +fi +sccache --show-stats || true diff --git a/.ci/docker/common/install_pytorch.sh b/.ci/docker/common/install_pytorch.sh index d008f0773fd..a4ba1c9394a 100755 --- a/.ci/docker/common/install_pytorch.sh +++ b/.ci/docker/common/install_pytorch.sh @@ -1,6 +1,7 @@ #!/bin/bash # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -23,12 +24,6 @@ else export SCCACHE_DIR=/tmp/sccache fi -install_domains() { - echo "Install torchvision and torchaudio" - pip_install --no-build-isolation --user "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}" - pip_install --no-build-isolation --user "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION}" -} - configure_pytorch_compiler() { local cxx_version cxx_version=$(/usr/bin/c++ --version | head -n1) @@ -44,69 +39,13 @@ configure_pytorch_compiler() { fi } -install_pytorch_and_domains() { - git clone https://github.com/pytorch/pytorch.git - - # Fetch the target commit - pushd pytorch || true - git checkout "${TORCH_VERSION}" - git submodule update --init --recursive - - chown -R ci-user . - - export _GLIBCXX_USE_CXX11_ABI=1 - if [[ "$(uname -m)" == "aarch64" ]]; then - export BUILD_IGNORE_SVE_UNAVAILABLE=1 - fi - if [[ -n "${PYTORCH_BUILD_MAX_JOBS:-}" ]]; then - export MAX_JOBS="${PYTORCH_BUILD_MAX_JOBS}" - fi - configure_pytorch_compiler - # PyTorch no longer supports "python setup.py bdist_wheel"; it now builds - # through scikit-build-core (PEP 517). Build the wheel with the standard - # frontend and keep build isolation off, so PyTorch builds against this - # environment's numpy and toolchain (avoids an ABI mismatch) and reuses - # sccache. - # - # With isolation off the frontend does not fetch the PEP 517 build - # requirements, so install them here. They go into a throwaway venv rather - # than into the image, because scikit-build-core registers a setuptools - # build_ext plugin: left in the image it hijacks every later - # "pip install --no-build-isolation" and turns on C++20 module scanning that - # the image compiler cannot satisfy. The venv inherits the image's - # site-packages, so PyTorch still builds against the same numpy. - # - # Keep in sync with pytorch/pyproject.toml [build-system].requires. - local build_venv=/tmp/pytorch-build-venv - rm -rf "${build_venv}" - conda_run python -m venv --system-site-packages "${build_venv}" - # No pip cmake: scikit-build-core would prefer it over the image's, and it - # searches site-packages, where MKL and libomp are not. - conda_run "${build_venv}/bin/pip" install build "scikit-build-core>=1.0" \ - ninja "packaging>=24.2" "typing-extensions>=4.10.0" pyyaml six numpy - # These images have no module scanner, and nothing here uses modules. - conda_run env CMAKE_CXX_SCAN_FOR_MODULES=OFF \ - "${build_venv}/bin/python" -m build --wheel --no-isolation - rm -rf "${build_venv}" - pip_install "$(echo dist/*.whl)" - - # A build with no BLAS succeeds silently. Run from / to import the wheel. - (cd / && conda_run python -c " -import torch -assert torch._C.has_lapack, 'built without LAPACK' -torch.linalg.qr(torch.randn(4, 4)) -") +configure_pytorch_compiler +export TORCH_VERSION +export PYTHON_EXECUTABLE="/opt/conda/envs/py_${PYTHON_VERSION}/bin/python" +conda_run bash "$(dirname "${BASH_SOURCE[0]}")/install_pinned_pytorch.sh" - # Grab the pinned audio and vision commits from PyTorch - TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION:-release/2.11} - export TORCHAUDIO_VERSION - TORCHVISION_VERSION=${TORCHVISION_VERSION:-release/0.29} - export TORCHVISION_VERSION - - install_domains - - if [[ "${TORCHCI_REPORTS:-}" == "1" ]]; then - (cd / && conda_run python -c ' +if [[ "${TORCHCI_REPORTS:-}" == "1" ]]; then + (cd / && conda_run python -c ' import os import torch from torch.testing._internal.torchci import environment, plugin, report @@ -117,14 +56,4 @@ if report.SCHEMA_VERSION != "1.0": raise RuntimeError("The image requires torchci report schema 1.0") environment.capture() ') - fi - - popd || true - # Clean up the cloned PyTorch repo to reduce the Docker image size - rm -rf pytorch - - # Print sccache stats for debugging - as_ci_user sccache --show-stats -} - -install_pytorch_and_domains +fi diff --git a/.ci/docker/ubuntu/Dockerfile b/.ci/docker/ubuntu/Dockerfile index 7f9f58ab439..bc47fab9a0e 100644 --- a/.ci/docker/ubuntu/Dockerfile +++ b/.ci/docker/ubuntu/Dockerfile @@ -72,13 +72,14 @@ ARG TORCHCI_REPORTS ARG SKIP_PYTORCH ARG PYTORCH_BUILD_MAX_JOBS COPY ./common/install_pytorch.sh install_pytorch.sh +COPY ./common/install_pinned_pytorch.sh install_pinned_pytorch.sh COPY ./common/utils.sh utils.sh # A host docker build let sccache reach its S3 bucket with the EC2 instance # role; a BuildKit pod has no instance metadata, so docker-builds passes the # credentials in as this secret. World-readable because the sccache server runs # as ci-user; a secret mount never lands in a layer. RUN --mount=type=secret,id=aws-credentials,mode=0444 \ - if [ -z "${SKIP_PYTORCH}" ]; then bash ./install_pytorch.sh; fi && rm install_pytorch.sh utils.sh + if [ -z "${SKIP_PYTORCH}" ]; then bash ./install_pytorch.sh; fi && rm install_pytorch.sh install_pinned_pytorch.sh utils.sh ARG LINTRUNNER # Install lintrunner if needed diff --git a/.ci/scripts/install_pinned_pytorch.sh b/.ci/scripts/install_pinned_pytorch.sh new file mode 100755 index 00000000000..bb8446b8cc8 --- /dev/null +++ b/.ci/scripts/install_pinned_pytorch.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +exec bash "${script_dir}/../docker/common/install_pinned_pytorch.sh" "$@" diff --git a/.ci/scripts/setup-linux.sh b/.ci/scripts/setup-linux.sh index feb8a128b17..50d0c353639 100755 --- a/.ci/scripts/setup-linux.sh +++ b/.ci/scripts/setup-linux.sh @@ -1,6 +1,7 @@ #!/bin/bash # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -21,6 +22,8 @@ if [[ "${EDITABLE:-false}" == "true" ]]; then else install_executorch --use-pt-pinned-commit fi +# Dependency installation must preserve the pinned PyTorch build. +bash .ci/scripts/install_pinned_pytorch.sh --verify-only build_executorch_runner "${BUILD_TOOL}" "${BUILD_MODE}" if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then diff --git a/.ci/scripts/setup-macos.sh b/.ci/scripts/setup-macos.sh index 38863597224..36dbfb6e2e7 100755 --- a/.ci/scripts/setup-macos.sh +++ b/.ci/scripts/setup-macos.sh @@ -1,6 +1,7 @@ #!/bin/bash # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -129,17 +130,19 @@ fi # separate copy from PyPI. sccache is initialized above so source-build # cache misses still hit the cache. print_cmake_info -install_pytorch_and_domains +bash .ci/scripts/install_pinned_pytorch.sh install_pip_dependencies # install_executorch's --use-pt-pinned-commit skips re-installing torch since -# install_pytorch_and_domains already installed the pinned build above. +# install_pinned_pytorch.sh already installed the pinned build above. if [[ "$EDITABLE" == "true" ]]; then install_executorch --use-pt-pinned-commit --editable else install_executorch --use-pt-pinned-commit fi +# Dependency installation must preserve the pinned PyTorch build. +bash .ci/scripts/install_pinned_pytorch.sh --verify-only build_executorch_runner "${BUILD_TOOL}" "${BUILD_MODE}" if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then diff --git a/.ci/scripts/tests/test_install_pinned_pytorch.py b/.ci/scripts/tests/test_install_pinned_pytorch.py new file mode 100644 index 00000000000..9b0b3258b0e --- /dev/null +++ b/.ci/scripts/tests/test_install_pinned_pytorch.py @@ -0,0 +1,346 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +INSTALLER = ROOT / ".ci/scripts/install_pinned_pytorch.sh" +PIN = "a" * 40 + +# Stand-ins exercise the shell flow without downloading or building PyTorch. +TOOL = r"""#!/usr/bin/env python3 +import json, os, pathlib, sys +name = pathlib.Path(sys.argv[0]).name +args = sys.argv[1:] +root = pathlib.Path(os.environ['TEST_ROOT']) +with (root / 'commands').open('a') as out: + out.write(json.dumps([name, *args]) + '\n') +if name == 'uname': + print('arm64' if args else os.environ.get('TEST_SYSTEM', 'Linux')) +elif name == 'install_name_tool': + if args[0] == '-delete_rpath': + marker = root / 'rpath-removed' + if marker.exists(): + sys.exit(1) + marker.touch() +elif name == 'git': + if args[0] == 'clone': + path = pathlib.Path(args[-1]) + path.mkdir() + (path / 'version.txt').write_text('2.15.0a0') + elif args[0] == 'ls-remote': + refs = os.environ.get('TEST_REMOTE_REFS', 'a' * 40 + '\trefs/heads/release/2.15') + print(refs) + if not refs: + sys.exit(2) + elif args[0] == 'rev-parse': + print('a' * (7 if '--short=7' in args else 40)) +elif name in ('chosen-python', 'python'): + if args[:2] == ['-m', 'venv']: + directory = pathlib.Path(args[-1]) / 'bin' + directory.mkdir(parents=True) + (directory / 'python').symlink_to(root / 'bin' / 'chosen-python') + elif args[:2] == ['-m', 'build']: + if os.environ.get('TEST_BUILD_FAIL'): + print('Build failed; installed pin: ' + (root / 'installed').read_text(), file=sys.stderr) + sys.exit(42) + platform = os.environ.get('TEST_WHEEL_PLATFORM') or ('macosx_15_0_arm64' if os.environ.get('TEST_SYSTEM') == 'Darwin' else 'linux_x86_64') + pathlib.Path('dist').mkdir(exist_ok=True) + pathlib.Path(f'dist/torch-2.15.0a0+gitaaaaaaa-cp312-cp312-{platform}.whl').touch() + elif args[:3] == ['-m', 'pip', 'download']: + if not os.environ.get('TEST_CACHE_HIT'): + sys.exit(1) + directory = pathlib.Path(args[args.index('--dest') + 1]) + directory.mkdir(exist_ok=True) + (directory / args[-1].rsplit('/', 1)[1].replace('%2B', '+')).touch() + elif args[:3] == ['-m', 'pip', 'install']: + if any(arg.endswith('.whl') for arg in args): + (root / 'installed').write_text(os.environ.get('TEST_BUILT_PIN', 'a' * 40)) + elif args[:3] == ['-m', 'pip', 'uninstall']: + (root / 'installed').write_text('') + elif args[0] == '-c': + code = args[1] + if 'sys.executable' in code: + print(sys.argv[0]) + elif 'git_version' in code: + print((root / 'installed').read_text()) + elif 'mac_ver' in code: + print('macosx_15_0_arm64') + elif 'python_version_tuple' in code: + print('312') + elif 'has_lapack' in code and os.environ.get('TEST_NO_LAPACK'): + sys.exit('PyTorch was built without LAPACK') + elif 'find_spec' in code: + print(root / 'torch') +""" + + +@unittest.skipIf(os.name == "nt", "The installer tests require a Unix environment") +class TestPinnedPytorch(unittest.TestCase): + def run_installer(self, installed="", ref=PIN, verify=False, system="Linux", **env): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "bin").mkdir() + (root / "installed").write_text(installed) + (root / "torch/lib").mkdir(parents=True) + (root / "torch/lib/libtorch.dylib").touch() + for name in ( + "git", + "uname", + "chosen-python", + "python", + "aws", + "install_name_tool", + ): + tool = root / "bin" / name + tool.write_text(TOOL) + tool.chmod(0o755) + pin = root / ".ci/docker/ci_commit_pins/pytorch.txt" + pin.parent.mkdir(parents=True) + pin.write_text(ref) + result = subprocess.run( + [ + "bash", + str(INSTALLER), + *(["--verify-only"] if verify else []), + str(root), + ], + env={ + **os.environ, + "PATH": f"{root}/bin:" + os.environ["PATH"], + "PYTHON_EXECUTABLE": str(root / "bin/chosen-python"), + "TMPDIR": str(root), + "TEST_ROOT": str(root), + "TEST_SYSTEM": system, + "GITHUB_RUNNER": "1", + **env, + }, + capture_output=True, + text=True, + ) + commands = root / "commands" + calls = ( + [json.loads(line) for line in commands.read_text().splitlines()] + if commands.exists() + else [] + ) + self.assertEqual(list(root.glob("executorch-pytorch.*")), []) + return result, calls + + def test_matching_pin_installs_domains_without_rebuilding_torch(self): + for ref in (PIN, "release/2.15"): + with self.subTest(ref=ref): + result, calls = self.run_installer( + installed=PIN, ref=ref, system="Darwin" + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(call[:2] == ["git", "clone"] for call in calls)) + self.assertFalse(any(call[1:3] == ["-m", "build"] for call in calls)) + self.assertFalse(any("uninstall" in call for call in calls)) + for package in ("audio", "vision"): + self.assertTrue( + any( + "--no-deps" in call + and any(f"{package}.git@" in arg for arg in call) + for call in calls + ) + ) + + def test_domain_source_overrides_are_preserved(self): + result, calls = self.run_installer( + installed=PIN, + TORCHAUDIO_VERSION="audio-pin", + TORCHVISION_VERSION="vision-pin", + ) + self.assertEqual(result.returncode, 0, result.stderr) + arguments = [argument for call in calls for argument in call] + self.assertIn("git+https://github.com/pytorch/audio.git@audio-pin", arguments) + self.assertIn("git+https://github.com/pytorch/vision.git@vision-pin", arguments) + + def test_matching_macos_pin_repairs_rpaths_only_in_install_mode(self): + for verify in (False, True): + with self.subTest(verify=verify): + result, calls = self.run_installer( + installed=PIN, system="Darwin", verify=verify + ) + self.assertEqual(result.returncode, 0, result.stderr) + repairs = [call[1] for call in calls if call[0] == "install_name_tool"] + self.assertEqual( + repairs, + [] if verify else ["-delete_rpath", "-delete_rpath", "-add_rpath"], + ) + + def test_verify_annotated_tag_uses_commit(self): + result, _ = self.run_installer( + installed=PIN, + ref="v2.15.0", + verify=True, + TEST_REMOTE_REFS="b" * 40 + + "\trefs/tags/v2.15.0\n" + + PIN + + "\trefs/tags/v2.15.0^{}", + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_invalid_remote_ref_fails_without_installing(self): + for refs in ( + "", + f"{PIN}\trefs/heads/release/2.15\n{'b' * 40}\trefs/tags/release/2.15", + ): + with self.subTest(refs=refs): + result, calls = self.run_installer( + ref="release/2.15", TEST_REMOTE_REFS=refs + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Cannot resolve PyTorch ref", result.stderr) + self.assertFalse(any(call[1:3] == ["-m", "pip"] for call in calls)) + + def test_verify_matching_pin_does_not_clone_or_install(self): + for ref in (PIN, "release/2.15"): + with self.subTest(ref=ref): + result, calls = self.run_installer(installed=PIN, ref=ref, verify=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(call[:2] == ["git", "clone"] for call in calls)) + self.assertFalse(any(call[1:3] == ["-m", "pip"] for call in calls)) + self.assertTrue( + any("has_lapack" in arg for call in calls for arg in call) + ) + + def test_verify_mismatch_does_not_install(self): + result, calls = self.run_installer(installed="b" * 40, verify=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("pin mismatch", result.stderr) + self.assertFalse( + any( + call[:2] == ["git", "clone"] or call[1:3] == ["-m", "pip"] + for call in calls + ) + ) + + def test_linux_replaces_installed_build_in_both_directions(self): + for installed in ("older", "newer"): + with self.subTest(installed=installed): + result, calls = self.run_installer(installed=installed) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue( + any(call[1:4] == ["-m", "pip", "uninstall"] for call in calls) + ) + self.assertTrue(any(call[1:3] == ["-m", "build"] for call in calls)) + self.assertFalse(any("--user" in call for call in calls)) + + def test_build_failure_preserves_installed_torch(self): + for system in ("Linux", "Darwin"): + with self.subTest(system=system): + result, calls = self.run_installer( + installed="old-pin", system=system, TEST_BUILD_FAIL="1" + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.returncode, 42) + self.assertIn("Build failed; installed pin: old-pin", result.stderr) + self.assertIn( + "Failed to prepare the replacement PyTorch wheel", result.stderr + ) + self.assertIn("have not been removed", result.stderr) + self.assertFalse(any("uninstall" in call for call in calls)) + + def test_branch_ref_resolves_before_matching(self): + result, calls = self.run_installer(installed=PIN, ref="release/2.15") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(any(call[:2] == ["git", "ls-remote"] for call in calls)) + self.assertFalse(any(call[1:3] == ["-m", "build"] for call in calls)) + + def test_wrong_built_commit_fails_verification(self): + result, _ = self.run_installer(TEST_BUILT_PIN="b" * 40) + self.assertNotEqual(result.returncode, 0) + self.assertIn("pin mismatch", result.stderr) + self.assertNotIn("have not been removed", result.stderr) + + def test_macos_uses_cached_wheel(self): + result, calls = self.run_installer(system="Darwin", TEST_CACHE_HIT="1") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(call[1:3] == ["-m", "build"] for call in calls)) + + def test_macos_cache_miss_builds_matching_wheel(self): + result, calls = self.run_installer(system="Darwin") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(any(call[1:3] == ["-m", "build"] for call in calls)) + + def test_macos_upload_requires_github_actions(self): + for actions, runner, upload in ( + ("", "", False), + ("false", "", False), + ("true", "", True), + ("true", "1", False), + ): + with self.subTest(actions=actions, runner=runner): + result, calls = self.run_installer( + system="Darwin", GITHUB_ACTIONS=actions, GITHUB_RUNNER=runner + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(any(call[0] == "aws" for call in calls), upload) + + def test_macos_rejects_wrong_cache_filename(self): + result, calls = self.run_installer( + installed="old-pin", system="Darwin", TEST_WHEEL_PLATFORM="wrong" + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("does not match cache key", result.stderr) + self.assertFalse(any("uninstall" in call for call in calls)) + + def test_verify_branch_mismatch_does_not_install(self): + result, calls = self.run_installer( + installed="b" * 40, ref="release/2.15", verify=True + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("pin mismatch", result.stderr) + self.assertFalse(any(call[1:3] == ["-m", "pip"] for call in calls)) + + def test_python_from_path_is_supported(self): + result, _ = self.run_installer(installed=PIN, PYTHON_EXECUTABLE="python") + self.assertEqual(result.returncode, 0, result.stderr) + + def test_matching_pin_still_checks_lapack(self): + result, _ = self.run_installer(installed=PIN, TEST_NO_LAPACK="1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("without LAPACK", result.stderr) + + def test_docker_wrapper_preserves_conda_environment(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copy(ROOT / ".ci/docker/common/install_pytorch.sh", root) + (root / "utils.sh").write_text( + "conda_run() {\n" + ' test "$TORCH_VERSION" = release/2.15 &&\n' + ' test "$PYTHON_EXECUTABLE" = /opt/conda/envs/py_3.12/bin/python &&\n' + ' test "$1" = bash &&\n' + ' test "$2" = "' + directory + '/install_pinned_pytorch.sh"\n' + "}\n" + ) + result = subprocess.run( + ["bash", str(root / "install_pytorch.sh")], + env={ + **os.environ, + "TORCH_VERSION": "release/2.15", + "PYTHON_VERSION": "3.12", + }, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_windows_is_rejected(self): + result, _ = self.run_installer(system="MINGW64_NT") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Unsupported system", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index c57d3618df1..cdb110a0f19 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -44,168 +44,6 @@ install_pip_dependencies() { popd || return } -dedupe_macos_loader_path_rpaths() { - if [[ "$(uname)" != "Darwin" ]]; then - return - fi - - local torch_lib_dir - local python_executable=${PYTHON_EXECUTABLE:-python} - pushd .. - torch_lib_dir=$("${python_executable}" -c "import importlib.util; print(importlib.util.find_spec('torch').submodule_search_locations[0])")/lib - popd - - if [[ -z "${torch_lib_dir}" || ! -d "${torch_lib_dir}" ]]; then - return - fi - - local torch_libs=( - "libtorch_cpu.dylib" - "libtorch.dylib" - "libc10.dylib" - ) - - for lib_name in "${torch_libs[@]}"; do - local lib_path="${torch_lib_dir}/${lib_name}" - if [[ ! -f "${lib_path}" ]]; then - continue - fi - - local removed=0 - # Repeatedly remove the @loader_path rpath entries until none remain. - while install_name_tool -delete_rpath @loader_path "${lib_path}" 2>/dev/null; do - removed=1 - done - - if [[ "${removed}" == "1" ]]; then - install_name_tool -add_rpath @loader_path "${lib_path}" || true - fi - done -} - -install_domains() { - local python_executable=${PYTHON_EXECUTABLE:-python} - echo "Install torchvision and torchaudio" - "${python_executable}" -m pip install --no-build-isolation \ - "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}" - "${python_executable}" -m pip install --no-build-isolation \ - "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION}" -} - -install_pytorch_and_domains() { - local python_executable=${PYTHON_EXECUTABLE:-python} - pushd .ci/docker || return - TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt) - popd || return - - git clone https://github.com/pytorch/pytorch.git - - # Fetch the target commit - pushd pytorch || return - git checkout "${TORCH_VERSION}" - - local system_name=$(uname) - if [[ "${system_name}" == "Darwin" ]]; then - local platform=$("${python_executable}" -c 'import sysconfig; import platform; v=platform.mac_ver()[0].split(".")[0]; platform=sysconfig.get_platform().split("-"); platform[1]=f"{v}_0"; print("_".join(platform))') - fi - local python_version=$("${python_executable}" -c 'import platform; v=platform.python_version_tuple(); print(f"{v[0]}{v[1]}")') - local torch_release=$(cat version.txt) - # Download key must match the upload key below (basename of dist/*.whl, - # which always carries the build's resolved +gitHASH). Branch-ref pins - # like `release/2.14` would otherwise produce `+gitrelease` here and - # never hit the cache. - local torch_short_hash=$(git rev-parse --short=7 HEAD) - local torch_wheel_path="cached_artifacts/pytorch/executorch/pytorch_wheels/${system_name}/${python_version}" - local torch_wheel_name="torch-${torch_release}%2Bgit${torch_short_hash}-cp${python_version}-cp${python_version}-${platform:-}.whl" - - local cached_torch_wheel="https://gha-artifacts.s3.us-east-1.amazonaws.com/${torch_wheel_path}/${torch_wheel_name}" - # Cache PyTorch wheel is only needed on MacOS, Linux CI already has this as part - # of the Docker image - local torch_wheel_not_found=0 - if [[ "${system_name}" == "Darwin" ]]; then - "${python_executable}" -m pip install "${cached_torch_wheel}" || torch_wheel_not_found=1 - else - torch_wheel_not_found=1 - fi - - # Found no such wheel, we will build it from source then - if [[ "${torch_wheel_not_found}" == "1" ]]; then - echo "No cached wheel found, continue with building PyTorch at ${TORCH_VERSION}" - - git submodule update --init --recursive - if [[ "$(uname -m)" == "aarch64" ]]; then - export BUILD_IGNORE_SVE_UNAVAILABLE=1 - fi - # PyTorch dropped setup.py. Build in a throwaway environment that can see the - # active one, so its build requirements, which pin a cmake that would be - # preferred over the one on PATH, cannot disturb what is installed here. - # - # Keep in sync with pytorch/pyproject.toml [build-system].requires. - local build_venv=/tmp/pytorch-build-venv - rm -rf "${build_venv}" - "${python_executable}" -m venv --system-site-packages "${build_venv}" - "${build_venv}/bin/pip" install build "scikit-build-core>=1.0" ninja \ - "packaging>=24.2" "typing-extensions>=4.10.0" pyyaml six numpy - USE_DISTRIBUTED=1 "${build_venv}/bin/python" -m build --wheel --no-isolation - rm -rf "${build_venv}" - "${python_executable}" -m pip install "$(echo dist/*.whl)" - # A build with no BLAS succeeds silently, so check rather than assume. - (cd / && "${python_executable}" -c " -import torch -assert torch._C.has_lapack, 'built without LAPACK' -torch.linalg.qr(torch.randn(4, 4)) -") - - # Invariant: the basename the build just produced must match the cache - # URL we'd reconstruct on the next run. If they diverge (someone edits - # torch_wheel_name above, or PyTorch renames its wheels), the cache - # will silently miss and every macOS run will fall back to a ~30-min - # source build. Fail loudly so the regression is caught immediately. - shopt -s nullglob - local built_wheels=(dist/*.whl) - shopt -u nullglob - if [[ ${#built_wheels[@]} -ne 1 ]]; then - echo "ERROR: expected exactly 1 wheel in dist/, found ${#built_wheels[@]}" >&2 - exit 1 - fi - local built_wheel_name - built_wheel_name=$(basename "${built_wheels[0]}") - local expected_wheel_name="${torch_wheel_name//\%2B/+}" - if [[ "${built_wheel_name}" != "${expected_wheel_name}" ]]; then - echo "ERROR: built torch wheel name does not match cache URL key:" >&2 - echo " built: ${built_wheel_name}" >&2 - echo " expected: ${expected_wheel_name}" >&2 - echo "Fix torch_wheel_name construction in install_pytorch_and_domains" >&2 - echo "in .ci/scripts/utils.sh" >&2 - exit 1 - fi - - # Only AWS runners have access to S3 - if command -v aws && [[ -z "${GITHUB_RUNNER:-}" ]]; then - for wheel_path in dist/*.whl; do - local wheel_name=$(basename "${wheel_path}") - echo "Caching ${wheel_name}" - aws s3 cp "${wheel_path}" "s3://gha-artifacts/${torch_wheel_path}/${wheel_name}" - done - fi - else - echo "Use cached wheel at ${cached_torch_wheel}" - fi - - dedupe_macos_loader_path_rpaths - # Grab the pinned audio and vision commits from PyTorch - TORCHAUDIO_VERSION=release/2.11 - export TORCHAUDIO_VERSION - TORCHVISION_VERSION=release/0.29 - export TORCHVISION_VERSION - - install_domains - - popd || return - # Print sccache stats for debugging - sccache --show-stats || true -} - build_executorch_runner_buck2() { # Build executorch runtime with retry as this step is flaky on macos CI retry buck2 build //examples/portable/executor_runner:executor_runner From 7ecb64bc65e6bcdec2b2e6e19af9c800c3c55ead Mon Sep 17 00:00:00 2001 From: Zingo Andersen Date: Wed, 11 Nov 2026 13:48:36 +0100 Subject: [PATCH 2/2] ci: consolidate pinned PyTorch installation - fix CI Signed-off-by: Zingo Andersen Change-Id: I9945ab4695bf36f59478f709bb66234a7e2b722c --- .ci/docker/README.md | 5 +++-- .ci/scripts/setup-linux.sh | 4 +++- .ci/scripts/tests/test_torchci_image_override.py | 12 +++++++----- .ci/scripts/tests/test_torchci_workflow.py | 3 ++- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.ci/docker/README.md b/.ci/docker/README.md index d2353c31bdf..fc086c1e55f 100644 --- a/.ci/docker/README.md +++ b/.ci/docker/README.md @@ -63,8 +63,9 @@ PYTHON_EXECUTABLE="$PWD/env/bin/python" \ It can also run after dependency installation to detect replacement of the pinned build. Linux and macOS CI verify after installing ExecuTorch, before -building the test runner. On Linux this also checks that the Docker image -provides the commit required by the checkout. +building the test runner. On Linux the check preserves the commit provided by +the image or PyTorch CI, which may intentionally differ from the shared checkout +pin. The shared implementation lives in `common/install_pinned_pytorch.sh` so it is also available in the Docker build context. Docker and macOS CI use this same diff --git a/.ci/scripts/setup-linux.sh b/.ci/scripts/setup-linux.sh index 50d0c353639..942c39f5ceb 100755 --- a/.ci/scripts/setup-linux.sh +++ b/.ci/scripts/setup-linux.sh @@ -17,13 +17,15 @@ echo "Build tool: $BUILD_TOOL, Mode: $BUILD_MODE" # As Linux job is running inside a Docker container, all of its dependencies # have already been installed, so we use PyTorch build from source here instead # of nightly. This allows CI to test against latest commits from PyTorch +# The image or PyTorch CI may intentionally provide a different source pin. +torch_pin=$(cd / && "${PYTHON_EXECUTABLE:-python}" -c 'import torch; print(torch.version.git_version)') if [[ "${EDITABLE:-false}" == "true" ]]; then install_executorch --use-pt-pinned-commit --editable else install_executorch --use-pt-pinned-commit fi # Dependency installation must preserve the pinned PyTorch build. -bash .ci/scripts/install_pinned_pytorch.sh --verify-only +TORCH_VERSION="${torch_pin}" bash .ci/scripts/install_pinned_pytorch.sh --verify-only build_executorch_runner "${BUILD_TOOL}" "${BUILD_MODE}" if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then diff --git a/.ci/scripts/tests/test_torchci_image_override.py b/.ci/scripts/tests/test_torchci_image_override.py index adad42a2601..d1ca67e565e 100644 --- a/.ci/scripts/tests/test_torchci_image_override.py +++ b/.ci/scripts/tests/test_torchci_image_override.py @@ -1,5 +1,6 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -93,14 +94,15 @@ def test_missing_or_mutable_pin_stops_before_docker(self): def test_docker_forwards_the_domain_pins_and_checks_installed_torch(self): dockerfile = (ROOT / ".ci/docker/ubuntu/Dockerfile").read_text() script = (ROOT / ".ci/docker/common/install_pytorch.sh").read_text() + installer = (ROOT / ".ci/docker/common/install_pinned_pytorch.sh").read_text() for variable in (*PACKAGES.values(), "TORCHCI_REPORTS"): self.assertIn(f"ARG {variable}\n", dockerfile) - self.assertIn("TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION:-release/2.11}", script) - self.assertIn( - "TORCHVISION_VERSION=${TORCHVISION_VERSION:-release/0.29}", script - ) + self.assertIn("${TORCHAUDIO_VERSION:-release/2.11}", installer) + self.assertIn("${TORCHVISION_VERSION:-release/0.29}", installer) self.assertLess( - script.index(" install_domains\n"), + script.index( + 'conda_run bash "$(dirname "${BASH_SOURCE[0]}")/install_pinned_pytorch.sh"' + ), script.index("if torch.version.git_version"), ) self.assertIn('report.SCHEMA_VERSION != "1.0"', script) diff --git a/.ci/scripts/tests/test_torchci_workflow.py b/.ci/scripts/tests/test_torchci_workflow.py index 6b9a67e3863..846538716fe 100644 --- a/.ci/scripts/tests/test_torchci_workflow.py +++ b/.ci/scripts/tests/test_torchci_workflow.py @@ -1,5 +1,6 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -16,7 +17,7 @@ def load_workflow(name): - return yaml.safe_load((WORKFLOWS / name).read_text()) + return yaml.safe_load((WORKFLOWS / name).read_text(encoding="utf-8")) class TorchciWorkflowTest(unittest.TestCase):