diff --git a/.ci/docker/README.md b/.ci/docker/README.md index 267eff4b0b1..fc086c1e55f 100644 --- a/.ci/docker/README.md +++ b/.ci/docker/README.md @@ -32,3 +32,42 @@ torch identity and schema-1.0 reporter after installing the domain libraries. These pins participate in the existing global Linux image hash. A pin change can require other Linux images to rebuild, even though only clang12 reads it. Validate the rebuilt image before enabling test-report uploads. + +## Reusing the pinned PyTorch installer + +From an ExecuTorch checkout, activate a venv or conda environment and run: + +```bash +bash .ci/scripts/install_pinned_pytorch.sh +bash .ci/scripts/install_pinned_pytorch.sh --verify-only +``` + +The installer reads `.ci/docker/ci_commit_pins/pytorch.txt`, resolves named refs, +and reuses an installed build with the same commit. Set `PYTHON_EXECUTABLE` to +select another interpreter, or `TORCH_VERSION` to override the source ref as the +Docker build does. A mismatched build is replaced, including when downgrading. +Install mode also installs the domain packages when the torch build is reused. +`--verify-only` checks the installed PyTorch commit and LAPACK support, returning +nonzero on a mismatch or failed check. It does not install packages or change the +active environment. Named refs are resolved with `git ls-remote`, without cloning. +This is not a check of every ExecuTorch build or test dependency. + +Use it inside a CI container to check that its prebuilt PyTorch matches the pin +in the current checkout, or locally to check the selected venv/conda environment +before running tests: + +```bash +PYTHON_EXECUTABLE="$PWD/env/bin/python" \ + bash .ci/scripts/install_pinned_pytorch.sh --verify-only +``` + +It can also run after dependency installation to detect replacement of the +pinned build. Linux and macOS CI verify after installing ExecuTorch, before +building the test runner. On Linux the check preserves the commit provided by +the image or PyTorch CI, which may intentionally differ from the shared checkout +pin. + +The shared implementation lives in `common/install_pinned_pytorch.sh` so it is +also available in the Docker build context. Docker and macOS CI use this same +implementation. System build dependencies (including BLAS/LAPACK) must already +be installed. Windows is not supported. diff --git a/.ci/docker/common/install_pinned_pytorch.sh b/.ci/docker/common/install_pinned_pytorch.sh new file mode 100755 index 00000000000..3231bd85894 --- /dev/null +++ b/.ci/docker/common/install_pinned_pytorch.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Called directly or through .ci/scripts/install_pinned_pytorch.sh. +# Activate a venv/conda environment first, or set PYTHON_EXECUTABLE. +set -euo pipefail + +dedupe_macos_loader_path_rpaths() { + if [[ "$(uname)" != "Darwin" ]]; then + return + fi + + local torch_lib_dir + torch_lib_dir=$(cd / && "${python_executable}" -c "import importlib.util; print(importlib.util.find_spec('torch').submodule_search_locations[0])")/lib + + if [[ -z "${torch_lib_dir}" || ! -d "${torch_lib_dir}" ]]; then + return + fi + + local torch_libs=( + "libtorch_cpu.dylib" + "libtorch.dylib" + "libc10.dylib" + ) + + for lib_name in "${torch_libs[@]}"; do + local lib_path="${torch_lib_dir}/${lib_name}" + if [[ ! -f "${lib_path}" ]]; then + continue + fi + + local removed=0 + # Repeatedly remove the @loader_path rpath entries until none remain. + while install_name_tool -delete_rpath @loader_path "${lib_path}" 2>/dev/null; do + removed=1 + done + + if [[ "${removed}" == "1" ]]; then + install_name_tool -add_rpath @loader_path "${lib_path}" || true + fi + done +} + +verify_only=false +if [[ "${1:-}" == "--verify-only" ]]; then + verify_only=true + shift +fi +if [[ $# -gt 1 || "${1:-}" == -* ]]; then + echo "Usage: $0 [--verify-only] [EXECUTORCH_DIRECTORY]" >&2 + exit 2 +fi +executorch_dir=$(cd "${1:-.}" && pwd) +python_executable=${PYTHON_EXECUTABLE:-python} +# Resolve the interpreter before entering the temporary source directory. +python_executable=$("${python_executable}" -c 'import sys; print(sys.executable)') +system_name=$(uname) +case "${system_name}" in + Linux | Darwin) ;; + *) echo "Unsupported system: ${system_name}" >&2; exit 1 ;; +esac + +torch_ref=${TORCH_VERSION:-$(cat "${executorch_dir}/.ci/docker/ci_commit_pins/pytorch.txt")} +torch_ref=$(printf '%s' "${torch_ref}" | tr -d '[:space:]') +git check-ref-format --branch "${torch_ref}" >/dev/null +installed_pin=$(cd / && "${python_executable}" -c \ + 'import torch; print(torch.version.git_version or "")' 2>/dev/null || true) +expected_pin=${torch_ref} + +verify_pin() { + if [[ "${installed_pin}" != "${expected_pin}" ]]; then + echo "PyTorch pin mismatch: expected ${expected_pin}, found ${installed_pin:-none}" >&2 + return 1 + fi + (cd / && "${python_executable}" -c ' +import torch +assert torch._C.has_lapack, "PyTorch was built without LAPACK" +torch.linalg.qr(torch.randn(4, 4)) +') +} + +install_domains() { + "${python_executable}" -m pip install numpy pillow + "${python_executable}" -m pip install --no-build-isolation --no-deps \ + "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION:-release/2.11}" \ + "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION:-release/0.29}" +} + +if [[ ! "${torch_ref}" =~ ^[0-9a-f]{40}$ ]]; then + expected_pin=$(git ls-remote --exit-code https://github.com/pytorch/pytorch.git \ + "${torch_ref}" "${torch_ref}^{}" | awk ' + { sub(/\^\{\}$/, "", $2); pins[$2] = $1 } + END { for (ref in pins) print pins[ref] } + ' | sort -u) || expected_pin="" + if [[ ! "${expected_pin}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Cannot resolve PyTorch ref to one commit: ${torch_ref}" >&2 + exit 1 + fi +fi + +if [[ "${installed_pin}" == "${expected_pin}" || "${verify_only}" == true ]]; then + verify_pin + echo "PyTorch pin ${expected_pin} is already installed" + if [[ "${verify_only}" == false ]]; then + dedupe_macos_loader_path_rpaths + install_domains + fi + exit 0 +fi + +build_root=$(mktemp -d "${TMPDIR:-/tmp}/executorch-pytorch.XXXXXXXX") +trap ' + status=$? + if [[ ${status} -ne 0 ]]; then + echo "Failed to prepare the replacement PyTorch wheel." >&2 + echo "Existing torch and domain packages have not been removed." >&2 + fi + rm -rf "${build_root}" + exit "${status}" +' EXIT +git clone --filter=blob:none --no-checkout https://github.com/pytorch/pytorch.git "${build_root}/pytorch" +cd "${build_root}/pytorch" +git fetch --depth=1 origin "${expected_pin}" +git checkout --detach FETCH_HEAD + +cached=false +if [[ "${system_name}" == Darwin ]]; then + platform=$("${python_executable}" -c 'import sysconfig; import platform; v=platform.mac_ver()[0].split(".")[0]; platform=sysconfig.get_platform().split("-"); platform[1]=f"{v}_0"; print("_".join(platform))') + python_version=$("${python_executable}" -c 'import platform; v=platform.python_version_tuple(); print(f"{v[0]}{v[1]}")') + torch_release=$(cat version.txt) + torch_short_hash=$(git rev-parse --short=7 HEAD) + torch_wheel_path="cached_artifacts/pytorch/executorch/pytorch_wheels/${system_name}/${python_version}" + torch_wheel_name="torch-${torch_release}+git${torch_short_hash}-cp${python_version}-cp${python_version}-${platform}.whl" + cached_torch_wheel="https://gha-artifacts.s3.us-east-1.amazonaws.com/${torch_wheel_path}/${torch_wheel_name//+/%2B}" + if "${python_executable}" -m pip download --no-deps --dest dist "${cached_torch_wheel}"; then + cached=true + fi +fi + +if [[ "${cached}" == false ]]; then + echo "Building PyTorch at ${expected_pin}" + git submodule update --init --recursive + if [[ "$(uname -m)" == aarch64 ]]; then + export BUILD_IGNORE_SVE_UNAVAILABLE=1 + fi + if [[ "${system_name}" == Linux ]]; then + export _GLIBCXX_USE_CXX11_ABI=1 + fi + if [[ -n "${PYTORCH_BUILD_MAX_JOBS:-}" ]]; then + export MAX_JOBS="${PYTORCH_BUILD_MAX_JOBS}" + fi + build_venv="${build_root}/build-env" + "${python_executable}" -m venv --system-site-packages "${build_venv}" + # Keep in sync with pytorch/pyproject.toml [build-system].requires. + # Do not install pip cmake: it hides the image's MKL/libomp search paths. + "${build_venv}/bin/python" -m pip install build "scikit-build-core>=1.0" \ + ninja "packaging>=24.2" "typing-extensions>=4.10.0" pyyaml six numpy + CMAKE_CXX_SCAN_FOR_MODULES=OFF USE_DISTRIBUTED=1 \ + "${build_venv}/bin/python" -m build --wheel --no-isolation +fi + +shopt -s nullglob +wheels=(dist/*.whl) +shopt -u nullglob +if [[ ${#wheels[@]} -ne 1 ]]; then + echo "Expected one PyTorch wheel, found ${#wheels[@]}" >&2 + exit 1 +fi +if [[ "${system_name}" == Darwin ]]; then + if [[ "$(basename "${wheels[0]}")" != "${torch_wheel_name}" ]]; then + echo "Built PyTorch wheel does not match cache key: ${torch_wheel_name}" >&2 + exit 1 + fi +fi + +trap 'rm -rf "${build_root}"' EXIT +# Remove the old builds explicitly so equal package versions also replace +# different source commits, and domain dependencies cannot replace the pin. +"${python_executable}" -m pip uninstall -y torch torchaudio torchvision +"${python_executable}" -m pip install "${wheels[0]}" + +dedupe_macos_loader_path_rpaths +install_domains +installed_pin=$(cd / && "${python_executable}" -c 'import torch; print(torch.version.git_version or "")') +verify_pin + +# Preserve the existing macOS wheel cache on AWS CI runners. +if [[ "${system_name}" == Darwin && "${cached}" == false && "${GITHUB_ACTIONS:-}" == true && -z "${GITHUB_RUNNER:-}" ]] && command -v aws >/dev/null; then + aws s3 cp "${wheels[0]}" "s3://gha-artifacts/${torch_wheel_path}/${torch_wheel_name}" +fi +sccache --show-stats || true diff --git a/.ci/docker/common/install_pytorch.sh b/.ci/docker/common/install_pytorch.sh index d008f0773fd..a4ba1c9394a 100755 --- a/.ci/docker/common/install_pytorch.sh +++ b/.ci/docker/common/install_pytorch.sh @@ -1,6 +1,7 @@ #!/bin/bash # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -23,12 +24,6 @@ else export SCCACHE_DIR=/tmp/sccache fi -install_domains() { - echo "Install torchvision and torchaudio" - pip_install --no-build-isolation --user "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}" - pip_install --no-build-isolation --user "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION}" -} - configure_pytorch_compiler() { local cxx_version cxx_version=$(/usr/bin/c++ --version | head -n1) @@ -44,69 +39,13 @@ configure_pytorch_compiler() { fi } -install_pytorch_and_domains() { - git clone https://github.com/pytorch/pytorch.git - - # Fetch the target commit - pushd pytorch || true - git checkout "${TORCH_VERSION}" - git submodule update --init --recursive - - chown -R ci-user . - - export _GLIBCXX_USE_CXX11_ABI=1 - if [[ "$(uname -m)" == "aarch64" ]]; then - export BUILD_IGNORE_SVE_UNAVAILABLE=1 - fi - if [[ -n "${PYTORCH_BUILD_MAX_JOBS:-}" ]]; then - export MAX_JOBS="${PYTORCH_BUILD_MAX_JOBS}" - fi - configure_pytorch_compiler - # PyTorch no longer supports "python setup.py bdist_wheel"; it now builds - # through scikit-build-core (PEP 517). Build the wheel with the standard - # frontend and keep build isolation off, so PyTorch builds against this - # environment's numpy and toolchain (avoids an ABI mismatch) and reuses - # sccache. - # - # With isolation off the frontend does not fetch the PEP 517 build - # requirements, so install them here. They go into a throwaway venv rather - # than into the image, because scikit-build-core registers a setuptools - # build_ext plugin: left in the image it hijacks every later - # "pip install --no-build-isolation" and turns on C++20 module scanning that - # the image compiler cannot satisfy. The venv inherits the image's - # site-packages, so PyTorch still builds against the same numpy. - # - # Keep in sync with pytorch/pyproject.toml [build-system].requires. - local build_venv=/tmp/pytorch-build-venv - rm -rf "${build_venv}" - conda_run python -m venv --system-site-packages "${build_venv}" - # No pip cmake: scikit-build-core would prefer it over the image's, and it - # searches site-packages, where MKL and libomp are not. - conda_run "${build_venv}/bin/pip" install build "scikit-build-core>=1.0" \ - ninja "packaging>=24.2" "typing-extensions>=4.10.0" pyyaml six numpy - # These images have no module scanner, and nothing here uses modules. - conda_run env CMAKE_CXX_SCAN_FOR_MODULES=OFF \ - "${build_venv}/bin/python" -m build --wheel --no-isolation - rm -rf "${build_venv}" - pip_install "$(echo dist/*.whl)" - - # A build with no BLAS succeeds silently. Run from / to import the wheel. - (cd / && conda_run python -c " -import torch -assert torch._C.has_lapack, 'built without LAPACK' -torch.linalg.qr(torch.randn(4, 4)) -") +configure_pytorch_compiler +export TORCH_VERSION +export PYTHON_EXECUTABLE="/opt/conda/envs/py_${PYTHON_VERSION}/bin/python" +conda_run bash "$(dirname "${BASH_SOURCE[0]}")/install_pinned_pytorch.sh" - # Grab the pinned audio and vision commits from PyTorch - TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION:-release/2.11} - export TORCHAUDIO_VERSION - TORCHVISION_VERSION=${TORCHVISION_VERSION:-release/0.29} - export TORCHVISION_VERSION - - install_domains - - if [[ "${TORCHCI_REPORTS:-}" == "1" ]]; then - (cd / && conda_run python -c ' +if [[ "${TORCHCI_REPORTS:-}" == "1" ]]; then + (cd / && conda_run python -c ' import os import torch from torch.testing._internal.torchci import environment, plugin, report @@ -117,14 +56,4 @@ if report.SCHEMA_VERSION != "1.0": raise RuntimeError("The image requires torchci report schema 1.0") environment.capture() ') - fi - - popd || true - # Clean up the cloned PyTorch repo to reduce the Docker image size - rm -rf pytorch - - # Print sccache stats for debugging - as_ci_user sccache --show-stats -} - -install_pytorch_and_domains +fi diff --git a/.ci/docker/ubuntu/Dockerfile b/.ci/docker/ubuntu/Dockerfile index 7f9f58ab439..bc47fab9a0e 100644 --- a/.ci/docker/ubuntu/Dockerfile +++ b/.ci/docker/ubuntu/Dockerfile @@ -72,13 +72,14 @@ ARG TORCHCI_REPORTS ARG SKIP_PYTORCH ARG PYTORCH_BUILD_MAX_JOBS COPY ./common/install_pytorch.sh install_pytorch.sh +COPY ./common/install_pinned_pytorch.sh install_pinned_pytorch.sh COPY ./common/utils.sh utils.sh # A host docker build let sccache reach its S3 bucket with the EC2 instance # role; a BuildKit pod has no instance metadata, so docker-builds passes the # credentials in as this secret. World-readable because the sccache server runs # as ci-user; a secret mount never lands in a layer. RUN --mount=type=secret,id=aws-credentials,mode=0444 \ - if [ -z "${SKIP_PYTORCH}" ]; then bash ./install_pytorch.sh; fi && rm install_pytorch.sh utils.sh + if [ -z "${SKIP_PYTORCH}" ]; then bash ./install_pytorch.sh; fi && rm install_pytorch.sh install_pinned_pytorch.sh utils.sh ARG LINTRUNNER # Install lintrunner if needed diff --git a/.ci/scripts/install_pinned_pytorch.sh b/.ci/scripts/install_pinned_pytorch.sh new file mode 100755 index 00000000000..bb8446b8cc8 --- /dev/null +++ b/.ci/scripts/install_pinned_pytorch.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +exec bash "${script_dir}/../docker/common/install_pinned_pytorch.sh" "$@" diff --git a/.ci/scripts/setup-linux.sh b/.ci/scripts/setup-linux.sh index feb8a128b17..942c39f5ceb 100755 --- a/.ci/scripts/setup-linux.sh +++ b/.ci/scripts/setup-linux.sh @@ -1,6 +1,7 @@ #!/bin/bash # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -16,11 +17,15 @@ echo "Build tool: $BUILD_TOOL, Mode: $BUILD_MODE" # As Linux job is running inside a Docker container, all of its dependencies # have already been installed, so we use PyTorch build from source here instead # of nightly. This allows CI to test against latest commits from PyTorch +# The image or PyTorch CI may intentionally provide a different source pin. +torch_pin=$(cd / && "${PYTHON_EXECUTABLE:-python}" -c 'import torch; print(torch.version.git_version)') if [[ "${EDITABLE:-false}" == "true" ]]; then install_executorch --use-pt-pinned-commit --editable else install_executorch --use-pt-pinned-commit fi +# Dependency installation must preserve the pinned PyTorch build. +TORCH_VERSION="${torch_pin}" bash .ci/scripts/install_pinned_pytorch.sh --verify-only build_executorch_runner "${BUILD_TOOL}" "${BUILD_MODE}" if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then diff --git a/.ci/scripts/setup-macos.sh b/.ci/scripts/setup-macos.sh index 38863597224..36dbfb6e2e7 100755 --- a/.ci/scripts/setup-macos.sh +++ b/.ci/scripts/setup-macos.sh @@ -1,6 +1,7 @@ #!/bin/bash # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -129,17 +130,19 @@ fi # separate copy from PyPI. sccache is initialized above so source-build # cache misses still hit the cache. print_cmake_info -install_pytorch_and_domains +bash .ci/scripts/install_pinned_pytorch.sh install_pip_dependencies # install_executorch's --use-pt-pinned-commit skips re-installing torch since -# install_pytorch_and_domains already installed the pinned build above. +# install_pinned_pytorch.sh already installed the pinned build above. if [[ "$EDITABLE" == "true" ]]; then install_executorch --use-pt-pinned-commit --editable else install_executorch --use-pt-pinned-commit fi +# Dependency installation must preserve the pinned PyTorch build. +bash .ci/scripts/install_pinned_pytorch.sh --verify-only build_executorch_runner "${BUILD_TOOL}" "${BUILD_MODE}" if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then diff --git a/.ci/scripts/tests/test_install_pinned_pytorch.py b/.ci/scripts/tests/test_install_pinned_pytorch.py new file mode 100644 index 00000000000..9b0b3258b0e --- /dev/null +++ b/.ci/scripts/tests/test_install_pinned_pytorch.py @@ -0,0 +1,346 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +INSTALLER = ROOT / ".ci/scripts/install_pinned_pytorch.sh" +PIN = "a" * 40 + +# Stand-ins exercise the shell flow without downloading or building PyTorch. +TOOL = r"""#!/usr/bin/env python3 +import json, os, pathlib, sys +name = pathlib.Path(sys.argv[0]).name +args = sys.argv[1:] +root = pathlib.Path(os.environ['TEST_ROOT']) +with (root / 'commands').open('a') as out: + out.write(json.dumps([name, *args]) + '\n') +if name == 'uname': + print('arm64' if args else os.environ.get('TEST_SYSTEM', 'Linux')) +elif name == 'install_name_tool': + if args[0] == '-delete_rpath': + marker = root / 'rpath-removed' + if marker.exists(): + sys.exit(1) + marker.touch() +elif name == 'git': + if args[0] == 'clone': + path = pathlib.Path(args[-1]) + path.mkdir() + (path / 'version.txt').write_text('2.15.0a0') + elif args[0] == 'ls-remote': + refs = os.environ.get('TEST_REMOTE_REFS', 'a' * 40 + '\trefs/heads/release/2.15') + print(refs) + if not refs: + sys.exit(2) + elif args[0] == 'rev-parse': + print('a' * (7 if '--short=7' in args else 40)) +elif name in ('chosen-python', 'python'): + if args[:2] == ['-m', 'venv']: + directory = pathlib.Path(args[-1]) / 'bin' + directory.mkdir(parents=True) + (directory / 'python').symlink_to(root / 'bin' / 'chosen-python') + elif args[:2] == ['-m', 'build']: + if os.environ.get('TEST_BUILD_FAIL'): + print('Build failed; installed pin: ' + (root / 'installed').read_text(), file=sys.stderr) + sys.exit(42) + platform = os.environ.get('TEST_WHEEL_PLATFORM') or ('macosx_15_0_arm64' if os.environ.get('TEST_SYSTEM') == 'Darwin' else 'linux_x86_64') + pathlib.Path('dist').mkdir(exist_ok=True) + pathlib.Path(f'dist/torch-2.15.0a0+gitaaaaaaa-cp312-cp312-{platform}.whl').touch() + elif args[:3] == ['-m', 'pip', 'download']: + if not os.environ.get('TEST_CACHE_HIT'): + sys.exit(1) + directory = pathlib.Path(args[args.index('--dest') + 1]) + directory.mkdir(exist_ok=True) + (directory / args[-1].rsplit('/', 1)[1].replace('%2B', '+')).touch() + elif args[:3] == ['-m', 'pip', 'install']: + if any(arg.endswith('.whl') for arg in args): + (root / 'installed').write_text(os.environ.get('TEST_BUILT_PIN', 'a' * 40)) + elif args[:3] == ['-m', 'pip', 'uninstall']: + (root / 'installed').write_text('') + elif args[0] == '-c': + code = args[1] + if 'sys.executable' in code: + print(sys.argv[0]) + elif 'git_version' in code: + print((root / 'installed').read_text()) + elif 'mac_ver' in code: + print('macosx_15_0_arm64') + elif 'python_version_tuple' in code: + print('312') + elif 'has_lapack' in code and os.environ.get('TEST_NO_LAPACK'): + sys.exit('PyTorch was built without LAPACK') + elif 'find_spec' in code: + print(root / 'torch') +""" + + +@unittest.skipIf(os.name == "nt", "The installer tests require a Unix environment") +class TestPinnedPytorch(unittest.TestCase): + def run_installer(self, installed="", ref=PIN, verify=False, system="Linux", **env): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "bin").mkdir() + (root / "installed").write_text(installed) + (root / "torch/lib").mkdir(parents=True) + (root / "torch/lib/libtorch.dylib").touch() + for name in ( + "git", + "uname", + "chosen-python", + "python", + "aws", + "install_name_tool", + ): + tool = root / "bin" / name + tool.write_text(TOOL) + tool.chmod(0o755) + pin = root / ".ci/docker/ci_commit_pins/pytorch.txt" + pin.parent.mkdir(parents=True) + pin.write_text(ref) + result = subprocess.run( + [ + "bash", + str(INSTALLER), + *(["--verify-only"] if verify else []), + str(root), + ], + env={ + **os.environ, + "PATH": f"{root}/bin:" + os.environ["PATH"], + "PYTHON_EXECUTABLE": str(root / "bin/chosen-python"), + "TMPDIR": str(root), + "TEST_ROOT": str(root), + "TEST_SYSTEM": system, + "GITHUB_RUNNER": "1", + **env, + }, + capture_output=True, + text=True, + ) + commands = root / "commands" + calls = ( + [json.loads(line) for line in commands.read_text().splitlines()] + if commands.exists() + else [] + ) + self.assertEqual(list(root.glob("executorch-pytorch.*")), []) + return result, calls + + def test_matching_pin_installs_domains_without_rebuilding_torch(self): + for ref in (PIN, "release/2.15"): + with self.subTest(ref=ref): + result, calls = self.run_installer( + installed=PIN, ref=ref, system="Darwin" + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(call[:2] == ["git", "clone"] for call in calls)) + self.assertFalse(any(call[1:3] == ["-m", "build"] for call in calls)) + self.assertFalse(any("uninstall" in call for call in calls)) + for package in ("audio", "vision"): + self.assertTrue( + any( + "--no-deps" in call + and any(f"{package}.git@" in arg for arg in call) + for call in calls + ) + ) + + def test_domain_source_overrides_are_preserved(self): + result, calls = self.run_installer( + installed=PIN, + TORCHAUDIO_VERSION="audio-pin", + TORCHVISION_VERSION="vision-pin", + ) + self.assertEqual(result.returncode, 0, result.stderr) + arguments = [argument for call in calls for argument in call] + self.assertIn("git+https://github.com/pytorch/audio.git@audio-pin", arguments) + self.assertIn("git+https://github.com/pytorch/vision.git@vision-pin", arguments) + + def test_matching_macos_pin_repairs_rpaths_only_in_install_mode(self): + for verify in (False, True): + with self.subTest(verify=verify): + result, calls = self.run_installer( + installed=PIN, system="Darwin", verify=verify + ) + self.assertEqual(result.returncode, 0, result.stderr) + repairs = [call[1] for call in calls if call[0] == "install_name_tool"] + self.assertEqual( + repairs, + [] if verify else ["-delete_rpath", "-delete_rpath", "-add_rpath"], + ) + + def test_verify_annotated_tag_uses_commit(self): + result, _ = self.run_installer( + installed=PIN, + ref="v2.15.0", + verify=True, + TEST_REMOTE_REFS="b" * 40 + + "\trefs/tags/v2.15.0\n" + + PIN + + "\trefs/tags/v2.15.0^{}", + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_invalid_remote_ref_fails_without_installing(self): + for refs in ( + "", + f"{PIN}\trefs/heads/release/2.15\n{'b' * 40}\trefs/tags/release/2.15", + ): + with self.subTest(refs=refs): + result, calls = self.run_installer( + ref="release/2.15", TEST_REMOTE_REFS=refs + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Cannot resolve PyTorch ref", result.stderr) + self.assertFalse(any(call[1:3] == ["-m", "pip"] for call in calls)) + + def test_verify_matching_pin_does_not_clone_or_install(self): + for ref in (PIN, "release/2.15"): + with self.subTest(ref=ref): + result, calls = self.run_installer(installed=PIN, ref=ref, verify=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(call[:2] == ["git", "clone"] for call in calls)) + self.assertFalse(any(call[1:3] == ["-m", "pip"] for call in calls)) + self.assertTrue( + any("has_lapack" in arg for call in calls for arg in call) + ) + + def test_verify_mismatch_does_not_install(self): + result, calls = self.run_installer(installed="b" * 40, verify=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("pin mismatch", result.stderr) + self.assertFalse( + any( + call[:2] == ["git", "clone"] or call[1:3] == ["-m", "pip"] + for call in calls + ) + ) + + def test_linux_replaces_installed_build_in_both_directions(self): + for installed in ("older", "newer"): + with self.subTest(installed=installed): + result, calls = self.run_installer(installed=installed) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue( + any(call[1:4] == ["-m", "pip", "uninstall"] for call in calls) + ) + self.assertTrue(any(call[1:3] == ["-m", "build"] for call in calls)) + self.assertFalse(any("--user" in call for call in calls)) + + def test_build_failure_preserves_installed_torch(self): + for system in ("Linux", "Darwin"): + with self.subTest(system=system): + result, calls = self.run_installer( + installed="old-pin", system=system, TEST_BUILD_FAIL="1" + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.returncode, 42) + self.assertIn("Build failed; installed pin: old-pin", result.stderr) + self.assertIn( + "Failed to prepare the replacement PyTorch wheel", result.stderr + ) + self.assertIn("have not been removed", result.stderr) + self.assertFalse(any("uninstall" in call for call in calls)) + + def test_branch_ref_resolves_before_matching(self): + result, calls = self.run_installer(installed=PIN, ref="release/2.15") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(any(call[:2] == ["git", "ls-remote"] for call in calls)) + self.assertFalse(any(call[1:3] == ["-m", "build"] for call in calls)) + + def test_wrong_built_commit_fails_verification(self): + result, _ = self.run_installer(TEST_BUILT_PIN="b" * 40) + self.assertNotEqual(result.returncode, 0) + self.assertIn("pin mismatch", result.stderr) + self.assertNotIn("have not been removed", result.stderr) + + def test_macos_uses_cached_wheel(self): + result, calls = self.run_installer(system="Darwin", TEST_CACHE_HIT="1") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(call[1:3] == ["-m", "build"] for call in calls)) + + def test_macos_cache_miss_builds_matching_wheel(self): + result, calls = self.run_installer(system="Darwin") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(any(call[1:3] == ["-m", "build"] for call in calls)) + + def test_macos_upload_requires_github_actions(self): + for actions, runner, upload in ( + ("", "", False), + ("false", "", False), + ("true", "", True), + ("true", "1", False), + ): + with self.subTest(actions=actions, runner=runner): + result, calls = self.run_installer( + system="Darwin", GITHUB_ACTIONS=actions, GITHUB_RUNNER=runner + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(any(call[0] == "aws" for call in calls), upload) + + def test_macos_rejects_wrong_cache_filename(self): + result, calls = self.run_installer( + installed="old-pin", system="Darwin", TEST_WHEEL_PLATFORM="wrong" + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("does not match cache key", result.stderr) + self.assertFalse(any("uninstall" in call for call in calls)) + + def test_verify_branch_mismatch_does_not_install(self): + result, calls = self.run_installer( + installed="b" * 40, ref="release/2.15", verify=True + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("pin mismatch", result.stderr) + self.assertFalse(any(call[1:3] == ["-m", "pip"] for call in calls)) + + def test_python_from_path_is_supported(self): + result, _ = self.run_installer(installed=PIN, PYTHON_EXECUTABLE="python") + self.assertEqual(result.returncode, 0, result.stderr) + + def test_matching_pin_still_checks_lapack(self): + result, _ = self.run_installer(installed=PIN, TEST_NO_LAPACK="1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("without LAPACK", result.stderr) + + def test_docker_wrapper_preserves_conda_environment(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copy(ROOT / ".ci/docker/common/install_pytorch.sh", root) + (root / "utils.sh").write_text( + "conda_run() {\n" + ' test "$TORCH_VERSION" = release/2.15 &&\n' + ' test "$PYTHON_EXECUTABLE" = /opt/conda/envs/py_3.12/bin/python &&\n' + ' test "$1" = bash &&\n' + ' test "$2" = "' + directory + '/install_pinned_pytorch.sh"\n' + "}\n" + ) + result = subprocess.run( + ["bash", str(root / "install_pytorch.sh")], + env={ + **os.environ, + "TORCH_VERSION": "release/2.15", + "PYTHON_VERSION": "3.12", + }, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_windows_is_rejected(self): + result, _ = self.run_installer(system="MINGW64_NT") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Unsupported system", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/.ci/scripts/tests/test_torchci_image_override.py b/.ci/scripts/tests/test_torchci_image_override.py index adad42a2601..d1ca67e565e 100644 --- a/.ci/scripts/tests/test_torchci_image_override.py +++ b/.ci/scripts/tests/test_torchci_image_override.py @@ -1,5 +1,6 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -93,14 +94,15 @@ def test_missing_or_mutable_pin_stops_before_docker(self): def test_docker_forwards_the_domain_pins_and_checks_installed_torch(self): dockerfile = (ROOT / ".ci/docker/ubuntu/Dockerfile").read_text() script = (ROOT / ".ci/docker/common/install_pytorch.sh").read_text() + installer = (ROOT / ".ci/docker/common/install_pinned_pytorch.sh").read_text() for variable in (*PACKAGES.values(), "TORCHCI_REPORTS"): self.assertIn(f"ARG {variable}\n", dockerfile) - self.assertIn("TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION:-release/2.11}", script) - self.assertIn( - "TORCHVISION_VERSION=${TORCHVISION_VERSION:-release/0.29}", script - ) + self.assertIn("${TORCHAUDIO_VERSION:-release/2.11}", installer) + self.assertIn("${TORCHVISION_VERSION:-release/0.29}", installer) self.assertLess( - script.index(" install_domains\n"), + script.index( + 'conda_run bash "$(dirname "${BASH_SOURCE[0]}")/install_pinned_pytorch.sh"' + ), script.index("if torch.version.git_version"), ) self.assertIn('report.SCHEMA_VERSION != "1.0"', script) diff --git a/.ci/scripts/tests/test_torchci_workflow.py b/.ci/scripts/tests/test_torchci_workflow.py index 99226d9073b..388c72201fa 100644 --- a/.ci/scripts/tests/test_torchci_workflow.py +++ b/.ci/scripts/tests/test_torchci_workflow.py @@ -1,5 +1,6 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index c57d3618df1..cdb110a0f19 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -44,168 +44,6 @@ install_pip_dependencies() { popd || return } -dedupe_macos_loader_path_rpaths() { - if [[ "$(uname)" != "Darwin" ]]; then - return - fi - - local torch_lib_dir - local python_executable=${PYTHON_EXECUTABLE:-python} - pushd .. - torch_lib_dir=$("${python_executable}" -c "import importlib.util; print(importlib.util.find_spec('torch').submodule_search_locations[0])")/lib - popd - - if [[ -z "${torch_lib_dir}" || ! -d "${torch_lib_dir}" ]]; then - return - fi - - local torch_libs=( - "libtorch_cpu.dylib" - "libtorch.dylib" - "libc10.dylib" - ) - - for lib_name in "${torch_libs[@]}"; do - local lib_path="${torch_lib_dir}/${lib_name}" - if [[ ! -f "${lib_path}" ]]; then - continue - fi - - local removed=0 - # Repeatedly remove the @loader_path rpath entries until none remain. - while install_name_tool -delete_rpath @loader_path "${lib_path}" 2>/dev/null; do - removed=1 - done - - if [[ "${removed}" == "1" ]]; then - install_name_tool -add_rpath @loader_path "${lib_path}" || true - fi - done -} - -install_domains() { - local python_executable=${PYTHON_EXECUTABLE:-python} - echo "Install torchvision and torchaudio" - "${python_executable}" -m pip install --no-build-isolation \ - "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}" - "${python_executable}" -m pip install --no-build-isolation \ - "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION}" -} - -install_pytorch_and_domains() { - local python_executable=${PYTHON_EXECUTABLE:-python} - pushd .ci/docker || return - TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt) - popd || return - - git clone https://github.com/pytorch/pytorch.git - - # Fetch the target commit - pushd pytorch || return - git checkout "${TORCH_VERSION}" - - local system_name=$(uname) - if [[ "${system_name}" == "Darwin" ]]; then - local platform=$("${python_executable}" -c 'import sysconfig; import platform; v=platform.mac_ver()[0].split(".")[0]; platform=sysconfig.get_platform().split("-"); platform[1]=f"{v}_0"; print("_".join(platform))') - fi - local python_version=$("${python_executable}" -c 'import platform; v=platform.python_version_tuple(); print(f"{v[0]}{v[1]}")') - local torch_release=$(cat version.txt) - # Download key must match the upload key below (basename of dist/*.whl, - # which always carries the build's resolved +gitHASH). Branch-ref pins - # like `release/2.14` would otherwise produce `+gitrelease` here and - # never hit the cache. - local torch_short_hash=$(git rev-parse --short=7 HEAD) - local torch_wheel_path="cached_artifacts/pytorch/executorch/pytorch_wheels/${system_name}/${python_version}" - local torch_wheel_name="torch-${torch_release}%2Bgit${torch_short_hash}-cp${python_version}-cp${python_version}-${platform:-}.whl" - - local cached_torch_wheel="https://gha-artifacts.s3.us-east-1.amazonaws.com/${torch_wheel_path}/${torch_wheel_name}" - # Cache PyTorch wheel is only needed on MacOS, Linux CI already has this as part - # of the Docker image - local torch_wheel_not_found=0 - if [[ "${system_name}" == "Darwin" ]]; then - "${python_executable}" -m pip install "${cached_torch_wheel}" || torch_wheel_not_found=1 - else - torch_wheel_not_found=1 - fi - - # Found no such wheel, we will build it from source then - if [[ "${torch_wheel_not_found}" == "1" ]]; then - echo "No cached wheel found, continue with building PyTorch at ${TORCH_VERSION}" - - git submodule update --init --recursive - if [[ "$(uname -m)" == "aarch64" ]]; then - export BUILD_IGNORE_SVE_UNAVAILABLE=1 - fi - # PyTorch dropped setup.py. Build in a throwaway environment that can see the - # active one, so its build requirements, which pin a cmake that would be - # preferred over the one on PATH, cannot disturb what is installed here. - # - # Keep in sync with pytorch/pyproject.toml [build-system].requires. - local build_venv=/tmp/pytorch-build-venv - rm -rf "${build_venv}" - "${python_executable}" -m venv --system-site-packages "${build_venv}" - "${build_venv}/bin/pip" install build "scikit-build-core>=1.0" ninja \ - "packaging>=24.2" "typing-extensions>=4.10.0" pyyaml six numpy - USE_DISTRIBUTED=1 "${build_venv}/bin/python" -m build --wheel --no-isolation - rm -rf "${build_venv}" - "${python_executable}" -m pip install "$(echo dist/*.whl)" - # A build with no BLAS succeeds silently, so check rather than assume. - (cd / && "${python_executable}" -c " -import torch -assert torch._C.has_lapack, 'built without LAPACK' -torch.linalg.qr(torch.randn(4, 4)) -") - - # Invariant: the basename the build just produced must match the cache - # URL we'd reconstruct on the next run. If they diverge (someone edits - # torch_wheel_name above, or PyTorch renames its wheels), the cache - # will silently miss and every macOS run will fall back to a ~30-min - # source build. Fail loudly so the regression is caught immediately. - shopt -s nullglob - local built_wheels=(dist/*.whl) - shopt -u nullglob - if [[ ${#built_wheels[@]} -ne 1 ]]; then - echo "ERROR: expected exactly 1 wheel in dist/, found ${#built_wheels[@]}" >&2 - exit 1 - fi - local built_wheel_name - built_wheel_name=$(basename "${built_wheels[0]}") - local expected_wheel_name="${torch_wheel_name//\%2B/+}" - if [[ "${built_wheel_name}" != "${expected_wheel_name}" ]]; then - echo "ERROR: built torch wheel name does not match cache URL key:" >&2 - echo " built: ${built_wheel_name}" >&2 - echo " expected: ${expected_wheel_name}" >&2 - echo "Fix torch_wheel_name construction in install_pytorch_and_domains" >&2 - echo "in .ci/scripts/utils.sh" >&2 - exit 1 - fi - - # Only AWS runners have access to S3 - if command -v aws && [[ -z "${GITHUB_RUNNER:-}" ]]; then - for wheel_path in dist/*.whl; do - local wheel_name=$(basename "${wheel_path}") - echo "Caching ${wheel_name}" - aws s3 cp "${wheel_path}" "s3://gha-artifacts/${torch_wheel_path}/${wheel_name}" - done - fi - else - echo "Use cached wheel at ${cached_torch_wheel}" - fi - - dedupe_macos_loader_path_rpaths - # Grab the pinned audio and vision commits from PyTorch - TORCHAUDIO_VERSION=release/2.11 - export TORCHAUDIO_VERSION - TORCHVISION_VERSION=release/0.29 - export TORCHVISION_VERSION - - install_domains - - popd || return - # Print sccache stats for debugging - sccache --show-stats || true -} - build_executorch_runner_buck2() { # Build executorch runtime with retry as this step is flaky on macos CI retry buck2 build //examples/portable/executor_runner:executor_runner