fix(downloader): select protoc builds for macOS and Linux arm64 (osx, aarch_64)

skills/dload-fetch-tool/references/registry-entry.md

@@ -22,14 +22,14 @@ Filters the release's asset list. dload then runs OS/arch detection on every mat
 - match all OS/arch variants of the tool;
 - leave out sibling tools, checksums, signatures, source archives.
 
-Build variants stay in the pattern too. Among the assets for the host, dload prefers the lowest x86-64 microarchitecture level (`amd64` or `amd64_v1` over `amd64_v2`, `_v3`, `_v4`, also after `x86_64` or `x86-64`: the CPU is not checked, so a higher level is only a fallback, still picked when it is the only build for the host), then the host libc (`musl` builds on Alpine and Android, the others elsewhere), then the name with the fewest extra tokens — so `bun-linux-x64.zip` wins over `-baseline`, `-profile` and `-debug` twins — then archives. Android builds are dropped on Linux whenever the entry has a `binary` (without one, every other platform is only ranked lower), and so are OS packages (`.deb`, `.rpm`, `.apk`, `.snap`, `.flatpak`, `.msi`, `.msix`, `.msixbundle`, `.appx`, `.appxbundle`, `.nupkg`, `.dmg`, `.pkg`) and archives dload has no extractor for (known archive formats other than `.zip`, `.tar.gz`, `.tgz`, `.tar`, `.gz`, `.phar` — such as `.tar.xz`, `.tar.bz2`, `.zst`, `.pkg.tar.zst`, `.7z`, `.rar`) everywhere: dload cannot unpack them, so a release that has only such assets for the host gives way to an older one among the 10 newest releases that satisfy `version` (pin `version` to go further back). On ARM macOS and Windows, x86-64 builds are a fallback: Rosetta 2 and the Windows emulation run them. On macOS, a universal build — `universal`, `universal2` or `all` right next to `darwin` or `macos`, like `darwin-universal` or `macOS_all` — counts as native on both Intel and Apple Silicon, so it is kept by a strict selection and ranks with the host-arch build; elsewhere it has no arch.
+Build variants stay in the pattern too. Among the assets for the host, dload prefers the lowest x86-64 microarchitecture level (`amd64` or `amd64_v1` over `amd64_v2`, `_v3`, `_v4`, also after `x86_64` or `x86-64`: the CPU is not checked, so a higher level is only a fallback, still picked when it is the only build for the host), then the host libc (`musl` builds on Alpine and Android, the others elsewhere), then the name with the fewest extra tokens — so `bun-linux-x64.zip` wins over `-baseline`, `-profile` and `-debug` twins — then archives. Android builds are dropped on Linux whenever the entry has a `binary` (without one, every other platform is only ranked lower), and so are OS packages (`.deb`, `.rpm`, `.apk`, `.snap`, `.flatpak`, `.msi`, `.msix`, `.msixbundle`, `.appx`, `.appxbundle`, `.nupkg`, `.dmg`, `.pkg`) and archives dload has no extractor for (known archive formats other than `.zip`, `.tar.gz`, `.tgz`, `.tar`, `.gz`, `.phar` — such as `.tar.xz`, `.tar.bz2`, `.zst`, `.pkg.tar.zst`, `.7z`, `.rar`) everywhere: dload cannot unpack them, so a release that has only such assets for the host gives way to an older one among the 10 newest releases that satisfy `version` (pin `version` to go further back). On ARM macOS and Windows, x86-64 builds are a fallback: Rosetta 2 and the Windows emulation run them. On macOS, a universal build — `universal`, `universal2`, `universal_binary` or `all` right next to `darwin`, `macos` or `osx`, like `darwin-universal`, `osx-universal_binary` or `macOS_all` — runs natively on both Intel and Apple Silicon, so it is kept by a strict selection and ranks right after the host-arch build, ahead of an x86-64 build under Rosetta; elsewhere it has no arch.
 
 Tokens the OS/arch matchers recognise (case-insensitive, bounded by `_` or a word boundary):
 
 | Kind | Tokens |
 |---|---|
-| OS   | `windows`, `linux`, `darwin`, `macos`, `alpine`, `bsd`, `freebsd`, `win32`, `win64`, `android` (wins over `linux`) |
-| Arch | `amd64`, `arm64`, `aarch64`, `x86_64`, `x86-64`, `x64`, `win64` |
+| OS   | `windows`, `linux`, `darwin`, `macos`, `osx`, `alpine`, `bsd`, `freebsd`, `win32`, `win64`, `android` (wins over `linux`) |
+| Arch | `amd64`, `arm64`, `aarch64`, `aarch_64`, `aarch-64`, `x86_64`, `x86-64`, `x64`, `win64` |
 | Libc | `musl`, `musleabi*`, `alpine`; `gnu`, `gnueabi*`, `glibc`; none means glibc |
 
 | Situation | Pattern |

skills/dload-fetch-tool/references/troubleshooting.md

@@ -37,12 +37,12 @@ curl -s "https://api.github.com/repos/<owner>/<repo>/releases/tags/<tag>" \
 
 After filtering, dload picks the host's variant with:
 
-- OS: `/(?:\b|_)(windows|linux|darwin|macos|alpine|bsd|freebsd|win32|win64)(?:\b|_)/i`; an `android` token makes it Android, a bare `musl` makes it Linux
-- Arch: `/(?:\b|_)(amd64|arm64|aarch64|x86[_-]64|x64|win64)(?:\b|_)/i`
+- OS: `/(?:\b|_)(windows|linux|darwin|macos|osx|alpine|bsd|freebsd|win32|win64)(?:\b|_)/i`; an `android` token makes it Android, a bare `musl` makes it Linux
+- Arch: `/(?:\b|_)(amd64|arm64|aarch[_-]?64|x86[_-]64|x64|win64)(?:\b|_)/i`
 
 A candidate that matches neither is discarded. Common offenders:
 
-- Non-standard tokens: `linux64`, `osx`. A universal macOS build (`darwin-universal`, `macOS_all`) has no arch token but is kept on macOS: `universal`, `universal2` or `all` next to `darwin`/`macos` counts as the host arch.
+- Non-standard tokens: `linux64`, `macosx`. A universal macOS build (`darwin-universal`, `osx-universal_binary`, `macOS_all`) has no arch token but is kept on macOS: `universal`, `universal2`, `universal_binary` or `all` next to `darwin`/`macos`/`osx` runs on the host arch, ranked right after the host-arch build.
 - No arch separator: `tool-linux64.tar.gz` — the OS matches, no arch token is extractable.
 
 Compare with the host: `php -r "echo PHP_OS_FAMILY, ' / ', php_uname('m'), PHP_EOL;"`.

src/Module/Common/Architecture.php

@@ -45,14 +45,14 @@ public static function tryFromString(string $arch): ?self
     {
         return match ($arch) {
             'AMD64', 'amd64', 'x64', 'x86_64', 'x86-64', 'win64' => self::X86_64,
-            'arm64', 'aarch64' => self::ARM_64,
+            'arm64', 'aarch64', 'aarch_64', 'aarch-64' => self::ARM_64,
             default => null,
         };
     }
 
     public static function tryFromBuildName(string $name): ?self
     {
-        if (\preg_match('/(?:\b|_)(amd64|arm64|aarch64|x86[_-]64|x64|win64)(?:\b|_)/i', $name, $matches) !== 1) {
+        if (\preg_match('/(?:\b|_)(amd64|arm64|aarch[_-]?64|x86[_-]64|x64|win64)(?:\b|_)/i', $name, $matches) !== 1) {
             return null;
         }
 

src/Module/Common/OperatingSystem.php

@@ -64,7 +64,7 @@ public static function tryFromString(string $name): ?self
         return match (\strtolower($name)) {
             'windows', 'win32', 'win64' => self::Windows,
             'bsd', 'freebsd' => self::BSD,
-            'darwin', 'macos' => self::Darwin,
+            'darwin', 'macos', 'osx' => self::Darwin,
             'android' => self::Android,
             // The libc is a separate trait, see {@see Libc}
             'linux', 'alpine', 'unknown-musl' => self::Linux,
@@ -80,7 +80,7 @@ public static function tryFromBuildName(string $name): ?self
         }
 
         if (\preg_match(
-            '/(?:\b|_)(windows|linux|darwin|macos|alpine|bsd|freebsd|win32|win64)(?:\b|_)/i',
+            '/(?:\b|_)(windows|linux|darwin|macos|osx|alpine|bsd|freebsd|win32|win64)(?:\b|_)/i',
             $name,
             $matches,
         ) === 1) {

src/Module/Downloader/Internal/AssetSelection/AssetName.php

@@ -22,14 +22,14 @@ final class AssetName
     /**
      * Platform tokens: OS, architecture, libc and the vendor part of target triples.
      */
-    private const PLATFORM = '/(?<![a-z0-9])(?:x86[_-]64|amd64|arm64|aarch64|x64|win64|win32|windows|linux|darwin|macos|apple|alpine|android|bsd|freebsd|unknown|pc|musl(?:eabi(?:hf)?)?|gnu(?:eabi(?:hf)?)?|glibc)(?![a-z0-9])/';
+    private const PLATFORM = '/(?<![a-z0-9])(?:x86[_-]64|amd64|arm64|aarch[_-]?64|x64|win64|win32|windows|linux|darwin|macos|osx|apple|alpine|android|bsd|freebsd|unknown|pc|musl(?:eabi(?:hf)?)?|gnu(?:eabi(?:hf)?)?|glibc)(?![a-z0-9])/';
 
     /**
-     * macOS build for every Mac architecture, like `darwin-universal` or `macOS_all`.
+     * macOS build for every Mac architecture, like `darwin-universal`, `osx-universal_binary` or `macOS_all`.
      *
      * `universal` and `all` count only next to the macOS token: elsewhere they are ordinary words.
      */
-    private const UNIVERSAL = '/(?<![a-z0-9])(?:(?:darwin|macos)[-_.](?:universal2?|all)|(?:universal2?|all)[-_.](?:darwin|macos))(?![a-z0-9])/';
+    private const UNIVERSAL = '/(?<![a-z0-9])(?:(?:darwin|macos|osx)[-_.](?:universal(?:2|_binary)?|all)|(?:universal2?|all)[-_.](?:darwin|macos|osx))(?![a-z0-9])/';
 
     /**
      * x86-64 microarchitecture level right after the architecture, like `amd64_v3` or `x86-64-v2`,

src/Module/Downloader/Internal/AssetSelection/Rule/ArchitectureRule.php

@@ -11,10 +11,10 @@
 use Internal\DLoad\Module\Downloader\Internal\AssetSelection\Selection;
 
 /**
- * Prefers assets built for the host architecture, then the ones the host emulates.
+ * Prefers assets built for the host architecture, then universal macOS builds on macOS,
+ * then the ones the host emulates.
  *
- * A universal macOS build counts as built for the host architecture on macOS. A strict selection
- * removes the assets the host cannot run; otherwise they are ranked last.
+ * A strict selection removes the assets the host cannot run; otherwise they are ranked last.
  *
  * @internal
  * @psalm-internal Internal\DLoad\Module\Downloader
@@ -24,8 +24,11 @@ final class ArchitectureRule implements AssetRule
     /** Name of the rank this rule gives. */
     public const KEY = 'arch';
 
+    /** Rank of a macOS build for every Mac architecture: larger than the build for the host one. */
+    public const UNIVERSAL = 1;
+
     /** Rank of an asset the host runs only through emulation. */
-    public const EMULATED = 1;
+    public const EMULATED = 2;
 
     public function __construct(
         private readonly Architecture $architecture,
@@ -40,19 +43,19 @@ public function select(Selection $selection, callable $next): Selection
 
         return $next($selection->rank(
             self::KEY,
-            fn(Candidate $candidate): int => $this->rank($candidate) ?? 2,
+            fn(Candidate $candidate): int => $this->rank($candidate) ?? 3,
         ));
     }
 
     /**
-     * @return int<0, 1>|null Null when the host cannot run the asset.
+     * @return int<0, 2>|null Null when the host cannot run the asset.
      */
     private function rank(Candidate $candidate): ?int
     {
         $arch = $candidate->asset->getArchitecture();
         return match (true) {
             $arch === $this->architecture => 0,
-            $candidate->name->universal && $this->operatingSystem === OperatingSystem::Darwin => 0,
+            $candidate->name->universal && $this->operatingSystem === OperatingSystem::Darwin => self::UNIVERSAL,
             // Rosetta 2 on macOS and the built-in emulation of Windows run x86-64 builds on ARM
             $arch === Architecture::X86_64
                 && $this->architecture === Architecture::ARM_64

tests/Unit/Module/Common/ArchitectureTest.php

@@ -23,6 +23,10 @@ public static function provideBuildNames(): iterable
         yield ['temporal-test-server_1.33.0_macOS_arm64.tar.gz', Architecture::ARM_64];
         yield ['tool_1.0.0_linux_amd64_v3.tar.gz', Architecture::X86_64];
         yield ['tool-linux-x86-64-v3.tar.gz', Architecture::X86_64];
+        yield ['protoc-36.2-linux-aarch_64.zip', Architecture::ARM_64];
+        yield ['protoc-36.2-osx-aarch_64.zip', Architecture::ARM_64];
+        yield ['protoc-36.2-osx-x86_64.zip', Architecture::X86_64];
+        yield ['protoc-36.2-osx-universal_binary.zip', null];
     }
 
     #[DataProvider('provideBuildNames')]

tests/Unit/Module/Common/OperatingSystemTest.php

@@ -25,6 +25,8 @@ public static function provideBuildNames(): iterable
         yield ['temporal-test-server_1.33.0_macOS_arm64.tar.gz', OperatingSystem::Darwin];
         yield ['bun-linux-x64-android-baseline.zip', OperatingSystem::Android];
         yield ['tool-aarch64-linux-android.tar.gz', OperatingSystem::Android];
+        yield ['protoc-36.2-osx-aarch_64.zip', OperatingSystem::Darwin];
+        yield ['protoc-36.2-osx-universal_binary.zip', OperatingSystem::Darwin];
     }
 
     public static function provideHosts(): iterable
@@ -54,6 +56,12 @@ public function androidCanBeRequestedByName(): void
         Assert::same(OperatingSystem::tryFromString('android'), OperatingSystem::Android);
     }
 
+    #[Test]
+    public function osxIsReadAsDarwin(): void
+    {
+        Assert::same(OperatingSystem::tryFromString('osx'), OperatingSystem::Darwin);
+    }
+
     #[Test]
     public function anUnknownHostFamilyIsRejected(): void
     {

tests/Unit/Module/Downloader/AssetSelectionTest.php

@@ -127,6 +127,27 @@ final class AssetSelectionTest
         'vortex-driver-zig-x86_64-linux.zip',
     ];
 
+    /**
+     * Assets of the Protocol Buffers v36.2 release: `osx` for macOS and `aarch_64` for arm64.
+     */
+    private const PROTOC_ASSETS = [
+        'protobuf-36.2.bazel.tar.gz',
+        'protobuf-36.2.bazel.tar.gz.intoto.jsonl',
+        'protobuf-36.2.tar.gz',
+        'protobuf-36.2.zip',
+        'protoc-36.2-linux-aarch_64.zip',
+        'protoc-36.2-linux-ppcle_64.zip',
+        'protoc-36.2-linux-s390_64.zip',
+        'protoc-36.2-linux-x86_32.zip',
+        'protoc-36.2-linux-x86_64.zip',
+        'protoc-36.2-osx-aarch_64.zip',
+        'protoc-36.2-osx-universal_binary.zip',
+        'protoc-36.2-osx-x86_64.zip',
+        'protoc-36.2-win32.zip',
+        'protoc-36.2-win64.zip',
+        'tool_integrity.bzl',
+    ];
+
     private string $tempDir;
 
     public static function provideBunHosts(): \Generator
@@ -162,6 +183,15 @@ public static function provideMagoHosts(): \Generator
         yield 'Linux arm64 musl' => [Architecture::ARM_64, Libc::Musl, 'mago-1.51.2-aarch64-unknown-linux-musl.tar.gz'];
     }
 
+    public static function provideProtocHosts(): \Generator
+    {
+        yield 'Linux x64' => [OperatingSystem::Linux, Architecture::X86_64, 'protoc-36.2-linux-x86_64.zip'];
+        yield 'Linux arm64' => [OperatingSystem::Linux, Architecture::ARM_64, 'protoc-36.2-linux-aarch_64.zip'];
+        yield 'macOS x64' => [OperatingSystem::Darwin, Architecture::X86_64, 'protoc-36.2-osx-x86_64.zip'];
+        yield 'macOS arm64' => [OperatingSystem::Darwin, Architecture::ARM_64, 'protoc-36.2-osx-aarch_64.zip'];
+        yield 'Windows x64' => [OperatingSystem::Windows, Architecture::X86_64, 'protoc-36.2-win64.zip'];
+    }
+
     #[DataProvider('provideBunHosts')]
     #[Test]
     public function bunRegistryEntrySelectsThePlainBuild(
@@ -248,6 +278,39 @@ public function magoBuildForTheHostLibcIsSelected(Architecture $arch, Libc $libc
         Assert::same($result->file->getFilename(), $expected);
     }
 
+    #[DataProvider('provideProtocHosts')]
+    #[Test]
+    public function protocBuildForTheHostIsSelected(OperatingSystem $os, Architecture $arch, string $expected): void
+    {
+        $result = $this->download(
+            self::registryEntry('protoc'),
+            'protocolbuffers/protobuf',
+            'v36.2',
+            self::PROTOC_ASSETS,
+            $os,
+            $arch,
+            Libc::Gnu,
+        );
+
+        Assert::same($result->file->getFilename(), $expected);
+    }
+
+    #[Test]
+    public function protocUniversalBuildIsSelectedWithoutAnArmBuild(): void
+    {
+        $result = $this->download(
+            self::registryEntry('protoc'),
+            'protocolbuffers/protobuf',
+            'v36.2',
+            ['protoc-36.2-linux-aarch_64.zip', 'protoc-36.2-osx-universal_binary.zip', 'protoc-36.2-osx-x86_64.zip'],
+            OperatingSystem::Darwin,
+            Architecture::ARM_64,
+            Libc::Gnu,
+        );
+
+        Assert::same($result->file->getFilename(), 'protoc-36.2-osx-universal_binary.zip');
+    }
+
     #[BeforeTest]
     protected function setUp(): void
     {

tests/Unit/Module/Downloader/Internal/AssetSelection/AssetNameTest.php

@@ -29,6 +29,8 @@ public static function provideExtras(): iterable
         yield 'all macOS architectures' => ['tool_1.0.0_macOS_all.tar.gz', ['tool']];
         yield 'universal not next to macOS' => ['universal-ctags-linux-x64.tar.gz', ['universal', 'ctags']];
         yield 'microarchitecture level' => ['tool_1.0.0_linux_amd64_v3.tar.gz', ['tool']];
+        yield 'osx and aarch_64' => ['protoc-36.2-osx-aarch_64.zip', ['protoc']];
+        yield 'universal binary' => ['protoc-36.2-osx-universal_binary.zip', ['protoc']];
     }
 
     public static function provideUniversal(): iterable
@@ -37,6 +39,8 @@ public static function provideUniversal(): iterable
         yield 'universal before macOS' => ['tigerbeetle-universal-macos.zip', true];
         yield 'macOS_all' => ['tool_1.0.0_macOS_all.tar.gz', true];
         yield 'universal2' => ['tool-macos-universal2.zip', true];
+        yield 'osx-universal_binary' => ['protoc-36.2-osx-universal_binary.zip', true];
+        yield 'osx-universal' => ['tool-osx-universal.tar.gz', true];
         yield 'single architecture' => ['tool-darwin-arm64.tar.gz', false];
         yield 'all without macOS' => ['tool-all-linux-amd64.tar.gz', false];
         yield 'universal without macOS' => ['universal-ctags-linux-x64.tar.gz', false];

tests/Unit/Module/Downloader/Internal/AssetSelection/Rule/ArchitectureRuleTest.php

@@ -30,8 +30,16 @@ public static function provideEmulatingHosts(): iterable
 
     public static function provideMacArchitectures(): iterable
     {
-        yield 'Apple Silicon' => [Architecture::ARM_64, ArchitectureRule::EMULATED];
-        yield 'Intel' => [Architecture::X86_64, 0];
+        yield 'Apple Silicon' => [Architecture::ARM_64, [
+            'tool-darwin-universal.zip' => ArchitectureRule::UNIVERSAL,
+            'tool-darwin-x86_64.zip' => ArchitectureRule::EMULATED,
+            'tool-darwin-aarch64.zip' => 0,
+        ]];
+        yield 'Intel' => [Architecture::X86_64, [
+            'tool-darwin-universal.zip' => ArchitectureRule::UNIVERSAL,
+            'tool-darwin-x86_64.zip' => 0,
+            'tool-darwin-aarch64.zip' => 3,
+        ]];
     }
 
     #[Test]
@@ -52,9 +60,9 @@ public function aGradualSelectionRanksOtherArchitecturesLast(): void
         );
 
         Assert::same(RuleRunner::ranks($selection, 'arch'), [
-            'tool-x86_64.zip' => 2,
+            'tool-x86_64.zip' => 3,
             'tool-aarch64.zip' => 0,
-            'tool.zip' => 2,
+            'tool.zip' => 3,
         ]);
     }
 
@@ -65,7 +73,7 @@ public function armHostsThatEmulateX86KeepItsBuildsAsAFallback(OperatingSystem $
         $selection = RuleRunner::run(new ArchitectureRule(Architecture::ARM_64, $os), self::ASSETS);
 
         Assert::same(RuleRunner::ranks($selection, 'arch'), [
-            'tool-x86_64.zip' => 1,
+            'tool-x86_64.zip' => ArchitectureRule::EMULATED,
             'tool-aarch64.zip' => 0,
         ]);
     }
@@ -79,19 +87,20 @@ public function x86HostsDoNotRunArmBuilds(OperatingSystem $os): void
         Assert::same(RuleRunner::ranks($selection, 'arch'), ['tool-x86_64.zip' => 0]);
     }
 
+    /**
+     * @param array<non-empty-string, int> $expected
+     */
     #[DataProvider('provideMacArchitectures')]
     #[Test]
-    public function aUniversalBuildIsNativeOnMacOs(Architecture $arch, int $x86Rank): void
+    public function aUniversalBuildRanksRightAfterTheHostBuildOnMacOs(Architecture $arch, array $expected): void
     {
         $selection = RuleRunner::run(new ArchitectureRule($arch, OperatingSystem::Darwin), [
             'tool-darwin-universal.zip',
             'tool-darwin-x86_64.zip',
-        ]);
+            'tool-darwin-aarch64.zip',
+        ], strict: false);
 
-        Assert::same(RuleRunner::ranks($selection, 'arch'), [
-            'tool-darwin-universal.zip' => 0,
-            'tool-darwin-x86_64.zip' => $x86Rank,
-        ]);
+        Assert::same(RuleRunner::ranks($selection, 'arch'), $expected);
     }
 
     #[Test]
@@ -102,7 +111,7 @@ public function aUniversalMacOsBuildHasNoArchitectureElsewhere(): void
 
         Assert::same(RuleRunner::ranks(RuleRunner::run($rule, $assets), 'arch'), ['tool-linux-x86_64.zip' => 0]);
         Assert::same(RuleRunner::ranks(RuleRunner::run($rule, $assets, strict: false), 'arch'), [
-            'tool-darwin-universal.zip' => 2,
+            'tool-darwin-universal.zip' => 3,
             'tool-linux-x86_64.zip' => 0,
         ]);
     }