diff --git a/README-es.md b/README-es.md index 47426d0..aa9169d 100644 --- a/README-es.md +++ b/README-es.md @@ -48,6 +48,7 @@ Con DLoad puedes: - [Tipos de Descarga](#tipos-de-descarga) - [Restricciones de Versión](#restricciones-de-versión) - [Opciones de Configuración Avanzadas](#opciones-de-configuración-avanzadas) + - [Registro de Versiones](#registro-de-versiones) - [Construir RoadRunner Personalizado](#construir-roadrunner-personalizado) - [Configuración de Acción de Construcción](#configuración-de-acción-de-construcción) - [Atributos de Acción Velox](#atributos-de-acción-velox) @@ -172,6 +173,7 @@ También puedes descargar la versión más reciente desde [GitHub releases](http | `--stability` | Estabilidad del release (stable, beta) | stable | | `--config` | Ruta al archivo de configuración | ./dload.xml | | `--force`, `-f` | Forzar descarga aunque el binario ya exista | false | +| `--refresh`, `-r` | Comprobar si hay nuevos releases aunque el registro de versiones siga vigente | false | ### Ver Software @@ -348,6 +350,75 @@ Usa restricciones de versión estilo Composer: ``` +### Registro de Versiones + +Resolver una versión significa pedir a GitHub o GitLab la lista de releases del repositorio. DLoad +guarda lo que aprende en un **registro de versiones** local: una pequeña base de datos con los releases +y assets de cada repositorio conocido, un archivo JSON por repositorio. Las versiones nunca expiran. +Lo que expira es la *última comprobación* del repositorio: mientras sea más reciente que `cache-ttl`, +`dload get` se responde desde el registro sin una sola petición a la API. Cuando es más antigua, DLoad +pide a la API solo los releases publicados desde entonces, normalmente una única petición. + +Las páginas de releases se siguen cargando de forma perezosa. La primera ejecución obtiene solo las +páginas necesarias para encontrar un release que cumpla la versión pedida; los releases más antiguos se +cargan después, bajo demanda. + +El registro está activado por defecto y vive en el directorio de caché del usuario +(`$XDG_CACHE_HOME/dload`, `%LOCALAPPDATA%\dload\cache` en Windows, `~/.cache/dload` en otros casos): + +```xml + + + + + +``` + +| Atributo | Variable de entorno | Por defecto | Significado | +|-------------|---------------------|--------------------------------|--------------------------------------------------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | directorio de caché del usuario | Directorio del registro de versiones. | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Segundos que sigue siendo válida la última comprobación. `0` desactiva el registro. | + +La variable de entorno tiene prioridad sobre el atributo de `dload.xml`, y la opción de línea de comandos sobre ambos. + +```bash +# Comprobar si hay nuevos releases aunque la última comprobación siga vigente +./vendor/bin/dload get rr --refresh + +# Olvidar los repositorios de un software, o todo el registro +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear --force +``` + +> [!NOTE] +> El registro solo contiene metadatos de releases: tags, nombres y enlaces de descarga. Las descargas +> no pasan por él y nunca guarda credenciales, así que el directorio puede compartirse o guardarse en +> la caché de CI sin problemas. Si una comprobación falla por un error de red o un límite de la API, se +> usan los releases almacenados; un repositorio nunca visto sigue fallando de forma visible. Un +> release almacenado cuyos assets desaparecieron del origen se elimina del registro en cuanto falla +> su descarga, y la lista de releases se vuelve a obtener antes de que la ejecución se dé por vencida. +> Los borradores de releases de GitHub nunca se entregan; el registro guarda solo sus etiquetas como marcadores ocultos que conservan la posición en la lista. Un release que el proveedor inserta por debajo del inicio de su lista, como un release de GitLab con un `released_at` retroactivo, no lo detecta la comprobación; ejecute `cache:clear` para ese software para incorporarlo. + +En GitHub Actions el directorio puede conservarse entre ejecuciones del workflow, de modo que cada +ejecución gasta el límite de la API solo en los releases publicados desde la anterior: + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +El `github.run_id` en la clave hace que cada ejecución guarde su registro, y `restore-keys` permite +que la siguiente parta del más reciente. Los jobs paralelos de un mismo workflow no ven la caché de los +demás, ya que `actions/cache` la guarda al terminar cada job. + ## Construir RoadRunner Personalizado DLoad soporta la construcción de binarios personalizados de RoadRunner usando la herramienta Velox. Esto es útil cuando necesitas RoadRunner con combinaciones específicas de plugins que no están disponibles en las versiones pre-construidas. diff --git a/README-ru.md b/README-ru.md index eea812f..723aab7 100644 --- a/README-ru.md +++ b/README-ru.md @@ -49,6 +49,7 @@ DLoad решает распространённую проблему в PHP-пр - [Типы загрузки](#типы-загрузки) - [Ограничения версий](#ограничения-версий) - [Расширенные настройки](#расширенные-настройки) + - [Реестр версий](#реестр-версий) - [Сборка кастомного RoadRunner](#сборка-кастомного-roadrunner) - [Настройка действия сборки](#настройка-действия-сборки) - [Атрибуты Velox-действия](#атрибуты-velox-действия) @@ -173,6 +174,7 @@ composer require internal/dload -W | `--stability` | Стабильность релиза (stable, beta) | stable | | `--config` | Путь к конфигурационному файлу | ./dload.xml | | `--force`, `-f` | Принудительная загрузка даже если бинарник уже есть | false | +| `--refresh`, `-r` | Проверить репозитории на новые релизы, даже если реестр версий ещё свежий | false | ### Просмотр ПО @@ -349,6 +351,76 @@ DLoad поддерживает три типа загрузки, которые ``` +### Реестр версий + +Чтобы определить версию, DLoad запрашивает у GitHub или GitLab список релизов репозитория. Всё, +что он узнаёт, сохраняется в локальном **реестре версий**: небольшой базе релизов и ассетов каждого +известного репозитория, по одному JSON-файлу на репозиторий. Версии из реестра не устаревают. +Устаревает только *последняя проверка* репозитория: пока она моложе `cache-ttl`, `dload get` +отвечает из реестра без единого запроса к API. Когда проверка устарела, DLoad запрашивает у API +только релизы, вышедшие после неё, и обычно это один запрос. + +Страницы релизов по-прежнему загружаются лениво. Первый запуск получает столько страниц, сколько +нужно, чтобы найти релиз под запрошенную версию, а более старые релизы догружаются позже, по мере +надобности. + +Реестр включён по умолчанию и живёт в пользовательском каталоге кэша (`$XDG_CACHE_HOME/dload`, +`%LOCALAPPDATA%\dload\cache` в Windows, иначе `~/.cache/dload`): + +```xml + + + + + +``` + +| Атрибут | Переменная окружения | По умолчанию | Значение | +|-------------|----------------------|---------------------------|---------------------------------------------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | каталог кэша пользователя | Каталог реестра версий. | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Сколько секунд действует последняя проверка репозитория. `0` отключает реестр. | + +Переменная окружения имеет приоритет над атрибутом в `dload.xml`, а опция командной строки над обоими. + +```bash +# Проверить репозитории на новые релизы, даже если последняя проверка ещё свежая +./vendor/bin/dload get rr --refresh + +# Забыть репозитории, из которых берётся программа, или весь реестр целиком +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear --force +``` + +> [!NOTE] +> В реестре хранятся только метаданные релизов: теги, имена и ссылки на ассеты. Загрузки через него +> не проходят, учётные данные в нём не сохраняются, поэтому каталог можно свободно передавать между +> машинами и складывать в кэш CI. Если проверка не удалась из-за сетевой ошибки или лимита API, +> используются сохранённые релизы, а репозиторий, который раньше не встречался, по-прежнему +> завершится ошибкой. Сохранённый релиз, ассеты которого исчезли из источника, удаляется из +> реестра сразу после неудачной загрузки, а список релизов запрашивается заново, прежде чем +> запуск завершится ошибкой. +> Черновики релизов GitHub никогда не выдаются; реестр хранит только их теги как скрытые заглушки, занимающие позицию в списке. Релиз, который провайдер вставляет не в начало списка, например релиз GitLab с задним числом в `released_at`, проверка не замечает; чтобы его подхватить, выполните `cache:clear` для этой программы. + +В GitHub Actions каталог можно переносить между запусками workflow, тогда запуск тратит лимит API +только на релизы, вышедшие после предыдущего: + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +`github.run_id` в ключе заставляет каждый запуск сохранять свой реестр, а `restore-keys` позволяет +следующему запуску начать с самого свежего. Параллельные джобы одного workflow кэш друг друга не +видят: `actions/cache` сохраняет его по завершении джобы. + ## Сборка кастомного RoadRunner DLoad поддерживает сборку кастомных бинарников RoadRunner с помощью инструмента сборки Velox. Это полезно когда нужен RoadRunner с определёнными комбинациями плагинов, которые недоступны в готовых релизах. diff --git a/README-zh.md b/README-zh.md index cd65fff..6594d74 100644 --- a/README-zh.md +++ b/README-zh.md @@ -48,6 +48,7 @@ DLoad 解决了 PHP 项目中的一个实际问题：如何在分发 PHP 代码 - [下载类型](#下载类型) - [版本约束](#版本约束) - [高级配置选项](#高级配置选项) + - [版本注册表](#版本注册表) - [构建自定义 RoadRunner](#构建自定义-roadrunner) - [构建动作配置](#构建动作配置) - [Velox 动作属性](#velox-动作属性) @@ -172,6 +173,7 @@ composer require internal/dload -W | `--stability` | 发布稳定性 (stable, beta) | stable | | `--config` | 配置文件路径 | ./dload.xml | | `--force`, `-f` | 即使二进制文件已存在也强制下载 | false | +| `--refresh`, `-r` | 即使版本注册表仍然有效，也检查仓库是否有新发布 | false | ### 查看软件 @@ -348,6 +350,68 @@ DLoad 支持三种下载类型，它们决定了资源的处理方式： ``` +### 版本注册表 + +解析版本意味着向 GitHub 或 GitLab 请求仓库的发布列表。DLoad 会把获取到的信息保存在本地的 +**版本注册表**中：这是一个小型数据库，记录每个已知仓库的发布版本和资产，每个仓库一个 JSON 文件。 +其中的版本永不过期，过期的只是仓库的*最近一次检查*：只要检查时间比 `cache-ttl` 更新，`dload get` +就直接从注册表返回结果，不会发出任何 API 请求。检查过期后，DLoad 只向 API 请求此后发布的版本， +通常只需一次请求。 + +发布页面仍然按需加载。首次运行只获取找到满足所需版本的发布所需的页面，更早的发布会在之后真正需要时再加载。 + +注册表默认启用，位于用户缓存目录（`$XDG_CACHE_HOME/dload`，Windows 下为 `%LOCALAPPDATA%\dload\cache`， +其他情况为 `~/.cache/dload`）： + +```xml + + + + + +``` + +| 属性 | 环境变量 | 默认值 | 含义 | +|-------------|--------------------|--------------|----------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | 用户缓存目录 | 版本注册表所在目录。 | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | 最近一次检查保持有效的秒数。`0` 表示禁用注册表。 | + +环境变量优先于 `dload.xml` 中的属性，命令行选项优先于两者。 + +```bash +# 即使最近一次检查仍然有效，也强制检查仓库是否有新发布 +./vendor/bin/dload get rr --refresh + +# 忘记某个软件所使用的仓库，或清空整个注册表 +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear --force +``` + +> [!NOTE] +> 注册表只保存发布的元数据：标签、名称和资产下载链接。下载不会经过注册表，也不会保存任何凭据， +> 因此该目录可以自由共享或放入 CI 缓存。若因网络错误或 API 速率限制导致检查失败，会使用已保存的发布； +> 从未见过的仓库仍会明确报错。若某个已保存发布的资产在上游已被删除，下载失败后它会立即从注册表中移除， +> 并在本次运行放弃之前重新获取发布列表。 +> GitHub 的草稿发布永远不会被提供；注册表只以隐藏占位符的形式保存其标签，用于占据列表中的位置。若提供方将某个发布插入到列表开头以下的位置，例如 GitLab 中 `released_at` 被回填的发布，检查不会发现它；请对该软件运行 `cache:clear` 以获取它。 + +在 GitHub Actions 中可以在多次工作流运行之间保留该目录，这样每次运行只为上次运行之后发布的版本消耗速率限制： + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +键中的 `github.run_id` 使每次运行都保存自己的注册表，而 `restore-keys` 让下一次运行从最新的注册表开始。 +同一工作流中并行运行的作业彼此看不到缓存，因为 `actions/cache` 在作业结束时才保存缓存。 + ## 构建自定义 RoadRunner DLoad 支持使用 Velox 构建工具来构建自定义 RoadRunner 二进制文件。当你需要包含特定插件组合的 RoadRunner，而这些组合在预构建版本中不可用时，这功能就很有用了。 diff --git a/README.md b/README.md index 628c879..999c71e 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ With DLoad, you can: - [Download Types](#download-types) - [Version Constraints](#version-constraints) - [Advanced Configuration Options](#advanced-configuration-options) + - [Version Registry](#version-registry) - [Building Custom RoadRunner](#building-custom-roadrunner) - [Build Action Configuration](#build-action-configuration) - [Velox Action Attributes](#velox-action-attributes) @@ -174,6 +175,7 @@ Alternatively, you can download the latest release from [GitHub releases](https: | `--stability` | Release stability (stable, beta) | stable | | `--config` | Path to configuration file | ./dload.xml | | `--force`, `-f` | Force download even if binary exists | false | +| `--refresh`, `-r` | Check repositories for new releases even if the version registry is still fresh | false | ### View Software @@ -350,6 +352,75 @@ Use Composer-style version constraints: ``` +### Version Registry + +Resolving a version means asking GitHub or GitLab for the repository's release list. DLoad keeps +what it learns in a local **version registry**: a small database of the releases and assets every +known repository offers, one JSON file per repository. Versions never expire from it. What expires +is the *last check* of a repository: while the check is younger than `cache-ttl`, `dload get` is +answered from the registry without a single API request. When it is older, DLoad asks the API only +for the releases published since the last check, which is usually one request. + +Release pages are still loaded lazily. The first run fetches only as many pages as it takes to find +a release that satisfies the requested version, and older releases are fetched later, on demand, +when a run actually needs one of them. + +The registry is on by default and lives in the per-user cache directory (`$XDG_CACHE_HOME/dload`, +`%LOCALAPPDATA%\dload\cache` on Windows, `~/.cache/dload` otherwise): + +```xml + + + + + +``` + +| Attribute | Environment variable | Default | Meaning | +|-------------|----------------------|----------------------|------------------------------------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | user cache directory | Directory of the version registry. | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Seconds the last check of a repository stays valid. `0` disables the registry. | + +The environment variable takes precedence over the attribute in `dload.xml`, and the command line option over both. + +```bash +# Check the repositories for new releases even if the last check is still fresh +./vendor/bin/dload get rr --refresh + +# Forget the repositories a software package is served from, or the whole registry +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear --force +``` + +> [!NOTE] +> The registry holds release metadata only: tags, names and asset download links. Downloads never +> go through it and credentials are never stored in it, so the directory can be shared or committed +> to a CI cache freely. When a check fails because of a network error or a rate limit, the stored +> releases are used instead, and a repository that was never seen before still fails loudly. +> A stored release whose assets have disappeared upstream is dropped from the registry as soon as +> its download fails, and the release list is fetched again before the run gives up. +> GitHub draft releases are never served; the registry keeps only their tags, as hidden placeholders that hold the position in the listing. A release that a provider inserts below the top of its listing, such as a GitLab release with a backdated `released_at`, is not noticed by a check; run `cache:clear` for that software to pick it up. + +In GitHub Actions the directory can be carried between workflow runs, so a run spends the rate limit +only on releases published since the previous one: + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +The `github.run_id` in the key makes every workflow run save its registry, while `restore-keys` +lets the next run start from the most recent one. Jobs that run in parallel within one workflow do not +see each other's cache, since `actions/cache` saves it when a job ends. + ## Building Custom RoadRunner DLoad supports building custom RoadRunner binaries using the Velox build tool. This is useful when you need RoadRunner with custom plugin combinations that aren't available in pre-built releases. @@ -600,6 +671,9 @@ Add to CI/CD environment variables for automated downloads. > 1,000 requests per hour across all jobs of the repository. With a large job matrix the limit may run out, > and downloads from other repositories may be rejected. Use a personal access token if that happens. +Release lists are also kept in a local version registry, so repeated runs and runs that carry the +registry between them spend the rate limit only on new releases: see [Version Registry](#version-registry). + ## Failure Reporting `dload get` exits with a non-zero code when at least one requested package was not installed, and prints diff --git a/bin/dload b/bin/dload index 053b8e8..612ec56 100755 --- a/bin/dload +++ b/bin/dload @@ -51,6 +51,7 @@ use Symfony\Component\Console\CommandLoader\FactoryCommandLoader; Command\Show::getCommandName() => static fn() => new Command\Show(), Command\Init::getCommandName() => static fn() => new Command\Init(), Command\Build::getCommandName() => static fn() => new Command\Build(), + Command\CacheClear::getCommandName() => static fn() => new Command\CacheClear(), ]), ); $application->setDefaultCommand(Command\Get::getCommandName(), false); diff --git a/composer.lock b/composer.lock index 068b5ba..2c62b61 100644 --- a/composer.lock +++ b/composer.lock @@ -85,21 +85,21 @@ }, { "name": "internal/container", - "version": "1.0.1", + "version": "1.1.0", "source": { "type": "git", "url": "https://github.com/php-internal/container.git", - "reference": "3ea996260fe8459b6fd17e575bef11a0f022df92" + "reference": "a1a0cc23601c4d09bc21511cbccba8653069d50c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-internal/container/zipball/3ea996260fe8459b6fd17e575bef11a0f022df92", - "reference": "3ea996260fe8459b6fd17e575bef11a0f022df92", + "url": "https://api.github.com/repos/php-internal/container/zipball/a1a0cc23601c4d09bc21511cbccba8653069d50c", + "reference": "a1a0cc23601c4d09bc21511cbccba8653069d50c", "shasum": "" }, "require": { "internal/destroy": "^1.0", - "php": ">=8.2", + "php": ">=8.1", "psr/container": "1 - 2", "yiisoft/injector": "^1.2" }, @@ -107,15 +107,22 @@ "psr/container-implementation": "1.0 - 2.0" }, "require-dev": { + "bamarni/composer-bin-plugin": "^1.8", "llm/skills": "^1.12", "revolt/event-loop": "^1.0", - "spiral/code-style": "^2.3.1", + "roxblnfk/unpoly": "^1.8", "testo/codecov": "^0.2.1", "testo/fiber": "^0.1.3", - "testo/testo": "^0.10.46", - "vimeo/psalm": "^6.10 || ^7.0" + "testo/testo": "^0.10.46" }, "type": "library", + "extra": { + "bamarni-bin": { + "bin-links": true, + "forward-command": false, + "target-directory": "tools" + } + }, "autoload": { "psr-4": { "Internal\\Container\\": "src/" @@ -142,7 +149,7 @@ ], "support": { "issues": "https://github.com/php-internal/container/issues", - "source": "https://github.com/php-internal/container/tree/1.0.1" + "source": "https://github.com/php-internal/container/tree/1.1.0" }, "funding": [ { @@ -150,7 +157,7 @@ "type": "boosty" } ], - "time": "2026-09-07T21:11:07+00:00" + "time": "2026-09-08T09:20:40+00:00" }, { "name": "internal/destroy", @@ -213,20 +220,20 @@ }, { "name": "internal/path", - "version": "1.3.0", + "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/php-internal/path.git", - "reference": "3eca0088117a4b2a1523a320d6ebe093e206cbaa" + "reference": "e59fc0d530891483b5f47417ccfecb3d02b2218c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-internal/path/zipball/3eca0088117a4b2a1523a320d6ebe093e206cbaa", - "reference": "3eca0088117a4b2a1523a320d6ebe093e206cbaa", + "url": "https://api.github.com/repos/php-internal/path/zipball/e59fc0d530891483b5f47417ccfecb3d02b2218c", + "reference": "e59fc0d530891483b5f47417ccfecb3d02b2218c", "shasum": "" }, "require": { - "php": ">=8.2" + "php": ">=8.1" }, "require-dev": { "buggregator/trap": "^1.15", @@ -250,7 +257,7 @@ ], "authors": [ { - "name": "Aleksei Gagarin (roxblnfk)", + "name": "Aleksei Gagarin", "homepage": "https://github.com/roxblnfk" } ], @@ -267,7 +274,7 @@ ], "support": { "issues": "https://github.com/php-internal/path/issues", - "source": "https://github.com/php-internal/path/tree/1.3.0" + "source": "https://github.com/php-internal/path/tree/1.4.0" }, "funding": [ { @@ -275,7 +282,7 @@ "type": "boosty" } ], - "time": "2026-08-12T21:55:41+00:00" + "time": "2026-09-08T10:51:37+00:00" }, { "name": "internal/toml", @@ -897,16 +904,16 @@ }, { "name": "symfony/console", - "version": "v7.4.18", + "version": "v7.4.19", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "23d6f88a29f6d0eac45bd77d70307adf83ba7ab0" + "reference": "3a1973458b153f566a8d7499bc06ece323d60063" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/23d6f88a29f6d0eac45bd77d70307adf83ba7ab0", - "reference": "23d6f88a29f6d0eac45bd77d70307adf83ba7ab0", + "url": "https://api.github.com/repos/symfony/console/zipball/3a1973458b153f566a8d7499bc06ece323d60063", + "reference": "3a1973458b153f566a8d7499bc06ece323d60063", "shasum": "" }, "require": { @@ -971,7 +978,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.4.18" + "source": "https://github.com/symfony/console/tree/v7.4.19" }, "funding": [ { @@ -991,7 +998,7 @@ "type": "tidelift" } ], - "time": "2026-08-25T14:18:37+00:00" + "time": "2026-09-13T10:38:41+00:00" }, { "name": "symfony/deprecation-contracts", @@ -1066,16 +1073,16 @@ }, { "name": "symfony/http-client", - "version": "v7.4.18", + "version": "v7.4.19", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "68d81f78d127984ff2e741f0e0d9cb1078f8f3ea" + "reference": "3a523f38dc399337ec45b2ab9628e8c37ceb23c7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/68d81f78d127984ff2e741f0e0d9cb1078f8f3ea", - "reference": "68d81f78d127984ff2e741f0e0d9cb1078f8f3ea", + "url": "https://api.github.com/repos/symfony/http-client/zipball/3a523f38dc399337ec45b2ab9628e8c37ceb23c7", + "reference": "3a523f38dc399337ec45b2ab9628e8c37ceb23c7", "shasum": "" }, "require": { @@ -1143,7 +1150,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.4.18" + "source": "https://github.com/symfony/http-client/tree/v7.4.19" }, "funding": [ { @@ -1163,7 +1170,7 @@ "type": "tidelift" } ], - "time": "2026-08-30T13:49:59+00:00" + "time": "2026-09-08T13:28:43+00:00" }, { "name": "symfony/http-client-contracts", @@ -1751,16 +1758,16 @@ }, { "name": "symfony/string", - "version": "v7.4.15", + "version": "v7.4.19", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "e394af32256bf9e7bf80849d95e589167c10097b" + "reference": "02731ce2bd26514bec9efc0baef2cd0b0cf67f46" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/e394af32256bf9e7bf80849d95e589167c10097b", - "reference": "e394af32256bf9e7bf80849d95e589167c10097b", + "url": "https://api.github.com/repos/symfony/string/zipball/02731ce2bd26514bec9efc0baef2cd0b0cf67f46", + "reference": "02731ce2bd26514bec9efc0baef2cd0b0cf67f46", "shasum": "" }, "require": { @@ -1818,7 +1825,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v7.4.15" + "source": "https://github.com/symfony/string/tree/v7.4.19" }, "funding": [ { @@ -1838,7 +1845,7 @@ "type": "tidelift" } ], - "time": "2026-07-28T07:33:02+00:00" + "time": "2026-09-11T14:46:58+00:00" }, { "name": "yiisoft/injector", @@ -2737,22 +2744,23 @@ }, { "name": "buggregator/trap", - "version": "1.16.0", + "version": "1.16.1", "source": { "type": "git", "url": "https://github.com/buggregator/trap.git", - "reference": "a0634a3963b7708d53bffd383e4d1976bf57af06" + "reference": "46ae4ed023e7b1b96a367ef0033f12d5d99fb8c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/buggregator/trap/zipball/a0634a3963b7708d53bffd383e4d1976bf57af06", - "reference": "a0634a3963b7708d53bffd383e4d1976bf57af06", + "url": "https://api.github.com/repos/buggregator/trap/zipball/46ae4ed023e7b1b96a367ef0033f12d5d99fb8c0", + "reference": "46ae4ed023e7b1b96a367ef0033f12d5d99fb8c0", "shasum": "" }, "require": { "clue/stream-filter": "^1.6", "ext-filter": "*", "ext-sockets": "*", + "internal/container": "^1.1", "internal/destroy": "^1.0", "nyholm/psr7": "^1.8", "php": ">=8.1", @@ -2762,13 +2770,12 @@ "psr/http-message": "^1.1 || ^2", "psr/log": "^2 || ^3", "symfony/console": "^6.4 || ^7 || ^8", - "symfony/var-dumper": "^6.3 || ^7 || ^8", - "yiisoft/injector": "^1.2" + "symfony/var-dumper": "^6.3 || ^7 || ^8" }, "require-dev": { "dereuromark/composer-prefer-lowest": "^0.1.10", "ergebnis/phpunit-slow-test-detector": "^2.14", - "google/protobuf": "^3.25 || ^4.30", + "google/protobuf": "^3.25 || ^4.30 || ^v5.36.1", "phpunit/phpunit": "^10.5.10", "rector/rector": "^1.1", "roxblnfk/unpoly": "^1.8.1", @@ -2798,11 +2805,11 @@ ], "authors": [ { - "name": "Aleksei Gagarin (roxblnfk)", + "name": "Aleksei Gagarin", "homepage": "https://github.com/roxblnfk" }, { - "name": "Pavel Buchnev (butschster)", + "name": "Pavel Buchnev", "homepage": "https://github.com/butschster" } ], @@ -2825,7 +2832,7 @@ ], "support": { "issues": "https://github.com/buggregator/trap/issues", - "source": "https://github.com/buggregator/trap/tree/1.16.0" + "source": "https://github.com/buggregator/trap/tree/1.16.1" }, "funding": [ { @@ -2833,7 +2840,7 @@ "type": "boosty" } ], - "time": "2026-06-16T12:15:11+00:00" + "time": "2026-09-08T09:57:19+00:00" }, { "name": "clue/stream-filter", @@ -3892,16 +3899,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.8.0", + "version": "v5.9.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "044a6a392ff8ad0d61f14370a5fbbd0a0107152f" + "reference": "9e33da9553fe7786f0962b35f4e4ecf01be89def" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/044a6a392ff8ad0d61f14370a5fbbd0a0107152f", - "reference": "044a6a392ff8ad0d61f14370a5fbbd0a0107152f", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/9e33da9553fe7786f0962b35f4e4ecf01be89def", + "reference": "9e33da9553fe7786f0962b35f4e4ecf01be89def", "shasum": "" }, "require": { @@ -3943,22 +3950,22 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.8.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.9.0" }, - "time": "2026-07-04T14:30:18+00:00" + "time": "2026-09-13T18:51:52+00:00" }, { "name": "php-cs-fixer/shim", - "version": "v3.95.24", + "version": "v3.95.25", "source": { "type": "git", "url": "https://github.com/PHP-CS-Fixer/shim.git", - "reference": "fb929e8551bae0239936297f6402b513e98552e2" + "reference": "7bc6ed86d41533dbfa5604fb4dd4d6a656b87254" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PHP-CS-Fixer/shim/zipball/fb929e8551bae0239936297f6402b513e98552e2", - "reference": "fb929e8551bae0239936297f6402b513e98552e2", + "url": "https://api.github.com/repos/PHP-CS-Fixer/shim/zipball/7bc6ed86d41533dbfa5604fb4dd4d6a656b87254", + "reference": "7bc6ed86d41533dbfa5604fb4dd4d6a656b87254", "shasum": "" }, "require": { @@ -3995,7 +4002,7 @@ "description": "A tool to automatically fix PHP code style", "support": { "issues": "https://github.com/PHP-CS-Fixer/shim/issues", - "source": "https://github.com/PHP-CS-Fixer/shim/tree/v3.95.24" + "source": "https://github.com/PHP-CS-Fixer/shim/tree/v3.95.25" }, "funding": [ { @@ -4003,7 +4010,7 @@ "type": "github" } ], - "time": "2026-08-31T15:31:46+00:00" + "time": "2026-09-08T11:12:09+00:00" }, { "name": "php-http/message", @@ -4299,11 +4306,11 @@ }, { "name": "phpstan/phpstan", - "version": "2.2.13", + "version": "2.2.14", "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/9ba9ac76ee9c5cf5b56d58eb5deec6315b7a0260", - "reference": "9ba9ac76ee9c5cf5b56d58eb5deec6315b7a0260", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/9c672e7a8e791dfc3d30e55f683e73fc0b63a3ac", + "reference": "9c672e7a8e791dfc3d30e55f683e73fc0b63a3ac", "shasum": "" }, "require": { @@ -4359,7 +4366,7 @@ "type": "github" } ], - "time": "2026-09-03T20:38:19+00:00" + "time": "2026-09-12T21:39:33+00:00" }, { "name": "psr/event-dispatcher", @@ -4413,21 +4420,21 @@ }, { "name": "rector/rector", - "version": "2.6.6", + "version": "2.6.7", "source": { "type": "git", "url": "https://github.com/rectorphp/rector.git", - "reference": "ca069d6c79feaa6651b423c15d101a27a436093e" + "reference": "f4aa224edb9b0630fb43826d4cb6a49b62e48d4c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/rectorphp/rector/zipball/ca069d6c79feaa6651b423c15d101a27a436093e", - "reference": "ca069d6c79feaa6651b423c15d101a27a436093e", + "url": "https://api.github.com/repos/rectorphp/rector/zipball/f4aa224edb9b0630fb43826d4cb6a49b62e48d4c", + "reference": "f4aa224edb9b0630fb43826d4cb6a49b62e48d4c", "shasum": "" }, "require": { "php": "^7.4|^8.0", - "phpstan/phpstan": "^2.2.10" + "phpstan/phpstan": "^2.2.14" }, "conflict": { "rector/rector-doctrine": "*", @@ -4458,7 +4465,7 @@ ], "support": { "issues": "https://github.com/rectorphp/rector/issues", - "source": "https://github.com/rectorphp/rector/tree/2.6.6" + "source": "https://github.com/rectorphp/rector/tree/2.6.7" }, "funding": [ { @@ -4466,7 +4473,7 @@ "type": "github" } ], - "time": "2026-09-02T09:38:46+00:00" + "time": "2026-09-13T20:17:44+00:00" }, { "name": "revolt/event-loop", @@ -4858,16 +4865,16 @@ }, { "name": "symfony/finder", - "version": "v7.4.17", + "version": "v7.4.19", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "5ce28827081f6d1f0c32eaf3882750f19cb5bbe6" + "reference": "1b900bc6ae5ba60c5eee69c11ba44566190576fe" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/5ce28827081f6d1f0c32eaf3882750f19cb5bbe6", - "reference": "5ce28827081f6d1f0c32eaf3882750f19cb5bbe6", + "url": "https://api.github.com/repos/symfony/finder/zipball/1b900bc6ae5ba60c5eee69c11ba44566190576fe", + "reference": "1b900bc6ae5ba60c5eee69c11ba44566190576fe", "shasum": "" }, "require": { @@ -4902,7 +4909,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.4.17" + "source": "https://github.com/symfony/finder/tree/v7.4.19" }, "funding": [ { @@ -4922,7 +4929,7 @@ "type": "tidelift" } ], - "time": "2026-08-21T12:09:28+00:00" + "time": "2026-09-10T19:14:10+00:00" }, { "name": "symfony/polyfill-php84", @@ -5093,21 +5100,22 @@ }, { "name": "testo/assert", - "version": "0.1.14", + "version": "0.1.15", "source": { "type": "git", "url": "https://github.com/php-testo/assert.git", - "reference": "5f0ae9b80057e9c81655f3355c6f4820b5a4d830" + "reference": "e9287a5815676386b204aa0b43d8081ade985060" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-testo/assert/zipball/5f0ae9b80057e9c81655f3355c6f4820b5a4d830", - "reference": "5f0ae9b80057e9c81655f3355c6f4820b5a4d830", + "url": "https://api.github.com/repos/php-testo/assert/zipball/e9287a5815676386b204aa0b43d8081ade985060", + "reference": "e9287a5815676386b204aa0b43d8081ade985060", "shasum": "" }, "require": { + "internal/container": "^1.1", "php": ">=8.2", - "testo/testo": "0.10.44 - 1" + "testo/testo": "0.10.47 - 1" }, "type": "library", "extra": { @@ -5140,7 +5148,7 @@ "testo" ], "support": { - "source": "https://github.com/php-testo/assert/tree/0.1.14" + "source": "https://github.com/php-testo/assert/tree/0.1.15" }, "funding": [ { @@ -5148,31 +5156,32 @@ "type": "boosty" } ], - "time": "2026-08-27T22:11:20+00:00" + "time": "2026-09-14T13:28:57+00:00" }, { "name": "testo/bench", - "version": "0.1.11", + "version": "0.1.12", "source": { "type": "git", "url": "https://github.com/php-testo/bench.git", - "reference": "e16f372bb063126dc3b29905bfb9a8a1f5b3e5cd" + "reference": "3934a5f27a34e236ee99ba57a7e60654edbcab4c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-testo/bench/zipball/e16f372bb063126dc3b29905bfb9a8a1f5b3e5cd", - "reference": "e16f372bb063126dc3b29905bfb9a8a1f5b3e5cd", + "url": "https://api.github.com/repos/php-testo/bench/zipball/3934a5f27a34e236ee99ba57a7e60654edbcab4c", + "reference": "3934a5f27a34e236ee99ba57a7e60654edbcab4c", "shasum": "" }, "require": { + "internal/container": "^1.1", "php": ">=8.2", "testo/data": "^0.1.9", - "testo/filter": "^0.1.7", + "testo/filter": "^0.1.8", "testo/inline": "^0.1.9", - "testo/testo": "0.10.46 - 1" + "testo/testo": "0.10.47 - 1" }, "require-dev": { - "testo/assert": "^0.1.14" + "testo/assert": "^0.1.15" }, "suggest": { "testo/assert": "Enables the benchmark verdict as an assertion: current must be the fastest within tolerance, otherwise the test fails." @@ -5207,7 +5216,7 @@ "testo" ], "support": { - "source": "https://github.com/php-testo/bench/tree/0.1.11" + "source": "https://github.com/php-testo/bench/tree/0.1.12" }, "funding": [ { @@ -5215,7 +5224,7 @@ "type": "boosty" } ], - "time": "2026-09-07T20:37:39+00:00" + "time": "2026-09-14T13:30:16+00:00" }, { "name": "testo/bridge-mockery", @@ -5282,16 +5291,16 @@ }, { "name": "testo/bridge-rector", - "version": "0.2.5", + "version": "0.2.6", "source": { "type": "git", "url": "https://github.com/php-testo/bridge-rector.git", - "reference": "aa1c6f4f33b9a38fa3056714157858a1ecf4a743" + "reference": "44249489acf6de7e2e2a42703837a3c1f9f495d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-testo/bridge-rector/zipball/aa1c6f4f33b9a38fa3056714157858a1ecf4a743", - "reference": "aa1c6f4f33b9a38fa3056714157858a1ecf4a743", + "url": "https://api.github.com/repos/php-testo/bridge-rector/zipball/44249489acf6de7e2e2a42703837a3c1f9f495d4", + "reference": "44249489acf6de7e2e2a42703837a3c1f9f495d4", "shasum": "" }, "require": { @@ -5300,10 +5309,10 @@ "rector/rector": "^2.6.4" }, "require-dev": { - "testo/assert": "^0.1.14", + "testo/assert": "^0.1.15", "testo/data": "^0.1.9", - "testo/filter": "^0.1.6", - "testo/testo": "0.10.44 - 1" + "testo/filter": "^0.1.8", + "testo/testo": "0.10.47 - 1" }, "suggest": { "testo/testo": "To test your Rector rules inline with the bundled Testo\\Bridge\\Rector\\Testing harness." @@ -5338,7 +5347,7 @@ "testo" ], "support": { - "source": "https://github.com/php-testo/bridge-rector/tree/0.2.5" + "source": "https://github.com/php-testo/bridge-rector/tree/0.2.6" }, "funding": [ { @@ -5346,26 +5355,26 @@ "type": "boosty" } ], - "time": "2026-08-27T22:11:24+00:00" + "time": "2026-09-14T13:30:52+00:00" }, { "name": "testo/bridge-symfony-console", - "version": "0.1.12", + "version": "0.1.13", "source": { "type": "git", "url": "https://github.com/php-testo/bridge-symfony-console.git", - "reference": "cb81f39968130c7a8dff49d7863712c9aebca851" + "reference": "a02634523b87a1b88eb7a89039e5b9f4feae6679" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-testo/bridge-symfony-console/zipball/cb81f39968130c7a8dff49d7863712c9aebca851", - "reference": "cb81f39968130c7a8dff49d7863712c9aebca851", + "url": "https://api.github.com/repos/php-testo/bridge-symfony-console/zipball/a02634523b87a1b88eb7a89039e5b9f4feae6679", + "reference": "a02634523b87a1b88eb7a89039e5b9f4feae6679", "shasum": "" }, "require": { "php": ">=8.2", "symfony/console": "^6.4 || ^7 || ^8.0", - "testo/testo": "0.10.46 - 1" + "testo/testo": "0.10.47 - 1" }, "bin": [ "bin/testo" @@ -5398,7 +5407,7 @@ "testo" ], "support": { - "source": "https://github.com/php-testo/bridge-symfony-console/tree/0.1.12" + "source": "https://github.com/php-testo/bridge-symfony-console/tree/0.1.13" }, "funding": [ { @@ -5406,28 +5415,29 @@ "type": "boosty" } ], - "time": "2026-09-07T20:37:46+00:00" + "time": "2026-09-14T13:29:41+00:00" }, { "name": "testo/codecov", - "version": "0.2.1", + "version": "0.2.2", "source": { "type": "git", "url": "https://github.com/php-testo/codecov.git", - "reference": "dbb028f5e2c65b9ef6227bb9d67460bc10be37b8" + "reference": "26a42d74698d2db76c6e8c274625049762549d9c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-testo/codecov/zipball/dbb028f5e2c65b9ef6227bb9d67460bc10be37b8", - "reference": "dbb028f5e2c65b9ef6227bb9d67460bc10be37b8", + "url": "https://api.github.com/repos/php-testo/codecov/zipball/26a42d74698d2db76c6e8c274625049762549d9c", + "reference": "26a42d74698d2db76c6e8c274625049762549d9c", "shasum": "" }, "require": { "ext-xmlwriter": "*", + "internal/container": "^1.1", "php": ">=8.2", "testo/data": "^0.1.9", - "testo/inline": "^0.1.8", - "testo/testo": "0.10.44 - 1" + "testo/inline": "^0.1.9", + "testo/testo": "0.10.47 - 1" }, "type": "library", "extra": { @@ -5456,7 +5466,7 @@ "testo" ], "support": { - "source": "https://github.com/php-testo/codecov/tree/0.2.1" + "source": "https://github.com/php-testo/codecov/tree/0.2.2" }, "funding": [ { @@ -5464,7 +5474,7 @@ "type": "boosty" } ], - "time": "2026-08-27T22:11:28+00:00" + "time": "2026-09-14T13:29:43+00:00" }, { "name": "testo/convention", @@ -5637,21 +5647,22 @@ }, { "name": "testo/filter", - "version": "0.1.7", + "version": "0.1.8", "source": { "type": "git", "url": "https://github.com/php-testo/filter.git", - "reference": "c48cedd9b93cea192e12470fc3e0cc88a30234f6" + "reference": "d387beee996d5910bd26cec62abc952089ecc4b5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-testo/filter/zipball/c48cedd9b93cea192e12470fc3e0cc88a30234f6", - "reference": "c48cedd9b93cea192e12470fc3e0cc88a30234f6", + "url": "https://api.github.com/repos/php-testo/filter/zipball/d387beee996d5910bd26cec62abc952089ecc4b5", + "reference": "d387beee996d5910bd26cec62abc952089ecc4b5", "shasum": "" }, "require": { + "internal/container": "^1.1", "php": ">=8.2", - "testo/testo": "0.10.46 - 1" + "testo/testo": "0.10.47 - 1" }, "type": "library", "extra": { @@ -5683,7 +5694,7 @@ "testo" ], "support": { - "source": "https://github.com/php-testo/filter/tree/0.1.7" + "source": "https://github.com/php-testo/filter/tree/0.1.8" }, "funding": [ { @@ -5691,7 +5702,7 @@ "type": "boosty" } ], - "time": "2026-09-07T20:36:21+00:00" + "time": "2026-09-14T13:27:34+00:00" }, { "name": "testo/inline", @@ -5925,21 +5936,21 @@ }, { "name": "testo/test", - "version": "0.1.7", + "version": "0.1.8", "source": { "type": "git", "url": "https://github.com/php-testo/test.git", - "reference": "79ba7fbf3b9c10c08d5f63acb7b831c168fa48e2" + "reference": "8bfad5018a7fb467a8bc6a3d89596031f52b826f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-testo/test/zipball/79ba7fbf3b9c10c08d5f63acb7b831c168fa48e2", - "reference": "79ba7fbf3b9c10c08d5f63acb7b831c168fa48e2", + "url": "https://api.github.com/repos/php-testo/test/zipball/8bfad5018a7fb467a8bc6a3d89596031f52b826f", + "reference": "8bfad5018a7fb467a8bc6a3d89596031f52b826f", "shasum": "" }, "require": { "php": ">=8.2", - "testo/testo": "0.10.46 - 1" + "testo/testo": "0.10.47 - 1" }, "type": "library", "extra": { @@ -5971,7 +5982,7 @@ "testo" ], "support": { - "source": "https://github.com/php-testo/test/tree/0.1.7" + "source": "https://github.com/php-testo/test/tree/0.1.8" }, "funding": [ { @@ -5979,25 +5990,25 @@ "type": "boosty" } ], - "time": "2026-09-07T20:37:34+00:00" + "time": "2026-09-14T13:29:43+00:00" }, { "name": "testo/testo", - "version": "0.10.46", + "version": "0.10.47", "source": { "type": "git", "url": "https://github.com/php-testo/testo.git", - "reference": "ebb2dd4784472e85446dbac7b798453c72e94fa2" + "reference": "c498cf5fd4c3bc4441bce46bce2e151fa638cf58" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-testo/testo/zipball/ebb2dd4784472e85446dbac7b798453c72e94fa2", - "reference": "ebb2dd4784472e85446dbac7b798453c72e94fa2", + "url": "https://api.github.com/repos/php-testo/testo/zipball/c498cf5fd4c3bc4441bce46bce2e151fa638cf58", + "reference": "c498cf5fd4c3bc4441bce46bce2e151fa638cf58", "shasum": "" }, "require": { "ext-tokenizer": "*", - "internal/container": "^1.0", + "internal/container": "^1.1", "internal/destroy": "^1.0", "internal/path": "^1.3", "php": ">=8.2", @@ -6006,19 +6017,19 @@ "psr/log": "^2.0 || ^3.0", "symfony/console": "^6.4 || ^7 || ^8.0", "symfony/finder": "^6.4 || ^7 || ^8.0", - "testo/assert": "^0.1.14", - "testo/bench": "^0.1.11", - "testo/bridge-symfony-console": "^0.1.12", - "testo/codecov": "^0.2.1", + "testo/assert": "^0.1.15", + "testo/bench": "^0.1.12", + "testo/bridge-symfony-console": "^0.1.13", + "testo/codecov": "^0.2.2", "testo/convention": "^0.1.5", "testo/data": "^0.1.9", "testo/fiber": "^0.1.3", - "testo/filter": "^0.1.7", + "testo/filter": "^0.1.8", "testo/inline": "^0.1.9", "testo/lifecycle": "^0.1.6", "testo/repeat": "^0.1.9", "testo/retry": "^0.1.5", - "testo/test": "^0.1.7", + "testo/test": "^0.1.8", "yiisoft/injector": "^1.2" }, "replace": { @@ -6028,18 +6039,13 @@ "bamarni/composer-bin-plugin": "^1.8", "buggregator/trap": "^1.10", "llm/skills": "^1.3", - "php-vcr/php-vcr": "^1.6", - "rector/rector": "^2.6.4", "roxblnfk/unpoly": "1.8.2", "testo/bridge-infection": "^0.1.8", - "testo/bridge-mockery": "^0.1.2", - "testo/bridge-rector": "^0.2.5", - "testo/bridge-revolt": "^0.1.2", - "testo/bridge-vcr": "^0.1.1", "testo/facade": "^0.1.1" }, "suggest": { "llm/skills": "Ship Testo's bundled AI-agent skills into your project's skills directory automatically.", + "testo/bridge-double": "Auto-verifies Double mocks, stubs and spies after every test — fiber-safe, no manual verify() calls.", "testo/bridge-infection": "Provides integration with Infection PHP mutation testing framework.", "testo/bridge-mockery": "Provides integration with the Mockery mocking framework." }, @@ -6089,20 +6095,20 @@ "type": "boosty" } ], - "time": "2026-09-07T20:35:14+00:00" + "time": "2026-09-14T13:26:23+00:00" }, { "name": "vimeo/psalm", - "version": "6.16.1", + "version": "6.17.1", "source": { "type": "git", "url": "https://github.com/vimeo/psalm.git", - "reference": "f1f5de594dc76faf8784e02d3dc4716c91c6f6ac" + "reference": "ecdb39e1d470a661f423d08ec8d2db8bdc043ba7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/vimeo/psalm/zipball/f1f5de594dc76faf8784e02d3dc4716c91c6f6ac", - "reference": "f1f5de594dc76faf8784e02d3dc4716c91c6f6ac", + "url": "https://api.github.com/repos/vimeo/psalm/zipball/ecdb39e1d470a661f423d08ec8d2db8bdc043ba7", + "reference": "ecdb39e1d470a661f423d08ec8d2db8bdc043ba7", "shasum": "" }, "require": { @@ -6126,7 +6132,7 @@ "netresearch/jsonmapper": "^5.0", "nikic/php-parser": "^5.0.0", "php": "~8.1.31 || ~8.2.27 || ~8.3.16 || ~8.4.3 || ~8.5.0", - "sebastian/diff": "^4.0 || ^5.0 || ^6.0 || ^7.0 || ^8.0", + "sebastian/diff": "^4.0 || ^5.0 || ^6.0 || ^7.0 || ^8.0 || ^9.0", "spatie/array-to-xml": "^2.17.0 || ^3.0", "symfony/console": "^6.0 || ^7.0 || ^8.0", "symfony/filesystem": "~6.3.12 || ~6.4.3 || ^7.0.3 || ^8.0", @@ -6207,7 +6213,7 @@ "issues": "https://github.com/vimeo/psalm/issues", "source": "https://github.com/vimeo/psalm" }, - "time": "2026-03-19T10:56:09+00:00" + "time": "2026-09-12T13:51:46+00:00" }, { "name": "webmozart/assert", diff --git a/dload.xsd b/dload.xsd index 793255e..b737373 100644 --- a/dload.xsd +++ b/dload.xsd @@ -249,6 +249,16 @@ Temporary directory for downloads + + + Directory of the version registry (release lists database); the per-user cache directory when not set. The DLOAD_CACHE_DIR environment variable takes precedence + + + + + Number of seconds the last check of a repository for new releases stays valid; 0 disables the version registry. The DLOAD_CACHE_TTL environment variable takes precedence + + diff --git a/psalm-baseline.xml b/psalm-baseline.xml index a80494d..c884d1c 100644 --- a/psalm-baseline.xml +++ b/psalm-baseline.xml @@ -174,6 +174,7 @@ + diff --git a/src/Bootstrap.php b/src/Bootstrap.php index 887153a..65ab415 100644 --- a/src/Bootstrap.php +++ b/src/Bootstrap.php @@ -12,8 +12,15 @@ use Internal\DLoad\Module\Common\Internal\Injection\ConfigInflector; use Internal\DLoad\Module\Common\OperatingSystem; use Internal\DLoad\Module\Common\Stability; +use Internal\DLoad\Module\Config\Schema\Cache as CacheConfig; use Internal\DLoad\Module\HttpClient\Factory; use Internal\DLoad\Module\HttpClient\Internal\NyholmFactoryImpl; +use Internal\DLoad\Module\Registry\Internal\CacheDirectory; +use Internal\DLoad\Module\Registry\Internal\FileRegistryStorage; +use Internal\DLoad\Module\Registry\Internal\PassThroughRegistry; +use Internal\DLoad\Module\Registry\Internal\StoredVersionRegistry; +use Internal\DLoad\Module\Registry\RegistryStorage; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Internal\GitHub\Factory as GithubRepositoryFactory; use Internal\DLoad\Module\Repository\Internal\GitLab\Factory as GitLabRepositoryFactory; use Internal\DLoad\Module\Repository\RepositoryProvider; @@ -21,6 +28,8 @@ use Internal\DLoad\Module\Velox\Builder; use Internal\DLoad\Module\Velox\Internal\Client\BuildRoadRunner; use Internal\DLoad\Module\Velox\Internal\VeloxBuilder; +use Internal\DLoad\Service\Logger; +use Internal\Path; /** * Bootstraps the application by configuring the dependency container. @@ -113,6 +122,32 @@ public function withConfig( ->addRepositoryFactory($container->get(GithubRepositoryFactory::class)) ->addRepositoryFactory($container->get(GitLabRepositoryFactory::class)), ); + $this->container->bind( + RegistryStorage::class, + static function (Container $container) use ($environment): RegistryStorage { + $config = $container->get(CacheConfig::class); + + return new FileRegistryStorage( + $config->dir === null ? CacheDirectory::resolve($environment) : Path::create($config->dir), + $container->get(Logger::class), + ); + }, + ); + $this->container->bind( + VersionRegistry::class, + static function (Container $container): VersionRegistry { + $config = $container->get(CacheConfig::class); + + return $config->ttl <= 0 + ? new PassThroughRegistry() + : new StoredVersionRegistry( + $container->get(RegistryStorage::class), + $config->ttl, + $container->get(Logger::class), + $config->refresh, + ); + }, + ); $this->container->bind(BinaryProvider::class, BinaryProviderImpl::class); $this->container->bind(Factory::class, NyholmFactoryImpl::class); $this->container->bind(Builder::class, VeloxBuilder::class); diff --git a/src/Command/CacheClear.php b/src/Command/CacheClear.php new file mode 100644 index 0000000..67d62af --- /dev/null +++ b/src/Command/CacheClear.php @@ -0,0 +1,127 @@ +addArgument( + self::ARG_SOFTWARE, + InputArgument::OPTIONAL | InputArgument::IS_ARRAY, + 'Software whose repositories must be forgotten, e.g. "rr", "dolt". Everything when omitted.', + ); + $this->addOption( + self::OPTION_FORCE, + 'f', + InputOption::VALUE_NONE, + 'Clear the whole registry without asking for confirmation', + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + parent::execute($input, $output); + + $storage = $this->container->get(RegistryStorage::class); + + /** @var list $software */ + $software = \array_values(\array_filter( + (array) $input->getArgument(self::ARG_SOFTWARE), + static fn(mixed $name): bool => \is_string($name) && $name !== '', + )); + + if ($software === []) { + if (!$this->confirmed($input, $output)) { + $output->writeln('The version registry is left as it is.'); + + return Command::SUCCESS; + } + + $storage->clear(); + $output->writeln('The version registry has been cleared.'); + + return Command::SUCCESS; + } + + $removed = 0; + foreach ($this->recordsOf($storage, $software) as $record) { + $storage->remove($record->id); + $output->writeln(\sprintf('Forgot %s', OutputFormatter::escape((string) $record->id))); + ++$removed; + } + + $output->writeln(\sprintf('%d repository listing(s) removed.', $removed)); + + return Command::SUCCESS; + } + + /** + * Whether the whole registry may be dropped: a non-interactive run goes ahead, a person is asked. + */ + private function confirmed(InputInterface $input, OutputInterface $output): bool + { + if ((bool) $input->getOption(self::OPTION_FORCE) || !$input->isInteractive()) { + return true; + } + + /** @var QuestionHelper $helper */ + $helper = $this->getHelper('question'); + + return (bool) $helper->ask( + $input, + $output, + new ConfirmationQuestion('Forget every stored release listing? [y/N] ', false), + ); + } + + /** + * @param list $software + * @return \Generator + */ + private function recordsOf(RegistryStorage $storage, array $software): \Generator + { + foreach ($storage->all() as $record) { + \array_intersect($record->software, $software) === [] or yield $record; + } + } +} diff --git a/src/Command/Get.php b/src/Command/Get.php index ac23e49..345c0f6 100644 --- a/src/Command/Get.php +++ b/src/Command/Get.php @@ -43,6 +43,9 @@ * * # Force download even if binary exists * ./vendor/bin/dload get rr --force + * + * # Check for new releases even if the version registry is still fresh + * ./vendor/bin/dload get rr --refresh * ``` * * @internal @@ -72,6 +75,12 @@ public function configure(): void $this->addOption('os', null, InputOption::VALUE_OPTIONAL, 'Operating system, e.g. "linux", "darwin" etc.'); $this->addOption('stability', null, InputOption::VALUE_OPTIONAL, 'Minimum stability, e.g. "rc", "beta" etc.'); $this->addOption('force', 'f', InputOption::VALUE_NONE, 'Force download even if binary exists'); + $this->addOption( + 'refresh', + 'r', + InputOption::VALUE_NONE, + 'Check repositories for new releases even if the version registry is still fresh', + ); } /** diff --git a/src/Module/Common/Internal/Injection/ConfigInflector.php b/src/Module/Common/Internal/Injection/ConfigInflector.php index 036c5ee..0f02f03 100644 --- a/src/Module/Common/Internal/Injection/ConfigInflector.php +++ b/src/Module/Common/Internal/Injection/ConfigInflector.php @@ -70,9 +70,27 @@ public function inflect(object $object, Container $container): object return $object; } + /** + * Rank of a configuration source; a lower value is consulted first. + */ + private static function priority(ConfigAttribute $attribute): int + { + return match (true) { + $attribute instanceof InputArgument, $attribute instanceof InputOption => 0, + $attribute instanceof Env => 1, + $attribute instanceof PhpIni => 2, + default => 3, + }; + } + /** * Injects values into a property based on its configuration attributes. * + * The first source that has a value wins. Sources are consulted in a fixed order, whatever + * the order of the attributes on the property: what was typed on the command line beats the + * environment, and the environment beats the configuration file, so a CI job can override a + * committed `dload.xml` without editing it. + * * @param list<\ReflectionAttribute> $attributes */ private function injectValue( @@ -81,10 +99,14 @@ private function injectValue( \ReflectionProperty $property, array $attributes, ): void { - foreach ($attributes as $attribute) { - try { - $attribute = $attribute->newInstance(); + $instances = \array_map( + static fn(\ReflectionAttribute $attribute): ConfigAttribute => $attribute->newInstance(), + $attributes, + ); + \usort($instances, static fn(ConfigAttribute $a, ConfigAttribute $b): int => self::priority($a) <=> self::priority($b)); + foreach ($instances as $attribute) { + try { /** @var mixed $value */ $value = match (true) { $attribute instanceof XPath => $this->getXPath($attribute), diff --git a/src/Module/Config/Schema/Cache.php b/src/Module/Config/Schema/Cache.php new file mode 100644 index 0000000..2fdcb22 --- /dev/null +++ b/src/Module/Config/Schema/Cache.php @@ -0,0 +1,43 @@ +repoConfig = \array_shift($repositories); $repository = $this->repositoryProvider->getByConfig($context->repoConfig); + + // The registry keeps track of which software is served from which repository. The + // identity comes from the repository, not the config: the factory may have reduced + // a full URL to the path the repository stores its releases under. + $context->repositoryId = new RepositoryId($context->repoConfig->type, $repository->getName()); + $this->registry->attach($context->repositoryId, $context->software->getId()); $context->repositoryAttempt = $context->diagnostics->addRepository( type: $context->repoConfig->type, name: $repository->getName(), @@ -161,9 +172,12 @@ public function download( * @param DownloadContext $context Download context information * @return \Closure(): ReleaseInterface Closure that returns the selected release */ - private function processRepository(Repository $repository, DownloadContext $context): \Closure + private function processRepository(Repository $repository, DownloadContext $context, bool $mayRetry = true): \Closure { - return function () use ($repository, $context): ReleaseInterface { + return function () use ($repository, $context, $mayRetry): ReleaseInterface { + // Set when a release turned out to be deleted: the release list is outdated then + $forgotten = false; + $this->logger->info( 'Loading releases from `%s` repository %s', $context->repoConfig->type, @@ -200,7 +214,16 @@ private function processRepository(Repository $repository, DownloadContext $cont } process_release: - $releases === [] and throw new NotFound('No relevant release found.'); + if ($releases === []) { + // The list was outdated: ask the repository again once, with the deleted releases forgotten + /** @var bool $forgotten */ + if ($forgotten && $mayRetry) { + return $this->retryRepository($context); + } + + throw new NotFound('No relevant release found.'); + } + $context->release = \array_shift($releases); $context->releaseAttempt = $context->repositoryAttempt->addRelease($context->release->getName()); @@ -209,6 +232,15 @@ private function processRepository(Repository $repository, DownloadContext $cont try { await(coroutine($this->processRelease($context))); return $context->release; + } catch (ReleaseGone $e) { + // The registry must not offer this release again, and the list needs a fresh check + $tag = $context->release->getVersion()->string; + $tag === '' or $this->registry->forget($context->repositoryId, $tag); + $forgotten = true; + + $context->releaseAttempt->reason ??= $e->getMessage(); + $this->logger->debug($e->getMessage()); + goto process_release; } catch (NotFound $e) { $context->releaseAttempt->reason ??= $e->getMessage(); $this->logger->debug($e->getMessage()); @@ -218,6 +250,23 @@ private function processRepository(Repository $repository, DownloadContext $cont }; } + /** + * Fetches the release list anew after deleted releases were forgotten and tries once more. + * + * @throws NotFound When the fresh list has nothing suitable either. + */ + private function retryRepository(DownloadContext $context): ReleaseInterface + { + $this->logger->info('Release list of `%s` is outdated, fetching it again.', $context->repoConfig->uri); + $repository = $this->repositoryProvider->getByConfig($context->repoConfig); + + try { + return await(coroutine($this->processRepository($repository, $context, mayRetry: false))); + } finally { + $repository instanceof Destroyable and $repository->destroy(); + } + } + /** * Processes a release to find suitable assets. * @@ -391,8 +440,17 @@ private function findAssetWithGradualFiltering(DownloadContext $context): AssetI */ private function tryProcessAssets(array $assets, DownloadContext $context): AssetInterface { + // Stays true while every failed asset answered "not found": then the release itself is gone + $gone = $assets !== []; + process_asset: - $assets === [] and throw new NotFound('none of the matching assets could be downloaded'); + if ($assets === []) { + /** @var bool $gone */ + $gone and throw new ReleaseGone('every matching asset of the release is no longer available'); + + throw new NotFound('none of the matching assets could be downloaded'); + } + $context->asset = \array_shift($assets); $this->logger->debug('Trying to load asset `%s`', $context->asset->getName()); try { @@ -402,6 +460,7 @@ private function tryProcessAssets(array $assets, DownloadContext $context): Asse // Retrying other assets makes the situation worse: report the limit immediately throw $e; } catch (\Throwable $e) { + $gone = $gone && $e instanceof AssetNotFoundException; $context->releaseAttempt->addFailure($context->asset->getName(), $e); $this->logger->exception($e, important: false); goto process_asset; diff --git a/src/Module/Downloader/Exception/ReleaseGone.php b/src/Module/Downloader/Exception/ReleaseGone.php new file mode 100644 index 0000000..30b1c3f --- /dev/null +++ b/src/Module/Downloader/Exception/ReleaseGone.php @@ -0,0 +1,13 @@ + $env Environment variables. + * @param bool $windows Whether the platform conventions of Windows apply. + */ + public static function resolve(array $env, bool $windows = \DIRECTORY_SEPARATOR === '\\'): Path + { + $xdg = self::variable($env, 'XDG_CACHE_HOME'); + if ($xdg !== null) { + return Path::create($xdg)->join('dload'); + } + + $localAppData = self::variable($env, 'LOCALAPPDATA'); + if ($localAppData !== null && $windows) { + return Path::create($localAppData)->join('dload', 'cache'); + } + + $home = self::variable($env, 'HOME') ?? self::variable($env, 'USERPROFILE'); + if ($home !== null) { + return Path::create($home)->join('.cache', 'dload'); + } + + // The temporary directory is shared by every user of the host: keep the registries apart + $user = self::variable($env, 'USER') ?? self::variable($env, 'USERNAME') ?? self::processOwner(); + + return Path::create(\sys_get_temp_dir())->join('dload-cache-' . (string) \preg_replace('/[^A-Za-z0-9._-]+/', '_', $user)); + } + + /** + * @return non-empty-string + */ + private static function processOwner(): string + { + if (\function_exists('posix_geteuid')) { + return (string) \posix_geteuid(); + } + + $owner = \get_current_user(); + + return $owner === '' ? 'default' : $owner; + } + + /** + * @param array $env + * @return non-empty-string|null + */ + private static function variable(array $env, string $name): ?string + { + /** @var mixed $value */ + $value = $env[$name] ?? null; + $path = \is_string($value) ? \rtrim(\trim($value), '/\\') : ''; + + return $path === '' ? null : $path; + } +} diff --git a/src/Module/Registry/Internal/FileRegistryStorage.php b/src/Module/Registry/Internal/FileRegistryStorage.php new file mode 100644 index 0000000..652a66c --- /dev/null +++ b/src/Module/Registry/Internal/FileRegistryStorage.php @@ -0,0 +1,333 @@ +/repositories/github/roadrunner-server/roadrunner/index.json + * /repositories/github/roadrunner-server/roadrunner/releases-0001.json + * /repositories/gitlab/group/project/index.json + * ``` + * + * Files are written aside and renamed into place, so an interrupted or parallel run cannot + * leave a half-written file for anyone to read. Segments are written before the index, so the + * index never points at a file that is not there yet. A save or removal holds a lock on the + * repository, so two runs cannot interleave their files: without it one could drop the segments + * the other has just written as orphans, or publish an index over the other's segment content. + * + * ``` + * /locks/github_roadrunner-server_roadrunner.lock + * ``` + * + * The lock files live beside the repositories, not inside them, because Windows refuses to + * remove a directory holding an open file. + * + * @internal + * @psalm-internal Internal\DLoad + */ +final class FileRegistryStorage implements RegistryStorage +{ + private const REPOSITORIES_DIR = 'repositories'; + private const LOCKS_DIR = 'locks'; + private const INDEX_FILE = 'index.json'; + private const SEGMENT_PREFIX = 'releases-'; + private const EXTENSION = '.json'; + private const TEMP_EXTENSION = '.tmp'; + private const LOCK_EXTENSION = '.lock'; + + /** Age after which a leftover temporary file of a crashed run is removed, in seconds. */ + private const STALE_TEMP_AGE = 3600; + + /** Pause between two attempts to take a lock held by another run, in microseconds. */ + private const LOCK_RETRY_DELAY = 50_000; + + private readonly Path $root; + private readonly Path $locks; + + /** + * @param float $lockTimeout Seconds to wait for a lock held by another run before giving up. + */ + public function __construct( + Path $directory, + private readonly Logger $logger, + private readonly float $lockTimeout = 10.0, + ) { + $this->root = $directory->join(self::REPOSITORIES_DIR); + $this->locks = $directory->join(self::LOCKS_DIR); + } + + public function load(RepositoryId $id): ?RepositoryRecord + { + $record = $this->readIndex($this->directoryOf($id)); + + // Sanitizing and case-insensitive file systems may map two identities onto one directory + return $record?->id->equals($id) === true ? $record : null; + } + + public function save(RepositoryRecord $record): void + { + $this->locked($record->id, function () use ($record): void { + $directory = $this->directoryOf($record->id); + $directory->isDir() or FS::mkdir($directory); + $directory->isDir() or throw new \RuntimeException(\sprintf('Failed to create registry directory `%s`.', $directory)); + + foreach ($record->segments as $segment) { + $segment->dirty and $this->write( + $this->segmentFile($directory, $segment->key), + \array_map(static fn(ReleaseRecord $release): array => $release->toArray(), $segment->releases()), + ); + } + + $this->write($directory->join(self::INDEX_FILE), $record->toArray()); + + $this->removeOrphans($directory, $record); + }); + } + + public function all(): iterable + { + if (!$this->root->isDir()) { + return; + } + + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator((string) $this->root, \FilesystemIterator::SKIP_DOTS), + ); + + /** @var \SplFileInfo $file */ + foreach ($iterator as $file) { + if (!$file->isFile() || $file->getFilename() !== self::INDEX_FILE) { + continue; + } + + $record = $this->readIndex(Path::create($file->getPath())); + $record === null or yield $record; + } + } + + public function remove(RepositoryId $id): void + { + $this->locked($id, function () use ($id): void { + $directory = $this->directoryOf($id); + if (!$directory->isDir()) { + return; + } + + FS::removeDir($directory); + + // Owner and type directories are worth nothing once empty + for ($parent = $directory->parent(); (string) $parent !== (string) $this->root && $this->isEmptyDir($parent); $parent = $parent->parent()) { + FS::removeDir($parent); + } + }); + } + + public function clear(): void + { + $this->root->isDir() and FS::removeDir($this->root); + $this->locks->isDir() and FS::removeDir($this->locks); + } + + /** + * Keeps a path segment safe for every file system: anything but plain ASCII is replaced, + * a segment that would otherwise be empty or a directory reference gets a placeholder, and + * a Windows device name (`nul`, `con`, `com1`...) gets a prefix, as Windows opens the device + * whatever the extension. + * + * @return non-empty-string + */ + private static function sanitize(string $segment): string + { + $safe = (string) \preg_replace('/[^A-Za-z0-9._-]+/', '_', $segment); + + if ($safe === '' || \trim($safe, '.') === '') { + return '_'; + } + + return \preg_match('/^(?:con|prn|aux|nul|com[0-9]|lpt[0-9])(?:\.|$)/i', $safe) === 1 ? '_' . $safe : $safe; + } + + /** + * Runs the action while holding the exclusive lock of the repository. + * + * @throws \RuntimeException When the lock cannot be taken within the timeout. + */ + private function locked(RepositoryId $id, \Closure $action): void + { + $this->locks->isDir() or FS::mkdir($this->locks); + $file = $this->locks->join(\implode('_', \array_map(self::sanitize(...), [$id->type, ...\explode('/', $id->uri)])) . self::LOCK_EXTENSION); + + $handle = @\fopen((string) $file, 'c'); + $handle === false and throw new \RuntimeException(\sprintf('Failed to open registry lock `%s`.', $file)); + + try { + // Blocking `flock()` cannot give up; polling puts a bound on a lock a stuck run holds + $deadline = \microtime(true) + $this->lockTimeout; + while (!\flock($handle, \LOCK_EX | \LOCK_NB)) { + \microtime(true) < $deadline or throw new \RuntimeException( + \sprintf('Another run holds the registry record of `%s` for longer than %.0f second(s).', $id, $this->lockTimeout), + ); + \usleep(self::LOCK_RETRY_DELAY); + } + + try { + $action(); + } finally { + \flock($handle, \LOCK_UN); + } + } finally { + \fclose($handle); + } + } + + /** + * Reads the index of a record, or returns `null` when there is none or it cannot be used. + */ + private function readIndex(Path $directory): ?RepositoryRecord + { + $file = $directory->join(self::INDEX_FILE); + if (!$file->isFile()) { + return null; + } + + try { + $record = RepositoryRecord::fromArray( + $this->decode($file), + fn(string $key): array => $this->readSegment($directory, $key), + ); + + // A segment file missing now would fail the run when its turn comes + foreach ($record->segments as $segment) { + $this->segmentFile($directory, $segment->key)->isFile() or throw new \RuntimeException( + \sprintf('Registry record `%s` refers to a missing segment `%s`.', $directory, $segment->key), + ); + } + + return $record; + } catch (\Throwable $e) { + // A half-written, hand-edited or outdated record is not worth a failed download: + // report it and let the registry fetch the releases again. + $this->logger->exception($e, important: false); + + return null; + } + } + + /** + * @param non-empty-string $key + * @return list + * @throws \RuntimeException When the segment cannot be read; the record is dropped, so the next run starts afresh. + */ + private function readSegment(Path $directory, string $key): array + { + try { + $releases = []; + /** @var mixed $item */ + foreach ($this->decode($this->segmentFile($directory, $key)) as $item) { + \is_array($item) or throw new \UnexpectedValueException('Registry segment must hold release objects.'); + $releases[] = ReleaseRecord::fromArray($item); + } + + return $releases; + } catch (\Throwable $e) { + $directory->isDir() and FS::removeDir($directory); + + throw new \RuntimeException( + \sprintf('Registry segment `%s` of `%s` is unreadable; the record was dropped.', $key, $directory), + previous: $e, + ); + } + } + + /** + * @return array + * @throws \RuntimeException + * @throws \JsonException + */ + private function decode(Path $file): array + { + $content = @\file_get_contents((string) $file); + $content === false and throw new \RuntimeException(\sprintf('Failed to read registry file `%s`.', $file)); + + /** @var mixed $payload */ + $payload = \json_decode($content, true, 512, \JSON_THROW_ON_ERROR); + \is_array($payload) or throw new \UnexpectedValueException(\sprintf('Registry file `%s` must hold a JSON structure.', $file)); + + return $payload; + } + + /** + * Writes the payload aside and renames it into place. + * + * @throws \RuntimeException When the file cannot be written. + */ + private function write(Path $file, array $payload): void + { + $json = \json_encode($payload, \JSON_THROW_ON_ERROR | \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES); + + $temp = Path::create((string) $file . '.' . (int) \getmypid() . self::TEMP_EXTENSION); + @\file_put_contents((string) $temp, $json) === false and throw new \RuntimeException( + \sprintf('Failed to write registry file `%s`.', $temp), + ); + + if (!FS::moveFile($temp, $file)) { + FS::removeFile($temp); + throw new \RuntimeException(\sprintf('Failed to store registry file `%s`.', $file)); + } + } + + /** + * Removes segment files the index no longer refers to and temporary files a crashed run left behind. + */ + private function removeOrphans(Path $directory, RepositoryRecord $record): void + { + $known = [self::INDEX_FILE]; + foreach ($record->segments as $segment) { + $known[] = self::SEGMENT_PREFIX . self::sanitize($segment->key) . self::EXTENSION; + } + + foreach (new \FilesystemIterator((string) $directory, \FilesystemIterator::SKIP_DOTS) as $file) { + /** @var \SplFileInfo $file */ + $name = $file->getFilename(); + $stale = \str_ends_with($name, self::TEMP_EXTENSION) + ? \time() - $file->getMTime() > self::STALE_TEMP_AGE + : !\in_array($name, $known, true); + + $stale && $file->isFile() and FS::removeFile(Path::create($file->getPathname())); + } + } + + private function isEmptyDir(Path $directory): bool + { + return $directory->isDir() && !(new \FilesystemIterator((string) $directory, \FilesystemIterator::SKIP_DOTS))->valid(); + } + + private function directoryOf(RepositoryId $id): Path + { + return $this->root->join(...\array_map(self::sanitize(...), [$id->type, ...\explode('/', $id->uri)])); + } + + /** + * @param non-empty-string $key + */ + private function segmentFile(Path $directory, string $key): Path + { + return $directory->join(self::SEGMENT_PREFIX . self::sanitize($key) . self::EXTENSION); + } +} diff --git a/src/Module/Registry/Internal/PassThroughRegistry.php b/src/Module/Registry/Internal/PassThroughRegistry.php new file mode 100644 index 0000000..c1fee56 --- /dev/null +++ b/src/Module/Registry/Internal/PassThroughRegistry.php @@ -0,0 +1,38 @@ +pages() as $page) { + yield ReleaseRecord::visible($page->releases); + } + } + + public function attach(RepositoryId $id, string $software): void + { + // Nothing to record + } + + public function forget(RepositoryId $id, string $tag): void + { + // Nothing is stored + } +} diff --git a/src/Module/Registry/Internal/StoredVersionRegistry.php b/src/Module/Registry/Internal/StoredVersionRegistry.php new file mode 100644 index 0000000..578b75b --- /dev/null +++ b/src/Module/Registry/Internal/StoredVersionRegistry.php @@ -0,0 +1,235 @@ + $ttl Seconds the last check stays valid. + * @param bool $refresh Ignore the TTL and check the source for every repository once. + * @param null|\Closure(): int $clock Current unix time; defaults to `time()`. + */ + public function __construct( + private readonly RegistryStorage $storage, + private readonly int $ttl, + private readonly Logger $logger, + private readonly bool $refresh = false, + ?\Closure $clock = null, + ) { + $this->clock = $clock ?? static fn(): int => \time(); + } + + public function releases(RepositoryId $id, ReleaseSource $source): \Generator + { + $record = $this->storage->load($id) ?? RepositoryRecord::empty($id); + + if ($this->refresh || $record->isStale(($this->clock)(), $this->ttl)) { + $record = $this->check($record, $source); + } else { + $this->logger->debug('Releases of `%s` are served from the version registry.', (string) $id); + } + + foreach ($record->pages() as $page) { + yield ReleaseRecord::visible($page); + } + + if ($record->complete) { + return; + } + + // Older releases are loaded only when the consumer actually needs them + yield from $this->extend($record, $source); + } + + public function attach(RepositoryId $id, string $software): void + { + $record = $this->storage->load($id) ?? RepositoryRecord::empty($id); + $updated = $record->withSoftware($software); + + $updated === $record or $this->persist($updated); + } + + public function forget(RepositoryId $id, string $tag): void + { + $record = $this->storage->load($id); + if ($record === null || !$record->has($tag)) { + return; + } + + $this->logger->debug('Release `%s` of `%s` is gone: dropped from the version registry.', $tag, (string) $id); + + // Without the last check the next listing asks the source again + $this->persist($record->withoutRelease($tag)->withoutCheck()); + } + + /** + * @param list $page + */ + private static function hasKnown(RepositoryRecord $record, array $page): bool + { + foreach ($page as $release) { + if ($record->has($release->tag)) { + return true; + } + } + + return false; + } + + /** + * Fetches the releases published since the last check and stores the result. + * + * @throws RepositoryException When the source fails and nothing is stored to fall back on. + */ + private function check(RepositoryRecord $record, ReleaseSource $source): RepositoryRecord + { + try { + $fetched = []; + $complete = $record->complete; + + foreach ($source->pages() as $page) { + // A release the source could not read is missing from the page exactly like a + // deleted one, and `withHead()` would drop it; the stored list stays untouched + // and the next run checks again + if ($page->skipped > 0 && $record->count() > 0) { + $this->logger->debug( + 'The listing of `%s` has %d unreadable release(s); the stored %d are kept as they are.', + (string) $record->id, + $page->skipped, + $record->count(), + ); + + return $record; + } + + $fetched = [...$fetched, ...$page->releases]; + + // The listing ended during the check: everything is known now + $page->last and $complete = true; + + // Reaching a known release means everything newer has been fetched. A record + // without releases cannot hit one, so its check is the first page only. + if ($page->last || $record->count() === 0 || self::hasKnown($record, $page->releases)) { + break; + } + } + + return $this->persist( + $record + ->withHead($fetched) + ->withComplete($complete) + ->withCheckedAt(($this->clock)()), + ); + } catch (RepositoryException $e) { + $record->count() > 0 or throw $e; + + $this->report($e, $record); + + return $record; + } + } + + /** + * Loads the releases older than the stored ones page by page, persisting every page. + * + * @return \Generator, mixed, void> + * @throws RepositoryException + */ + private function extend(RepositoryRecord $record, ReleaseSource $source): \Generator + { + foreach ($source->pages($record->count()) as $page) { + $new = \array_values(\array_filter( + $page->releases, + static fn(ReleaseRecord $release): bool => !$record->has($release->tag), + )); + + $record = $this->persist($record->withTail($new)->withComplete($page->last)); + + $visible = ReleaseRecord::visible($new); + $visible === [] or yield $visible; + } + + $record->complete or $this->persist($record->withComplete(true)); + } + + /** + * Tells why the check was skipped. A rate limit is shown to the user once per run: the stored + * list still works, but it may lack newer releases until the limit resets. Anything else is + * an ordinary transient failure and stays in the debug output. + */ + private function report(RepositoryException $e, RepositoryRecord $record): void + { + $this->logger->exception($e, important: false); + + if ($e instanceof RateLimitException && !$this->rateLimitReported) { + $this->rateLimitReported = true; + $this->logger->error( + 'The API rate limit prevents checking `%s` for new releases; %d stored release(s) are used, newer ones may be missing. %s', + (string) $record->id, + $record->count(), + $e->getMessage(), + ); + + return; + } + + $this->logger->debug( + 'Failed to check `%s` for new releases, %d stored release(s) are used instead.', + (string) $record->id, + $record->count(), + ); + } + + /** + * Stores the record; a storage failure is reported and swallowed. + * + * Returns the record as stored, so later writes do not repeat the segments already written. + */ + private function persist(RepositoryRecord $record): RepositoryRecord + { + try { + $this->storage->save($record); + } catch (\Throwable $e) { + $this->logger->exception($e, important: false); + } + + return $record->persisted(); + } +} diff --git a/src/Module/Registry/Record/AssetRecord.php b/src/Module/Registry/Record/AssetRecord.php new file mode 100644 index 0000000..3c9e601 --- /dev/null +++ b/src/Module/Registry/Record/AssetRecord.php @@ -0,0 +1,80 @@ +|null $size Size in bytes when the source reports it. + * @param non-empty-string|null $contentType MIME type when the source reports it. + * @param non-empty-string|null $digest Content checksum as `:`, e.g. `sha256:9f86d0…`, when the source reports it. + */ + public function __construct( + public readonly string $name, + public readonly string $uri, + public readonly ?int $size = null, + public readonly ?string $contentType = null, + public readonly ?string $digest = null, + ) {} + + /** + * @param array $data + * @throws \InvalidArgumentException When the array does not describe an asset. + */ + public static function fromArray(array $data): self + { + $name = $data['name'] ?? null; + $uri = $data['uri'] ?? null; + \is_string($name) && $name !== '' && \is_string($uri) && $uri !== '' or throw new \InvalidArgumentException( + 'Asset record requires non-empty `name` and `uri`.', + ); + + /** @var mixed $size */ + $size = $data['size'] ?? null; + /** @var mixed $contentType */ + $contentType = $data['content_type'] ?? null; + /** @var mixed $digest */ + $digest = $data['digest'] ?? null; + + return new self( + name: $name, + uri: $uri, + size: \is_int($size) && $size >= 0 ? $size : null, + contentType: \is_string($contentType) && $contentType !== '' ? $contentType : null, + digest: \is_string($digest) && $digest !== '' ? $digest : null, + ); + } + + /** + * @return AssetArray + */ + public function toArray(): array + { + $result = ['name' => $this->name, 'uri' => $this->uri]; + $this->size === null or $result['size'] = $this->size; + $this->contentType === null or $result['content_type'] = $this->contentType; + $this->digest === null or $result['digest'] = $this->digest; + + return $result; + } +} diff --git a/src/Module/Registry/Record/ReleasePage.php b/src/Module/Registry/Record/ReleasePage.php new file mode 100644 index 0000000..4a19216 --- /dev/null +++ b/src/Module/Registry/Record/ReleasePage.php @@ -0,0 +1,28 @@ + $releases Releases of the page, newest first. + * @param bool $last Whether the listing has no page after this one. + * @param int<0, max> $skipped Number of releases of the page the source could not read. They + * are absent from `$releases` although they exist upstream. + */ + public function __construct( + public readonly array $releases, + public readonly bool $last, + public readonly int $skipped = 0, + ) {} +} diff --git a/src/Module/Registry/Record/ReleaseRecord.php b/src/Module/Registry/Record/ReleaseRecord.php new file mode 100644 index 0000000..4b064bb --- /dev/null +++ b/src/Module/Registry/Record/ReleaseRecord.php @@ -0,0 +1,103 @@ +, + * hidden?: bool, + * } + * + * @internal + */ +final class ReleaseRecord +{ + /** + * @param non-empty-string $tag Tag the release was made from; identifies the release within a repository. + * @param non-empty-string $name Human-readable release name. + * @param list $assets + * @param bool $hidden Whether the release is listed by the source but must not be served, as a + * GitHub draft is. It keeps its position, so the stored count still maps onto the paging + * of the source. + */ + public function __construct( + public readonly string $tag, + public readonly string $name, + public readonly ?\DateTimeImmutable $publishedAt = null, + public readonly bool $prerelease = false, + public readonly array $assets = [], + public readonly bool $hidden = false, + ) {} + + /** + * Leaves out the releases that must not be served. + * + * @param list $releases + * @return list + */ + public static function visible(array $releases): array + { + return \array_values(\array_filter($releases, static fn(ReleaseRecord $release): bool => !$release->hidden)); + } + + /** + * @param array $data + * @throws \InvalidArgumentException When the array does not describe a release. + */ + public static function fromArray(array $data): self + { + $tag = $data['tag'] ?? null; + \is_string($tag) && $tag !== '' or throw new \InvalidArgumentException('Release record requires a non-empty `tag`.'); + + /** @var mixed $name */ + $name = $data['name'] ?? null; + \is_string($name) && $name !== '' or $name = $tag; + + /** @var mixed $publishedAt */ + $publishedAt = $data['published_at'] ?? null; + + $assets = []; + /** @var mixed $asset */ + foreach (\is_array($data['assets'] ?? null) ? $data['assets'] : [] as $asset) { + \is_array($asset) and $assets[] = AssetRecord::fromArray($asset); + } + + return new self( + tag: $tag, + name: $name, + publishedAt: \is_string($publishedAt) && $publishedAt !== '' ? new \DateTimeImmutable($publishedAt) : null, + prerelease: (bool) ($data['prerelease'] ?? false), + assets: $assets, + hidden: (bool) ($data['hidden'] ?? false), + ); + } + + /** + * @return ReleaseArray + */ + public function toArray(): array + { + $data = [ + 'tag' => $this->tag, + 'name' => $this->name, + 'published_at' => $this->publishedAt?->format(\DateTimeInterface::ATOM), + 'prerelease' => $this->prerelease, + 'assets' => \array_map(static fn(AssetRecord $asset): array => $asset->toArray(), $this->assets), + ]; + $this->hidden and $data['hidden'] = true; + + return $data; + } +} diff --git a/src/Module/Registry/Record/ReleaseSegment.php b/src/Module/Registry/Record/ReleaseSegment.php new file mode 100644 index 0000000..660b7fc --- /dev/null +++ b/src/Module/Registry/Record/ReleaseSegment.php @@ -0,0 +1,111 @@ +|null */ + private ?array $releases; + + /** @var (\Closure(): list)|null */ + private ?\Closure $loader; + + /** + * @param non-empty-string $key Identifier of the segment, unique within the repository record. + * @param list $tags Tags of the releases, newest first. + * @param list|null $releases Releases, newest first; `null` when a loader provides them. + * @param (\Closure(): list)|null $loader Reads the releases from storage. + * @param bool $dirty Whether the storage does not hold this content yet. + */ + private function __construct( + public readonly string $key, + public readonly array $tags, + ?array $releases, + ?\Closure $loader, + public readonly bool $dirty, + ) { + $this->releases = $releases; + $this->loader = $loader; + } + + /** + * Segment assembled from releases that are not stored yet. + * + * @param non-empty-string $key + * @param non-empty-list $releases Newest first. + */ + public static function fresh(string $key, array $releases): self + { + return new self( + key: $key, + tags: \array_map(static fn(ReleaseRecord $release): string => $release->tag, $releases), + releases: $releases, + loader: null, + dirty: true, + ); + } + + /** + * Segment whose releases are read from storage when first needed. + * + * @param non-empty-string $key + * @param non-empty-list $tags Newest first. + * @param \Closure(): list $loader + */ + public static function stored(string $key, array $tags, \Closure $loader): self + { + return new self(key: $key, tags: $tags, releases: null, loader: $loader, dirty: false); + } + + /** + * @return list Newest first. + * @throws \RuntimeException When the releases cannot be read from storage. + */ + public function releases(): array + { + if ($this->releases === null) { + $this->releases = ($this->loader ?? static fn(): array => [])(); + $this->loader = null; + } + + return $this->releases; + } + + /** + * @return int<0, max> + */ + public function count(): int + { + return \count($this->tags); + } + + /** + * @param non-empty-string $tag + */ + public function has(string $tag): bool + { + return \in_array($tag, $this->tags, true); + } + + /** + * The same segment as the storage holds it now. + */ + public function persisted(): self + { + return $this->dirty + ? new self(key: $this->key, tags: $this->tags, releases: $this->releases, loader: null, dirty: false) + : $this; + } +} diff --git a/src/Module/Registry/Record/RepositoryRecord.php b/src/Module/Registry/Record/RepositoryRecord.php new file mode 100644 index 0000000..949b85e --- /dev/null +++ b/src/Module/Registry/Record/RepositoryRecord.php @@ -0,0 +1,497 @@ +, + * segments: list}>, + * } + * + * @internal + */ +final class RepositoryRecord +{ + /** Format version of the stored payload; bump when the structure changes incompatibly. */ + public const FORMAT_VERSION = 2; + + /** Greatest number of releases a segment holds. */ + public const SEGMENT_SIZE = 100; + + /** @var list Newest first. */ + public readonly array $segments; + + /** @var array Segment position of every stored tag. */ + private readonly array $index; + + /** + * @param int|null $checkedAt Unix timestamp of the last successful check against the source. + * @param bool $complete Whether the stored releases reach the end of the source listing. + * @param list $software Identifiers of the software packages served from this repository. + * @param list $segments Newest first. + */ + public function __construct( + public readonly RepositoryId $id, + public readonly ?int $checkedAt = null, + public readonly bool $complete = false, + public readonly array $software = [], + array $segments = [], + ) { + $index = []; + foreach ($segments as $position => $segment) { + foreach ($segment->tags as $tag) { + $index[$tag] ??= $position; + } + } + + $this->segments = $segments; + $this->index = $index; + } + + public static function empty(RepositoryId $id): self + { + return new self($id); + } + + /** + * Restores the index of a record; the releases of every segment come through the loader. + * + * @param array $data + * @param \Closure(non-empty-string): list $loader Reads the releases of a segment by its key. + * @throws \InvalidArgumentException When the array does not describe a repository record. + */ + public static function fromArray(array $data, \Closure $loader): self + { + ($data['version'] ?? null) === self::FORMAT_VERSION or throw new \InvalidArgumentException( + 'Unsupported repository record format.', + ); + + /** @var mixed $repository */ + $repository = $data['repository'] ?? null; + /** @var mixed $type */ + $type = \is_array($repository) ? ($repository['type'] ?? null) : null; + /** @var mixed $uri */ + $uri = \is_array($repository) ? ($repository['uri'] ?? null) : null; + \is_string($type) && $type !== '' && \is_string($uri) && $uri !== '' or throw new \InvalidArgumentException( + 'Repository record requires a repository type and URI.', + ); + + $segments = []; + /** @var mixed $segment */ + foreach (\is_array($data['segments'] ?? null) ? $data['segments'] : [] as $segment) { + \is_array($segment) or throw new \InvalidArgumentException('Repository record segment must be an object.'); + + /** @var mixed $key */ + $key = $segment['key'] ?? null; + \is_string($key) && $key !== '' or throw new \InvalidArgumentException('Repository record segment requires a `key`.'); + + $tags = []; + /** @var mixed $tag */ + foreach (\is_array($segment['tags'] ?? null) ? $segment['tags'] : [] as $tag) { + \is_string($tag) && $tag !== '' or throw new \InvalidArgumentException('Repository record segment tags must be non-empty strings.'); + $tags[] = $tag; + } + + $tags === [] or $segments[] = ReleaseSegment::stored($key, $tags, static fn(): array => $loader($key)); + } + + $software = []; + /** @var mixed $name */ + foreach (\is_array($data['software'] ?? null) ? $data['software'] : [] as $name) { + \is_string($name) && $name !== '' and $software[] = $name; + } + + /** @var mixed $checkedAt */ + $checkedAt = $data['checked_at'] ?? null; + + return new self( + id: new RepositoryId($type, $uri), + checkedAt: \is_int($checkedAt) ? $checkedAt : null, + complete: (bool) ($data['complete'] ?? false), + software: $software, + segments: $segments, + ); + } + + /** + * Releases segment by segment, newest first; a segment is read from storage when reached. + * + * @return \Generator, mixed, void> + * @throws \RuntimeException When a segment cannot be read from storage. + */ + public function pages(): \Generator + { + foreach ($this->segments as $segment) { + yield $segment->releases(); + } + } + + /** + * Every release, newest first; reads every segment. + * + * @return list + * @throws \RuntimeException When a segment cannot be read from storage. + */ + public function releases(): array + { + $releases = []; + foreach ($this->pages() as $page) { + $releases = [...$releases, ...$page]; + } + + return $releases; + } + + /** + * @return int<0, max> + */ + public function count(): int + { + return \count($this->index); + } + + /** + * @param non-empty-string $tag + */ + public function has(string $tag): bool + { + return isset($this->index[$tag]); + } + + /** + * Whether the last check is older than the given number of seconds, or never happened. + * + * @param int<0, max> $ttl + */ + public function isStale(int $now, int $ttl): bool + { + return $this->checkedAt === null || $now - $this->checkedAt > $ttl; + } + + /** + * Replaces the head of the list with freshly fetched releases. + * + * The fetched releases are the newest ones, and sources only ever add releases at the top, + * so from the first stored release they contain onwards both lists walk the same listing + * positions. A stored release missing from the fetched span at its position was deleted + * upstream and is dropped: keeping it would inflate `count()`, which the registry uses as + * the offset for loading the tail. Stored releases beyond the span are kept, and when the + * fetched releases contain no stored one they are the whole listing. + * + * @param list $fetched Newest first. + */ + public function withHead(array $fetched): self + { + $fetched = self::unique($fetched); + $fetchedTags = \array_fill_keys(\array_map(static fn(ReleaseRecord $release): string => $release->tag, $fetched), true); + + // Flat view of the stored tags with the segment each one belongs to + $stored = []; + foreach ($this->segments as $position => $segment) { + foreach ($segment->tags as $tag) { + $stored[] = [$tag, $position]; + } + } + + $overlap = null; + foreach ($stored as $offset => [$tag]) { + if (isset($fetchedTags[$tag])) { + $overlap = $offset; + break; + } + } + + if ($overlap === null) { + return $this->with(segments: self::pack($fetched, [], $this->nextKey())); + } + + // Listing positions the fetched releases still cover, counting from the overlap + $spanned = 0; + foreach ($fetched as $position => $release) { + if ($release->tag === $stored[$overlap][0]) { + $spanned = \count($fetched) - $position; + break; + } + } + + $keepFrom = \count($stored); + foreach (\array_slice($stored, $overlap, preserve_keys: true) as $offset => [$tag]) { + if ($spanned <= 0) { + $keepFrom = $offset; + break; + } + + isset($fetchedTags[$tag]) and --$spanned; + } + + // The segment holding the first kept release is split unless the release opens it; + // the segments after it stay as they are + $loose = $fetched; + $following = []; + if ($keepFrom < \count($stored)) { + [$tag, $position] = $stored[$keepFrom]; + $segment = $this->segments[$position]; + $start = (int) \array_search($tag, $segment->tags, true); + $start === 0 or $loose = [...$loose, ...\array_slice($segment->releases(), $start)]; + $following = \array_slice($this->segments, $start === 0 ? $position : $position + 1); + } + + return $this->with(segments: self::pack($loose, $following, $this->nextKey())); + } + + /** + * Appends older releases loaded on demand; already known tags are ignored. + * + * The last segment is filled up before a new one starts. + * + * @param list $fetched Newest first. + */ + public function withTail(array $fetched): self + { + $new = \array_values(\array_filter( + self::unique($fetched), + fn(ReleaseRecord $release): bool => !$this->has($release->tag), + )); + if ($new === []) { + return $this; + } + + $segments = $this->segments; + $last = \array_pop($segments); + if ($last === null || $last->count() >= self::SEGMENT_SIZE) { + $last === null or $segments[] = $last; + + return $this->with(segments: [...$segments, ...self::chunk($new, $this->nextKey())]); + } + + return $this->with(segments: [ + ...$segments, + ...self::chunk([...$last->releases(), ...$new], $this->nextKey(), $last->key), + ]); + } + + public function withCheckedAt(int $checkedAt): self + { + return $this->with(checkedAt: $checkedAt); + } + + /** + * Forgets the last check, so the record counts as stale until the source is asked again. + */ + public function withoutCheck(): self + { + return new self( + id: $this->id, + checkedAt: null, + complete: $this->complete, + software: $this->software, + segments: $this->segments, + ); + } + + /** + * Drops a release; an unknown tag leaves the record as it is. + * + * @param non-empty-string $tag + */ + public function withoutRelease(string $tag): self + { + if (!$this->has($tag)) { + return $this; + } + + $position = $this->index[$tag]; + $segments = $this->segments; + $remaining = \array_values(\array_filter( + $segments[$position]->releases(), + static fn(ReleaseRecord $release): bool => $release->tag !== $tag, + )); + + $remaining === [] + ? \array_splice($segments, $position, 1) + : $segments[$position] = ReleaseSegment::fresh($segments[$position]->key, $remaining); + + return $this->with(segments: $segments); + } + + public function withComplete(bool $complete): self + { + return $this->with(complete: $complete); + } + + /** + * @param non-empty-string $software + */ + public function withSoftware(string $software): self + { + return \in_array($software, $this->software, true) + ? $this + : $this->with(software: [...$this->software, $software]); + } + + /** + * The record as the storage holds it now: no segment is dirty any more. + */ + public function persisted(): self + { + return $this->with(segments: \array_map(static fn(ReleaseSegment $segment): ReleaseSegment => $segment->persisted(), $this->segments)); + } + + /** + * The index of the record; the releases of the segments are stored separately. + * + * @return RepositoryArray + */ + public function toArray(): array + { + return [ + 'version' => self::FORMAT_VERSION, + 'repository' => ['type' => $this->id->type, 'uri' => $this->id->uri], + 'checked_at' => $this->checkedAt, + 'complete' => $this->complete, + 'software' => $this->software, + 'segments' => \array_map( + static fn(ReleaseSegment $segment): array => ['key' => $segment->key, 'tags' => $segment->tags], + $this->segments, + ), + ]; + } + + /** + * Packs loose releases into segments in front of the given ones. + * + * The short remainder goes first, where the next check adds its releases, so the head segment + * fills up over several checks and the full segments behind it are never rewritten. When the + * remainder and the first following segment fit into one, they are joined. + * + * @param list $loose Newest first. + * @param list $following Segments that stay as they are. + * @param int<1, max> $nextKey + * @return list + */ + private static function pack(array $loose, array $following, int $nextKey): array + { + if ($loose === []) { + return $following; + } + + $remainder = \count($loose) % self::SEGMENT_SIZE; + $first = $following[0] ?? null; + if ($first !== null && $remainder > 0 && $remainder + $first->count() <= self::SEGMENT_SIZE) { + $loose = [...$loose, ...$first->releases()]; + \array_shift($following); + $remainder = \count($loose) % self::SEGMENT_SIZE; + } + + $head = []; + if ($remainder > 0) { + /** @var non-empty-list $partial */ + $partial = \array_slice($loose, 0, $remainder); + $head[] = ReleaseSegment::fresh(self::key($nextKey++), $partial); + } + + return [...$head, ...self::chunk(\array_slice($loose, $remainder), $nextKey), ...$following]; + } + + /** + * Splits releases into segments of at most `SEGMENT_SIZE`, keying them from `$nextKey` on. + * + * @param list $releases + * @param int<1, max> $nextKey + * @param non-empty-string|null $reuseKey Key for the first segment, when it replaces an existing one. + * @return list + */ + private static function chunk(array $releases, int $nextKey, ?string $reuseKey = null): array + { + $segments = []; + foreach (\array_chunk($releases, self::SEGMENT_SIZE) as $chunk) { + $segments[] = ReleaseSegment::fresh($reuseKey ?? self::key($nextKey++), $chunk); + $reuseKey = null; + } + + return $segments; + } + + /** + * @param int<1, max> $number + * @return non-empty-string + */ + private static function key(int $number): string + { + /** @var non-empty-string */ + return \sprintf('%04d', $number); + } + + /** + * Drops the releases repeating a tag seen before, so no tag lands in two segments. + * + * @param list $releases + * @return list + */ + private static function unique(array $releases): array + { + $seen = []; + $unique = []; + foreach ($releases as $release) { + isset($seen[$release->tag]) or $unique[] = $release; + $seen[$release->tag] = true; + } + + return $unique; + } + + /** + * @return int<1, max> + */ + private function nextKey(): int + { + $max = 0; + foreach ($this->segments as $segment) { + $max = \max($max, (int) $segment->key); + } + + return $max + 1; + } + + /** + * @param list|null $software + * @param list|null $segments + */ + private function with( + ?int $checkedAt = null, + ?bool $complete = null, + ?array $software = null, + ?array $segments = null, + ): self { + return new self( + id: $this->id, + checkedAt: $checkedAt ?? $this->checkedAt, + complete: $complete ?? $this->complete, + software: $software ?? $this->software, + segments: $segments ?? $this->segments, + ); + } +} diff --git a/src/Module/Registry/RegistryStorage.php b/src/Module/Registry/RegistryStorage.php new file mode 100644 index 0000000..4219ad2 --- /dev/null +++ b/src/Module/Registry/RegistryStorage.php @@ -0,0 +1,48 @@ + + */ + public function all(): iterable; + + /** + * Removes the record of a repository; a missing record is not an error. + */ + public function remove(RepositoryId $id): void; + + /** + * Removes every record. + */ + public function clear(): void; +} diff --git a/src/Module/Registry/ReleaseSource.php b/src/Module/Registry/ReleaseSource.php new file mode 100644 index 0000000..1ae4e4c --- /dev/null +++ b/src/Module/Registry/ReleaseSource.php @@ -0,0 +1,31 @@ + $offset Number of newest releases to skip. + * @return \Generator Pages of releases. + * @throws RepositoryException When a page cannot be loaded. + */ + public function pages(int $offset = 0): \Generator; +} diff --git a/src/Module/Registry/RepositoryId.php b/src/Module/Registry/RepositoryId.php new file mode 100644 index 0000000..f6e410b --- /dev/null +++ b/src/Module/Registry/RepositoryId.php @@ -0,0 +1,57 @@ +getName()); + * echo $id; // github:roadrunner-server/roadrunner + * ``` + * + * @internal + */ +final class RepositoryId implements \Stringable +{ + /** @var non-empty-string Repository type, e.g. `github` or `gitlab`. */ + public readonly string $type; + + /** @var non-empty-string Repository path within the type, e.g. `owner/repo`. */ + public readonly string $uri; + + /** + * @param non-empty-string $type + * @param non-empty-string $uri + * @throws \InvalidArgumentException When the URI has nothing but slashes and spaces. + */ + public function __construct(string $type, string $uri) + { + $normalized = \strtolower(\trim($uri, " \t\n\r/")); + $normalized === '' and throw new \InvalidArgumentException(\sprintf('Repository URI `%s` is empty.', $uri)); + + $this->type = \strtolower($type); + $this->uri = $normalized; + } + + public function equals(self $other): bool + { + return $this->type === $other->type && $this->uri === $other->uri; + } + + /** + * @return non-empty-string + */ + public function __toString(): string + { + return $this->type . ':' . $this->uri; + } +} diff --git a/src/Module/Registry/VersionRegistry.php b/src/Module/Registry/VersionRegistry.php new file mode 100644 index 0000000..e123b12 --- /dev/null +++ b/src/Module/Registry/VersionRegistry.php @@ -0,0 +1,56 @@ +releases($id, $source) as $page) { + * foreach ($page as $record) { + * // ... + * } + * } + * ``` + * + * @internal + */ +interface VersionRegistry +{ + /** + * Lists the releases of a repository newest first, page by page. + * + * Older releases that are not in the database yet are loaded from the source only when the + * iteration reaches them, so a consumer that stops early costs no extra request. + * + * @return \Generator, mixed, void> + * @throws RepositoryException When releases cannot be obtained from either the database or the source. + */ + public function releases(RepositoryId $id, ReleaseSource $source): \Generator; + + /** + * Records that a software package is served from the repository. + * + * @param non-empty-string $software Software identifier. + */ + public function attach(RepositoryId $id, string $software): void; + + /** + * Drops a release that turned out to be gone and marks the repository for a check. + * + * Called when the assets of a stored release cannot be downloaded any more: the next listing + * asks the source again instead of trusting the stored record. + * + * @param non-empty-string $tag Tag of the release as stored in the registry. + */ + public function forget(RepositoryId $id, string $tag): void; +} diff --git a/src/Module/Repository/Exception/AssetNotFoundException.php b/src/Module/Repository/Exception/AssetNotFoundException.php new file mode 100644 index 0000000..df3a1a9 --- /dev/null +++ b/src/Module/Repository/Exception/AssetNotFoundException.php @@ -0,0 +1,13 @@ + + */ + public function loaded(): array + { + return \array_values($this->cache); + } + /** * Returns the first item in the cache or from the generator. * diff --git a/src/Module/Repository/Internal/Collection.php b/src/Module/Repository/Internal/Collection.php index b8033b0..32dbda0 100644 --- a/src/Module/Repository/Internal/Collection.php +++ b/src/Module/Repository/Internal/Collection.php @@ -94,6 +94,18 @@ public function filter(callable $filter): static return $clone; } + /** + * Returns the items loaded so far, ignoring filters and without loading anything more. + * + * Iterating a lazy collection may cost requests; releasing what has been loaded must not. + * + * @return list + */ + public function loaded(): array + { + return \is_array($this->items) ? \array_values($this->items) : $this->items->loaded(); + } + /** * Maps each item in the collection using the provided callback. * diff --git a/src/Module/Repository/Internal/GitHub/Api/Client.php b/src/Module/Repository/Internal/GitHub/Api/Client.php index 0cf602f..aca156e 100644 --- a/src/Module/Repository/Internal/GitHub/Api/Client.php +++ b/src/Module/Repository/Internal/GitHub/Api/Client.php @@ -18,13 +18,20 @@ * HTTP client wrapper with GitHub-specific error handling and authentication. * * Converts unsuccessful responses (rate limits, invalid token, missing repository, etc.) - * into exceptions with actionable messages. Adds GitHub API token authentication when available. + * into exceptions with actionable messages. Adds the GitHub API token to requests bound for GitHub hosts. * * @internal * @psalm-internal Internal\DLoad\Module\Repository\Internal\GitHub */ final class Client { + /** + * Hosts the token may be sent to. Asset URLs come from the API response and from the version + * registry on disk, so a tampered file must not be able to point a request with the token at + * a host of its choosing. + */ + private const TRUSTED_HOSTS = ['github.com', 'githubusercontent.com']; + /** * @var array */ @@ -39,9 +46,6 @@ public function __construct( private readonly ClientInterface $client, private readonly GitHub $gitHubConfig, ) { - // Add authorization header if token is available - $this->gitHubConfig->token !== null and $this->defaultHeaders['authorization'] = 'Bearer ' . $this->gitHubConfig->token; - $this->validator = new ResponseValidator(authenticated: $this->gitHubConfig->token !== null); } @@ -52,9 +56,11 @@ public function __construct( */ public function request(Method|string $method, string|UriInterface $uri, array $headers = []): ResponseInterface { - $request = $this->httpFactory->request($method, $uri, $headers + $this->defaultHeaders); + $headers += $this->defaultHeaders; + $this->gitHubConfig->token !== null && self::isTrusted($uri) + and $headers += ['authorization' => 'Bearer ' . $this->gitHubConfig->token]; - return $this->sendRequest($request); + return $this->sendRequest($this->httpFactory->request($method, $uri, $headers)); } /** @@ -72,4 +78,21 @@ public function sendRequest(RequestInterface $request): ResponseInterface return $response; } + + private static function isTrusted(string|UriInterface $uri): bool + { + $host = $uri instanceof UriInterface ? $uri->getHost() : \parse_url($uri, \PHP_URL_HOST); + if (!\is_string($host) || $host === '') { + return false; + } + + $host = \strtolower($host); + foreach (self::TRUSTED_HOSTS as $trusted) { + if ($host === $trusted || \str_ends_with($host, '.' . $trusted)) { + return true; + } + } + + return false; + } } diff --git a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php index c3ce8fb..bc8899f 100644 --- a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php @@ -6,11 +6,11 @@ use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; use Internal\DLoad\Module\HttpClient\Method; +use Internal\DLoad\Module\Registry\Record\ReleasePage; use Internal\DLoad\Module\Repository\Exception\ApiException; use Internal\DLoad\Module\Repository\Exception\RepositoryException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\RepositoryInfo; -use Internal\DLoad\Module\Repository\Internal\Paginator; use Internal\DLoad\Service\Logger; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\UriInterface; @@ -28,6 +28,12 @@ final class RepositoryApi private const URL_REPOSITORY = 'https://api.github.com/repos/%s'; private const URL_RELEASES = 'https://api.github.com/repos/%s/releases'; + /** + * Number of releases to ask for in a single page. GitHub serves 30 by default and allows up to + * 100, so the maximum keeps the release list within as few requests as the API permits. + */ + public const RELEASES_PER_PAGE = 100; + /** * @var non-empty-string */ @@ -79,74 +85,78 @@ public function getRepository(): RepositoryInfo } /** + * Lists releases newest first, page by page, starting from the given page. + * + * A page is requested only when the generator advances to it, so a consumer that stops early + * costs no extra request. + * * @param int<1, max> $page - * @return Paginator + * @return \Generator * @throws RepositoryException */ - public function getReleases(int $page = 1): Paginator + public function releasePages(int $page = 1): \Generator { - $pageLoader = function () use ($page): \Generator { - $currentPage = $page; - - do { - $response = $this->releasesRequest($currentPage); - - /** @var list, - * prerelease: bool, - * draft: bool - * }> $data */ - $data = $this->decodeReleasesResponse($response); - - // If empty response, no more pages - if ($data === []) { - return; + $currentPage = $page; + + do { + $response = $this->releasesRequest($currentPage); + + /** @var list, + * prerelease: bool, + * draft: bool + * }> $data */ + $data = $this->decodeReleasesResponse($response); + + // If empty response, no more pages + if ($data === []) { + return; + } + + $releases = []; + $failure = null; + $skipped = 0; + foreach ($data as $releaseData) { + try { + $releases[] = ReleaseInfo::fromApiResponse($releaseData)->toRecord(); + } catch (\Throwable $e) { + $failure ??= $e; + ++$skipped; + $this->logger->exception($e, important: false); + // Skip invalid releases + continue; } - - $releases = []; - $failure = null; - foreach ($data as $releaseData) { - try { - $releases[] = ReleaseInfo::fromApiResponse($releaseData); - } catch (\Throwable $e) { - $failure ??= $e; - $this->logger->exception($e, important: false); - // Skip invalid releases - continue; - } - } - - // The whole page is unreadable: the response structure is not what we expect - if ($releases === [] && $failure !== null) { - throw new ApiException( - \sprintf( - 'GitHub API returned %d release(s) for repository `%s`, but none of them could be read: %s', - \count($data), - $this->repositoryPath, - $failure->getMessage(), - ), + } + + // The whole page is unreadable: the response structure is not what we expect + if ($releases === [] && $failure !== null) { + throw new ApiException( + \sprintf( + 'GitHub API returned %d release(s) for repository `%s`, but none of them could be read: %s', + \count($data), $this->repositoryPath, - $failure, - ); - } + $failure->getMessage(), + ), + $this->repositoryPath, + $failure, + ); + } - yield $releases; + $hasMorePages = $this->hasNextPage($response); - // Check if there are more pages - $hasMorePages = $this->hasNextPage($response); - $currentPage++; - } while ($hasMorePages); - }; + yield new ReleasePage($releases, !$hasMorePages, $skipped); - return Paginator::createFromGenerator($pageLoader(), null); + $currentPage++; + } while ($hasMorePages); } /** @@ -196,13 +206,12 @@ private function decodeReleasesResponse(ResponseInterface $response): array */ private function releasesRequest(int $page): ResponseInterface { - return $this->request( - Method::Get, - $this->httpFactory->uri( - \sprintf(self::URL_RELEASES, $this->repositoryPath), - ['page' => $page], - ), + $uri = $this->httpFactory->uri( + \sprintf(self::URL_RELEASES, $this->repositoryPath), + ['page' => $page, 'per_page' => self::RELEASES_PER_PAGE], ); + + return $this->request(Method::Get, $uri); } private function hasNextPage(ResponseInterface $response): bool diff --git a/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php b/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php index 43225da..adba8ae 100644 --- a/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php +++ b/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php @@ -4,6 +4,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; + /** * GitHub Asset Data Transfer Object. * @@ -17,12 +19,14 @@ final class AssetInfo * @param non-empty-string $downloadUrl * @param int<0, max> $size * @param non-empty-string $contentType + * @param non-empty-string|null $digest Checksum as `sha256:`; GitHub reports it for assets uploaded since 2025. */ public function __construct( public readonly string $name, public readonly string $downloadUrl, public readonly int $size, public readonly string $contentType, + public readonly ?string $digest = null, ) {} /** @@ -30,16 +34,34 @@ public function __construct( * name: string, * browser_download_url: string, * size: int, - * content_type: string + * content_type: string, + * digest?: string|null * } $data */ public static function fromApiResponse(array $data): self { + $digest = $data['digest'] ?? null; + return new self( name: $data['name'], downloadUrl: $data['browser_download_url'], size: $data['size'], contentType: $data['content_type'], + digest: $digest === '' ? null : $digest, + ); + } + + /** + * Maps the asset into the provider-neutral registry record. + */ + public function toRecord(): AssetRecord + { + return new AssetRecord( + name: $this->name, + uri: $this->downloadUrl, + size: $this->size, + contentType: $this->contentType, + digest: $this->digest, ); } } diff --git a/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php b/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php index 6478ea4..0462043 100644 --- a/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php +++ b/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php @@ -4,6 +4,9 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; + /** * GitHub Release Data Transfer Object. * @@ -20,7 +23,7 @@ final class ReleaseInfo public function __construct( public readonly string $name, public readonly string $tagName, - public readonly \DateTimeImmutable $publishedAt, + public readonly ?\DateTimeImmutable $publishedAt, public readonly array $assets, public readonly bool $prerelease, public readonly bool $draft, @@ -30,12 +33,13 @@ public function __construct( * @param array{ * name: string|null, * tag_name: string, - * published_at: string, + * published_at: string|null, * assets: array, * prerelease: bool, * draft: bool @@ -51,10 +55,32 @@ public static function fromApiResponse(array $data): self return new self( name: $data['name'] ?? $data['tag_name'], tagName: $data['tag_name'], - publishedAt: new \DateTimeImmutable($data['published_at']), + // A draft has not been published yet + publishedAt: $data['published_at'] === null ? null : new \DateTimeImmutable($data['published_at']), assets: $assets, prerelease: $data['prerelease'], draft: $data['draft'], ); } + + /** + * Maps the release into the provider-neutral registry record. + * + * A draft is visible to the token holder only, and the registry may be shared, so it becomes a + * hidden placeholder that carries nothing but the tag. + */ + public function toRecord(): ReleaseRecord + { + if ($this->draft) { + return new ReleaseRecord(tag: $this->tagName, name: $this->tagName, hidden: true); + } + + return new ReleaseRecord( + tag: $this->tagName, + name: $this->name, + publishedAt: $this->publishedAt, + prerelease: $this->prerelease, + assets: \array_map(static fn(AssetInfo $asset): AssetRecord => $asset->toRecord(), $this->assets), + ); + } } diff --git a/src/Module/Repository/Internal/GitHub/Api/ResponseValidator.php b/src/Module/Repository/Internal/GitHub/Api/ResponseValidator.php index 01ca4fb..3ebd3b1 100644 --- a/src/Module/Repository/Internal/GitHub/Api/ResponseValidator.php +++ b/src/Module/Repository/Internal/GitHub/Api/ResponseValidator.php @@ -26,6 +26,12 @@ protected function tokenEnvVariable(): string return 'GITHUB_TOKEN'; } + protected function isAssetUri(string $uri): bool + { + // Assets are served from github.com (and its CDN), the API lives on api.github.com + return \str_contains($uri, '/releases/download/'); + } + protected function repositoryFromUri(string $uri): ?string { // API calls: https://api.github.com/repos/owner/repo/releases diff --git a/src/Module/Repository/Internal/GitHub/Factory.php b/src/Module/Repository/Internal/GitHub/Factory.php index 77fd1c4..6773153 100644 --- a/src/Module/Repository/Internal/GitHub/Factory.php +++ b/src/Module/Repository/Internal/GitHub/Factory.php @@ -7,6 +7,7 @@ use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitHub; use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Client; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; use Internal\DLoad\Module\Repository\RepositoryFactory; @@ -30,6 +31,7 @@ public function __construct( private readonly HttpFactory $httpFactory, GitHub $gitHubConfig, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->gitHubClient = new Client( $httpFactory, @@ -50,7 +52,7 @@ public function create(RepositoryConfig $config): GitHubRepository $api = $this->createRepositoryApi($org, $repo); - return new GitHubRepository($api, $org, $repo, $this->logger); + return new GitHubRepository($api, $org, $repo, $this->logger, $this->registry); } /** diff --git a/src/Module/Repository/Internal/GitHub/GitHubAsset.php b/src/Module/Repository/Internal/GitHub/GitHubAsset.php index 020916c..41548f4 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubAsset.php +++ b/src/Module/Repository/Internal/GitHub/GitHubAsset.php @@ -9,8 +9,8 @@ use Internal\DLoad\Module\Common\OperatingSystem; use Internal\DLoad\Module\HttpClient\Method; use Internal\DLoad\Module\HttpClient\StreamReader; +use Internal\DLoad\Module\Registry\Record\AssetRecord; use Internal\DLoad\Module\Repository\Internal\Asset; -use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\AssetInfo; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Exception\RepositoryException; @@ -41,12 +41,12 @@ private function __construct( ); } - public static function fromDTO( + public static function fromRecord( RepositoryApi $api, GitHubRelease $release, - AssetInfo $dto, + AssetRecord $record, ): self { - return new self($api, $release, $dto->name, $dto->downloadUrl); + return new self($api, $release, $record->name, $record->uri); } /** diff --git a/src/Module/Repository/Internal/GitHub/GitHubRelease.php b/src/Module/Repository/Internal/GitHub/GitHubRelease.php index 8f0aef9..adffc49 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubRelease.php +++ b/src/Module/Repository/Internal/GitHub/GitHubRelease.php @@ -5,8 +5,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; use Internal\DLoad\Module\Repository\Collection\AssetsCollection; -use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Internal\Release; use Internal\DLoad\Module\Version\Version; @@ -30,17 +30,20 @@ private function __construct( parent::__construct($repository, $name, $version); } - public static function fromDTO( + /** + * @throws \InvalidArgumentException When the release tag is not a version. + */ + public static function fromRecord( RepositoryApi $api, GitHubRepository $repository, - ReleaseInfo $dto, + ReleaseRecord $record, ): self { - $version = Version::fromVersionString($dto->tagName); - $result = new self($repository, $dto->name, $version); + $version = Version::fromVersionString($record->tag); + $result = new self($repository, $record->name, $version); - $result->assets = AssetsCollection::create(static function () use ($api, $result, $dto): \Generator { - foreach ($dto->assets as $assetDTO) { - yield GitHubAsset::fromDTO($api, $result, $assetDTO); + $result->assets = AssetsCollection::create(static function () use ($api, $result, $record): \Generator { + foreach ($record->assets as $asset) { + yield GitHubAsset::fromRecord($api, $result, $asset); } }); diff --git a/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php b/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php new file mode 100644 index 0000000..6d334c3 --- /dev/null +++ b/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php @@ -0,0 +1,36 @@ + $first */ + $first = \intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1; + + foreach ($this->api->releasePages($first) as $page) { + yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last, $page->skipped); + $skip = 0; + } + } +} diff --git a/src/Module/Repository/Internal/GitHub/GitHubRepository.php b/src/Module/Repository/Internal/GitHub/GitHubRepository.php index 2c4d9be..a0c80e0 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubRepository.php +++ b/src/Module/Repository/Internal/GitHub/GitHubRepository.php @@ -5,9 +5,12 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\RepositoryId; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Collection\ReleasesCollection; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; +use Internal\DLoad\Module\Repository\Internal\Paginator; use Internal\DLoad\Module\Repository\Repository; use Internal\DLoad\Service\Logger; @@ -19,6 +22,9 @@ */ final class GitHubRepository implements Repository, Destroyable { + /** Repository type identifier in the version registry. */ + public const TYPE = 'github'; + private ?ReleasesCollection $releases = null; /** @@ -37,13 +43,17 @@ public function __construct( string $org, string $repo, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->name = $org . '/' . $repo; } /** * Returns a lazily loaded collection of repository releases. - * Pages are loaded only when needed during iteration or filtering. + * + * Releases come from the version registry, which serves stored ones without a request and + * asks the API only for what it does not know yet. Pages are loaded only when needed during + * iteration or filtering. */ public function getReleases(): ReleasesCollection { @@ -53,32 +63,37 @@ public function getReleases(): ReleasesCollection // Create a generator function for lazy loading release pages $pageLoader = function (): \Generator { - $page = 0; + // to avoid first eager loading because of generator + yield []; + + $pages = $this->registry->releases( + new RepositoryId(self::TYPE, $this->name), + new GitHubReleaseSource($this->api), + ); $anyPageLoaded = false; - do { + while (true) { try { - // to avoid first eager loading because of generator - yield []; + // Advancing the generator is what requests the next page + $anyPageLoaded ? $pages->next() : $pages->rewind(); - $paginator = $this->api->getReleases(++$page); - $releases = $paginator->getPageItems(); + if (!$pages->valid()) { + return; + } $toYield = []; - foreach ($releases as $releaseDTO) { + foreach ($pages->current() ?? [] as $record) { try { - $toYield[] = GitHubRelease::fromDTO($this->api, $this, $releaseDTO); + $toYield[] = GitHubRelease::fromRecord($this->api, $this, $record); } catch (\Throwable $e) { $this->logger->exception($e, important: false); // Skip invalid releases continue; } } - yield $toYield; - $anyPageLoaded = true; - // Check if there are more pages by getting next page - $hasMorePages = $paginator->getNextPage() !== null; + $anyPageLoaded = true; + yield $toYield; } catch (\Throwable $e) { # The first page is mandatory: when it fails, there is nothing to download and the reason # (invalid token, rate limit, missing repository, etc.) must reach the user. @@ -93,11 +108,11 @@ public function getReleases(): ReleasesCollection $this->logger->exception($e, important: false); return; } - } while ($hasMorePages); + } }; // Create paginator - $paginator = \Internal\DLoad\Module\Repository\Internal\Paginator::createFromGenerator($pageLoader(), null); + $paginator = Paginator::createFromGenerator($pageLoader(), null); // Create a collection with the paginator $this->releases = ReleasesCollection::create($paginator); @@ -112,9 +127,10 @@ public function getName(): string public function destroy(): void { - $this->releases === null or $this->releases->map( - static fn(object $release) => $release instanceof Destroyable and $release->destroy(), - ); + // Only what was loaded is released: iterating the collection would request the remaining pages + foreach ($this->releases?->loaded() ?? [] as $release) { + $release instanceof Destroyable and $release->destroy(); + } unset($this->releases); } diff --git a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php index 6af6c71..5dec664 100644 --- a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php @@ -6,11 +6,11 @@ use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; use Internal\DLoad\Module\HttpClient\Method; +use Internal\DLoad\Module\Registry\Record\ReleasePage; use Internal\DLoad\Module\Repository\Exception\ApiException; use Internal\DLoad\Module\Repository\Exception\RepositoryException; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\RepositoryInfo; -use Internal\DLoad\Module\Repository\Internal\Paginator; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\UriInterface; @@ -28,6 +28,12 @@ final class RepositoryApi private const URL_RELEASES = 'https://gitlab.com/api/v4/projects/%s/releases'; private const URL_RELEASE_ASSET = 'https://gitlab.com/api/v4/projects/%s/releases/%s/downloads/%s'; + /** + * Number of releases to ask for in a single page. GitLab serves 20 by default and allows up to + * 100, so the maximum keeps the release list within as few requests as the API permits. + */ + public const RELEASES_PER_PAGE = 100; + /** * @var non-empty-string */ @@ -88,76 +94,79 @@ public function getRepository(): RepositoryInfo } /** + * Lists releases newest first, page by page, starting from the given page. + * + * A page is requested only when the generator advances to it, so a consumer that stops early + * costs no extra request. + * * @param int<1, max> $page - * @return Paginator + * @return \Generator * @throws RepositoryException */ - public function getReleases(int $page = 1): Paginator + public function releasePages(int $page = 1): \Generator { - $pageLoader = function () use ($page): \Generator { - $currentPage = $page; - - do { - $response = $this->releasesRequest($currentPage); - - /** @var list - * }, - * upcoming_release: bool - * }> $data */ - $data = $this->decodeReleasesResponse($response); - - // If empty response, no more pages - if ($data === []) { - return; + $currentPage = $page; + + do { + $response = $this->releasesRequest($currentPage); + + /** @var list + * }, + * upcoming_release: bool + * }> $data */ + $data = $this->decodeReleasesResponse($response); + + // If empty response, no more pages + if ($data === []) { + return; + } + + $releases = []; + $failure = null; + $skipped = 0; + foreach ($data as $releaseData) { + try { + $releases[] = ReleaseInfo::fromApiResponse($releaseData)->toRecord(); + } catch (\Throwable $e) { + $failure ??= $e; + ++$skipped; + // Skip invalid releases + continue; } - - $releases = []; - $failure = null; - foreach ($data as $releaseData) { - try { - $releases[] = ReleaseInfo::fromApiResponse($releaseData); - } catch (\Throwable $e) { - $failure ??= $e; - // Skip invalid releases - continue; - } - } - - // The whole page is unreadable: the response structure is not what we expect - if ($releases === [] && $failure !== null) { - throw new ApiException( - \sprintf( - 'GitLab API returned %d release(s) for project `%s`, but none of them could be read: %s', - \count($data), - $this->repositoryPath, - $failure->getMessage(), - ), + } + + // The whole page is unreadable: the response structure is not what we expect + if ($releases === [] && $failure !== null) { + throw new ApiException( + \sprintf( + 'GitLab API returned %d release(s) for project `%s`, but none of them could be read: %s', + \count($data), $this->repositoryPath, - $failure, - ); - } + $failure->getMessage(), + ), + $this->repositoryPath, + $failure, + ); + } - yield $releases; + $hasMorePages = $this->hasNextPage($response); - // Check if there are more pages - $hasMorePages = $this->hasNextPage($response); - $currentPage++; - } while ($hasMorePages); - }; + yield new ReleasePage($releases, !$hasMorePages, $skipped); - return Paginator::createFromGenerator($pageLoader(), null); + $currentPage++; + } while ($hasMorePages); } /** @@ -207,13 +216,12 @@ private function decodeReleasesResponse(ResponseInterface $response): array */ private function releasesRequest(int $page): ResponseInterface { - return $this->request( - Method::Get, - $this->httpFactory->uri( - \sprintf(self::URL_RELEASES, \urlencode($this->repositoryPath)), - ['page' => $page], - ), + $uri = $this->httpFactory->uri( + \sprintf(self::URL_RELEASES, \urlencode($this->repositoryPath)), + ['page' => $page, 'per_page' => self::RELEASES_PER_PAGE], ); + + return $this->request(Method::Get, $uri); } private function hasNextPage(ResponseInterface $response): bool diff --git a/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php b/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php index 46512ea..0c1a025 100644 --- a/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php +++ b/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php @@ -4,6 +4,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; + /** * GitLab Asset Data Transfer Object. * @@ -39,4 +41,12 @@ public static function fromApiResponse(array $data): self linkType: $data['link_type'] ?? null, ); } + + /** + * Maps the asset into the provider-neutral registry record. + */ + public function toRecord(): AssetRecord + { + return new AssetRecord(name: $this->name, uri: $this->downloadUrl); + } } diff --git a/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php b/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php index 07d6ee2..236a7da 100644 --- a/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php +++ b/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php @@ -4,6 +4,9 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; + /** * GitLab Release Data Transfer Object. * @@ -58,4 +61,18 @@ public static function fromApiResponse(array $data): self prerelease: $data['upcoming_release'], ); } + + /** + * Maps the release into the provider-neutral registry record. + */ + public function toRecord(): ReleaseRecord + { + return new ReleaseRecord( + tag: $this->tagName, + name: $this->name, + publishedAt: $this->publishedAt, + prerelease: $this->prerelease, + assets: \array_map(static fn(AssetInfo $asset): AssetRecord => $asset->toRecord(), $this->assets), + ); + } } diff --git a/src/Module/Repository/Internal/GitLab/Api/ResponseValidator.php b/src/Module/Repository/Internal/GitLab/Api/ResponseValidator.php index 6e135ce..712da5c 100644 --- a/src/Module/Repository/Internal/GitLab/Api/ResponseValidator.php +++ b/src/Module/Repository/Internal/GitLab/Api/ResponseValidator.php @@ -31,6 +31,12 @@ protected function repositoryTerm(): string return 'project'; } + protected function isAssetUri(string $uri): bool + { + // https://gitlab.com/api/v4/projects/group%2Fproject/releases/v1.0.0/downloads/asset.zip + return \preg_match('~/releases/[^/?#]+/downloads/~', $uri) === 1; + } + protected function repositoryFromUri(string $uri): ?string { // https://gitlab.com/api/v4/projects/group%2Fproject/releases diff --git a/src/Module/Repository/Internal/GitLab/Factory.php b/src/Module/Repository/Internal/GitLab/Factory.php index 68d601d..2ccf644 100644 --- a/src/Module/Repository/Internal/GitLab/Factory.php +++ b/src/Module/Repository/Internal/GitLab/Factory.php @@ -7,6 +7,7 @@ use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitLab; use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Client; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; use Internal\DLoad\Module\Repository\RepositoryFactory; @@ -30,6 +31,7 @@ public function __construct( private readonly HttpFactory $httpFactory, GitLab $gitLabConfig, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->gitLabClient = new Client( $httpFactory, @@ -49,7 +51,7 @@ public function create(RepositoryConfig $config): GitLabRepository $uri = \is_string($path) && $path !== '' ? $path : $config->uri; $api = $this->createRepositoryApi($uri); - return new GitLabRepository($api, $uri, $this->logger); + return new GitLabRepository($api, $uri, $this->logger, $this->registry); } /** diff --git a/src/Module/Repository/Internal/GitLab/GitLabAsset.php b/src/Module/Repository/Internal/GitLab/GitLabAsset.php index 5b0f7a7..927c1b6 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabAsset.php +++ b/src/Module/Repository/Internal/GitLab/GitLabAsset.php @@ -8,8 +8,8 @@ use Internal\DLoad\Module\Common\Architecture; use Internal\DLoad\Module\Common\OperatingSystem; use Internal\DLoad\Module\HttpClient\StreamReader; +use Internal\DLoad\Module\Registry\Record\AssetRecord; use Internal\DLoad\Module\Repository\Internal\Asset; -use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\AssetInfo; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Exception\RepositoryException; @@ -40,12 +40,12 @@ private function __construct( ); } - public static function fromDTO( + public static function fromRecord( RepositoryApi $api, GitLabRelease $release, - AssetInfo $dto, + AssetRecord $record, ): self { - return new self($api, $release, $dto->name, $dto->downloadUrl); + return new self($api, $release, $record->name, $record->uri); } /** diff --git a/src/Module/Repository/Internal/GitLab/GitLabRelease.php b/src/Module/Repository/Internal/GitLab/GitLabRelease.php index f4bc854..8760d2c 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabRelease.php +++ b/src/Module/Repository/Internal/GitLab/GitLabRelease.php @@ -5,8 +5,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; use Internal\DLoad\Module\Repository\Collection\AssetsCollection; -use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Internal\Release; use Internal\DLoad\Module\Version\Version; @@ -30,17 +30,20 @@ private function __construct( parent::__construct($repository, $name, $version); } - public static function fromDTO( + /** + * @throws \InvalidArgumentException When the release tag is not a version. + */ + public static function fromRecord( RepositoryApi $api, GitLabRepository $repository, - ReleaseInfo $dto, + ReleaseRecord $record, ): self { - $version = Version::fromVersionString($dto->tagName); - $result = new self($repository, $dto->name, $version); + $version = Version::fromVersionString($record->tag); + $result = new self($repository, $record->name, $version); - $result->assets = AssetsCollection::create(static function () use ($api, $result, $dto): \Generator { - foreach ($dto->assets as $assetDTO) { - yield GitLabAsset::fromDTO($api, $result, $assetDTO); + $result->assets = AssetsCollection::create(static function () use ($api, $result, $record): \Generator { + foreach ($record->assets as $asset) { + yield GitLabAsset::fromRecord($api, $result, $asset); } }); diff --git a/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php b/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php new file mode 100644 index 0000000..cb0e22d --- /dev/null +++ b/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php @@ -0,0 +1,36 @@ + $first */ + $first = \intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1; + + foreach ($this->api->releasePages($first) as $page) { + yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last, $page->skipped); + $skip = 0; + } + } +} diff --git a/src/Module/Repository/Internal/GitLab/GitLabRepository.php b/src/Module/Repository/Internal/GitLab/GitLabRepository.php index 94f3dff..6410e4a 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabRepository.php +++ b/src/Module/Repository/Internal/GitLab/GitLabRepository.php @@ -5,9 +5,12 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\RepositoryId; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Collection\ReleasesCollection; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; +use Internal\DLoad\Module\Repository\Internal\Paginator; use Internal\DLoad\Module\Repository\Repository; use Internal\DLoad\Service\Logger; @@ -19,6 +22,9 @@ */ final class GitLabRepository implements Repository, Destroyable { + /** Repository type identifier in the version registry. */ + public const TYPE = 'gitlab'; + private ?ReleasesCollection $releases = null; /** @@ -35,13 +41,17 @@ public function __construct( private readonly RepositoryApi $api, string $projectPath, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->name = $projectPath; } /** * Returns a lazily loaded collection of repository releases. - * Pages are loaded only when needed during iteration or filtering. + * + * Releases come from the version registry, which serves stored ones without a request and + * asks the API only for what it does not know yet. Pages are loaded only when needed during + * iteration or filtering. */ public function getReleases(): ReleasesCollection { @@ -51,38 +61,43 @@ public function getReleases(): ReleasesCollection // Create a generator function for lazy loading release pages $pageLoader = function (): \Generator { - $page = 0; + // to avoid first eager loading because of generator + yield []; + + $pages = $this->registry->releases( + new RepositoryId(self::TYPE, $this->name), + new GitLabReleaseSource($this->api), + ); $anyPageLoaded = false; - do { + while (true) { try { - // to avoid first eager loading because of generator - yield []; + // Advancing the generator is what requests the next page + $anyPageLoaded ? $pages->next() : $pages->rewind(); - $paginator = $this->api->getReleases(++$page); - $releases = $paginator->getPageItems(); + if (!$pages->valid()) { + return; + } $toYield = []; - foreach ($releases as $releaseDTO) { + foreach ($pages->current() ?? [] as $record) { try { - $toYield[] = GitLabRelease::fromDTO($this->api, $this, $releaseDTO); + $toYield[] = GitLabRelease::fromRecord($this->api, $this, $record); } catch (\Throwable) { // Skip invalid releases continue; } } - yield $toYield; - $anyPageLoaded = true; - // Check if there are more pages by getting next page - $hasMorePages = $paginator->getNextPage() !== null; + $anyPageLoaded = true; + yield $toYield; } catch (\Throwable $e) { # The first page is mandatory: when it fails, there is nothing to download and the reason # (invalid token, rate limit, missing project, etc.) must reach the user. $anyPageLoaded or throw $e; # A rate limit leaves the release list incomplete: hiding it would produce a report - # that claims the project has nothing more, so it must reach the user as well. + # that claims the repository has nothing more, so it must reach the user as well. $e instanceof RateLimitException and throw $e; # Already loaded releases are enough to continue, so a failure of a subsequent page @@ -90,11 +105,11 @@ public function getReleases(): ReleasesCollection $this->logger->exception($e, important: false); return; } - } while ($hasMorePages); + } }; // Create paginator - $paginator = \Internal\DLoad\Module\Repository\Internal\Paginator::createFromGenerator($pageLoader(), null); + $paginator = Paginator::createFromGenerator($pageLoader(), null); // Create a collection with the paginator $this->releases = ReleasesCollection::create($paginator); @@ -109,9 +124,10 @@ public function getName(): string public function destroy(): void { - $this->releases === null or $this->releases->map( - static fn(object $release) => $release instanceof Destroyable and $release->destroy(), - ); + // Only what was loaded is released: iterating the collection would request the remaining pages + foreach ($this->releases?->loaded() ?? [] as $release) { + $release instanceof Destroyable and $release->destroy(); + } unset($this->releases); } diff --git a/src/Module/Repository/Internal/ResponseValidator.php b/src/Module/Repository/Internal/ResponseValidator.php index 608a089..6b62b96 100644 --- a/src/Module/Repository/Internal/ResponseValidator.php +++ b/src/Module/Repository/Internal/ResponseValidator.php @@ -6,6 +6,7 @@ use Internal\DLoad\Module\Repository\Exception\AccessDeniedException; use Internal\DLoad\Module\Repository\Exception\ApiException; +use Internal\DLoad\Module\Repository\Exception\AssetNotFoundException; use Internal\DLoad\Module\Repository\Exception\AuthenticationException; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryException; @@ -63,6 +64,16 @@ public function validate(RequestInterface $request, ResponseInterface $response) $this->accessDeniedMessage($apiMessage, $repository), $repository, ), + // A missing asset is not a missing repository: the listing was fine, the file is gone + $status === 404 && $this->isAssetUri((string) $request->getUri()) => new AssetNotFoundException( + \sprintf( + '%s asset is no longer available: HTTP 404 for %s. ' + . 'The release may have been deleted or its assets replaced since the release list was fetched.', + $this->providerName(), + (string) $request->getUri(), + ), + $repository, + ), $status === 404 => new RepositoryNotFoundException( $this->notFoundMessage($apiMessage, $repository, $endpoint), $repository, @@ -135,6 +146,13 @@ protected function repositoryTerm(): string */ abstract protected function repositoryFromUri(string $uri): ?string; + /** + * Whether the URI points to a release asset rather than to the API. + * + * A 404 for an asset means the release is gone, not that the repository does not exist. + */ + abstract protected function isAssetUri(string $uri): bool; + /** * @return positive-int|null Requests per hour allowed without a token. */ diff --git a/tests/Acceptance/DLoadTest.php b/tests/Acceptance/DLoadTest.php index 4c7c19e..5b21c00 100644 --- a/tests/Acceptance/DLoadTest.php +++ b/tests/Acceptance/DLoadTest.php @@ -231,8 +231,12 @@ protected function cleanup(): void */ private function buildDLoad(string $xmlConfig): DLoad { + // The version registry must not leak into the user's cache directory from a test run + $environment = \getenv(); + $environment['DLOAD_CACHE_DIR'] = (string) $this->testRuntimeDir->join('registry'); + $container = Bootstrap::init() - ->withConfig($xmlConfig, [], [], \getenv()) + ->withConfig($xmlConfig, [], [], $environment) ->finish(); $container->set($input = new ArgvInput(), InputInterface::class); $container->set($output = new BufferedOutput(), OutputInterface::class); diff --git a/tests/Integration/Command/CacheClearTest.php b/tests/Integration/Command/CacheClearTest.php new file mode 100644 index 0000000..592bbef --- /dev/null +++ b/tests/Integration/Command/CacheClearTest.php @@ -0,0 +1,151 @@ +seed('a/b', ['rr']); + $this->seed('c/d', ['temporal', 'tctl']); + $this->seed('e/f', ['dolt']); + + $tester = $this->run(['software' => ['rr', 'tctl']]); + + Assert::string($tester->getDisplay())->contains('2 repository listing(s) removed.'); + Assert::same(self::uris($this->storage), ['e/f']); + } + + #[Test] + public function unknownSoftwareRemovesNothing(): void + { + $this->seed('a/b', ['rr']); + + $tester = $this->run(['software' => ['unknown']]); + + Assert::string($tester->getDisplay())->contains('0 repository listing(s) removed.'); + Assert::same(self::uris($this->storage), ['a/b']); + } + + #[Test] + public function clearsEverythingWithoutAskingWhenNotInteractive(): void + { + $this->seed('a/b', ['rr']); + + $tester = $this->run([], interactive: false); + + Assert::string($tester->getDisplay())->contains('has been cleared'); + Assert::same(self::uris($this->storage), []); + } + + #[Test] + public function clearingEverythingInteractivelyRequiresConfirmation(): void + { + $this->seed('a/b', ['rr']); + + $declined = $this->run([], interactive: true, answers: ['n']); + Assert::string($declined->getDisplay())->contains('left as it is'); + Assert::same(self::uris($this->storage), ['a/b']); + + $confirmed = $this->run([], interactive: true, answers: ['y']); + Assert::string($confirmed->getDisplay())->contains('has been cleared'); + Assert::same(self::uris($this->storage), []); + } + + #[Test] + public function forceSkipsTheConfirmation(): void + { + $this->seed('a/b', ['rr']); + + $tester = $this->run(['--force' => true], interactive: true); + + Assert::string($tester->getDisplay())->contains('has been cleared'); + Assert::same(self::uris($this->storage), []); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->directory = \sys_get_temp_dir() . '/dload-cache-clear-' . \bin2hex(\random_bytes(6)); + $this->storage = new FileRegistryStorage(Path::create($this->directory), new Logger()); + \mkdir($this->directory, recursive: true); + \file_put_contents($this->directory . '/dload.xml', ''); + \putenv('DLOAD_CACHE_DIR=' . $this->directory); + } + + #[AfterTest] + protected function cleanup(): void + { + \putenv('DLOAD_CACHE_DIR'); + \is_dir($this->directory) and FS::removeDir(Path::create($this->directory)); + } + + /** + * @return list + */ + private static function uris(FileRegistryStorage $storage): array + { + $uris = \array_map( + static fn(RepositoryRecord $record): string => $record->id->uri, + \iterator_to_array($storage->all(), false), + ); + \sort($uris); + + return $uris; + } + + /** + * @param non-empty-string $uri + * @param list $software + */ + private function seed(string $uri, array $software): void + { + $this->storage->save(new RepositoryRecord(new RepositoryId('github', $uri), software: $software)); + } + + /** + * @param array $input + * @param list $answers + */ + private function run(array $input, bool $interactive = false, array $answers = []): CommandTester + { + $application = new Application(); + // Symfony Console 8 renamed `add()` to `addCommand()` + \method_exists($application, 'addCommand') + ? $application->addCommand(new CacheClear()) + : $application->add(new CacheClear()); + + $tester = new CommandTester($application->find('cache:clear')); + $answers === [] or $tester->setInputs($answers); + $tester->execute( + $input + ['--config' => $this->directory . '/dload.xml'], + ['interactive' => $interactive], + ); + + return $tester; + } +} diff --git a/tests/Integration/Module/Registry/VersionRegistryBindingTest.php b/tests/Integration/Module/Registry/VersionRegistryBindingTest.php new file mode 100644 index 0000000..86ef447 --- /dev/null +++ b/tests/Integration/Module/Registry/VersionRegistryBindingTest.php @@ -0,0 +1,151 @@ + $this->directory]); + + Assert::instanceOf($container->get(VersionRegistry::class), StoredVersionRegistry::class); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/dload/repositories/github/a/b/index.json')); + } + + #[Test] + public function environmentVariableSetsTheDirectory(): void + { + $container = self::bootstrap(environment: ['DLOAD_CACHE_DIR' => $this->directory]); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/repositories/github/a/b/index.json')); + } + + #[Test] + public function xmlAttributeSetsTheDirectory(): void + { + $container = self::bootstrap(xml: \sprintf('', $this->directory)); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/repositories/github/a/b/index.json')); + } + + #[Test] + public function environmentOverridesTheXmlAttribute(): void + { + $container = self::bootstrap( + xml: '', + environment: ['DLOAD_CACHE_DIR' => $this->directory, 'DLOAD_CACHE_TTL' => '0'], + ); + + Assert::instanceOf($container->get(VersionRegistry::class), PassThroughRegistry::class); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/repositories/github/a/b/index.json')); + } + + #[Test] + public function zeroTtlDisablesTheRegistry(): void + { + $container = self::bootstrap(environment: ['DLOAD_CACHE_DIR' => $this->directory, 'DLOAD_CACHE_TTL' => '0']); + + Assert::instanceOf($container->get(VersionRegistry::class), PassThroughRegistry::class); + } + + #[Test] + public function repositoryProviderIsBoundWithGithubAndGitLabFactories(): void + { + $provider = self::bootstrap()->get(RepositoryProvider::class); + + Assert::instanceOf($provider, RepositoryProvider::class); + Assert::instanceOf( + $provider->getByConfig(RepositoryConfig::fromArray(['type' => 'github', 'uri' => 'a/b'])), + GitHubRepository::class, + ); + Assert::instanceOf( + $provider->getByConfig(RepositoryConfig::fromArray(['type' => 'gitlab', 'uri' => 'a/b'])), + GitLabRepository::class, + ); + } + + #[Test] + public function xmlConfigIsReadFromAFilePath(): void + { + $configFile = \sys_get_temp_dir() . '/dload-config-' . \bin2hex(\random_bytes(6)) . '.xml'; + \file_put_contents($configFile, \sprintf('', $this->directory)); + + try { + $container = self::bootstrap(xml: $configFile); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/repositories/github/a/b/index.json')); + } finally { + \unlink($configFile); + } + } + + #[Test] + public function missingConfigFilePathThrows(): void + { + Expect::exception(\InvalidArgumentException::class)->withMessage('Config file not found.'); + + self::bootstrap(xml: \sys_get_temp_dir() . '/dload-missing-' . \bin2hex(\random_bytes(6)) . '.xml'); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->directory = \sys_get_temp_dir() . '/dload-registry-binding-' . \bin2hex(\random_bytes(6)); + } + + #[AfterTest] + protected function cleanup(): void + { + \is_dir($this->directory) and FS::removeDir(Path::create($this->directory)); + } + + /** + * @param array $environment + */ + private static function bootstrap(?string $xml = null, array $environment = []): Container + { + return Bootstrap::init() + ->withConfig(xml: $xml, environment: $environment) + ->finish(); + } +} diff --git a/tests/Unit/Module/Downloader/DownloaderTest.php b/tests/Unit/Module/Downloader/DownloaderTest.php new file mode 100644 index 0000000..266f92c --- /dev/null +++ b/tests/Unit/Module/Downloader/DownloaderTest.php @@ -0,0 +1,496 @@ + [OperatingSystem::Linux, Architecture::X86_64]; + yield 'OS only' => [OperatingSystem::Linux, null]; + yield 'architecture only' => [null, Architecture::X86_64]; + } + + #[Test] + public function deletedReleaseIsForgottenAndTheNextOneIsUsed(): void + { + $repository = new RepositoryStub('owner/repo'); + $gone = self::release($repository, 'v2.0.0', assets: false); + $alive = self::release($repository, 'v1.9.0', assets: true); + $repository = new RepositoryStub('owner/repo', ReleasesCollection::create([$gone, $alive])); + + $result = $this->download([$repository]); + + Assert::same($result->version->string, 'v1.9.0'); + Assert::same($this->registry->forgotten, [['github:owner/repo', 'v2.0.0']]); + Assert::same($this->registry->attached, [['rr', 'github:owner/repo']]); + } + + #[Test] + public function registryIdComesFromTheRepositoryNotTheConfiguredUri(): void + { + // The factory reduces a full URL to the path; the registry must key the record the same way + $repository = new RepositoryStub('owner/repo'); + $gone = self::release($repository, 'v2.0.0', assets: false); + $alive = self::release($repository, 'v1.9.0', assets: true); + $repository = new RepositoryStub('owner/repo', ReleasesCollection::create([$gone, $alive])); + + $this->download([$repository], uri: 'https://github.com/Owner/Repo'); + + Assert::same($this->registry->attached, [['rr', 'github:owner/repo']]); + Assert::same($this->registry->forgotten, [['github:owner/repo', 'v2.0.0']]); + } + + #[Test] + public function outdatedListIsFetchedAgainWhenNothingIsLeft(): void + { + // The stored list knows only the deleted release; a fresh list has its replacement + $stale = new RepositoryStub('owner/repo'); + $stale = new RepositoryStub('owner/repo', ReleasesCollection::create([ + self::release($stale, 'v2.0.0', assets: false), + ])); + $fresh = new RepositoryStub('owner/repo'); + $fresh = new RepositoryStub('owner/repo', ReleasesCollection::create([ + self::release($fresh, 'v2.0.1', assets: true), + ])); + $factory = new SequenceRepositoryFactoryStub([$stale, $fresh]); + + $result = $this->download($factory); + + Assert::same($result->version->string, 'v2.0.1'); + Assert::same($factory->created, 2); + Assert::same($this->registry->forgotten, [['github:owner/repo', 'v2.0.0']]); + } + + #[Test] + public function outdatedListIsFetchedAgainOnlyOnce(): void + { + $stale = new RepositoryStub('owner/repo'); + $stale = new RepositoryStub('owner/repo', ReleasesCollection::create([ + self::release($stale, 'v2.0.0', assets: false), + ])); + $factory = new SequenceRepositoryFactoryStub([$stale]); + + try { + $this->download($factory); + Assert::fail('DownloadFailed is expected when the fresh list has nothing suitable either.'); + } catch (DownloadFailed $e) { + Assert::same($factory->created, 2); + Assert::string($e->report)->contains('no longer available'); + } + } + + #[Test] + public function releaseWithOtherFailuresIsNotForgotten(): void + { + $repository = new RepositoryStub('owner/repo'); + $broken = new ReleaseStub($repository, 'v2.0.0', Version::fromVersionString('v2.0.0')); + $broken->setAssets([ + new GoneAssetStub($broken, 'rr-linux-amd64.tar.gz'), + // A working asset: the release is not gone, the first asset just was + new AssetStub($broken, 'rr-linux-amd64.zip', 'https://x/rr.zip'), + ]); + $repository = new RepositoryStub('owner/repo', ReleasesCollection::create([$broken])); + + $result = $this->download([$repository]); + + Assert::same($result->version->string, 'v2.0.0'); + Assert::same($this->registry->forgotten, []); + } + + #[Test] + public function releaseFailingForOtherReasonsIsNotForgotten(): void + { + $repository = new RepositoryStub('owner/repo'); + $broken = new ReleaseStub($repository, 'v2.0.0', Version::fromVersionString('v2.0.0')); + $broken->setAssets([ + new GoneAssetStub($broken, 'rr-linux-amd64.tar.gz'), + // Every asset fails, but a network error says nothing about the release being gone + new GoneAssetStub($broken, 'rr-linux-amd64.zip', new \RuntimeException('Connection reset by peer')), + ]); + $factory = new SequenceRepositoryFactoryStub([ + new RepositoryStub('owner/repo', ReleasesCollection::create([$broken])), + ]); + + try { + $this->download($factory); + Assert::fail('DownloadFailed is expected when no asset could be downloaded.'); + } catch (DownloadFailed) { + Assert::same($this->registry->forgotten, []); + Assert::same($factory->created, 1); + } + } + + #[DataProvider('provideGradualStrategies')] + #[Test] + public function gradualFilteringSelectsAssetByEachStrategy(?OperatingSystem $os, ?Architecture $arch): void + { + $repository = $this->repoWithAsset('rr-linux-amd64.tar.gz', $os, $arch); + + $result = $this->run([$repository], self::software()); + + Assert::same($result->version->string, 'v1.0.0'); + } + + #[Test] + public function gradualFilteringTriesEveryStrategyBeforeFailing(): void + { + // One asset matches OS+arch, OS-only and arch-only, so every strategy selects it; its + // generic download error is not "gone", so the search moves on to the next, wider strategy. + $repository = $this->repoWithAsset( + 'rr-linux-amd64.tar.gz', + OperatingSystem::Linux, + Architecture::X86_64, + new \RuntimeException('boom'), + ); + + try { + $this->run([$repository], self::software()); + Assert::fail('DownloadFailed is expected when no strategy can download the asset.'); + } catch (DownloadFailed) { + // A plain error must not forget the release from the registry + Assert::same($this->registry->forgotten, []); + } + } + + #[Test] + public function strictFilteringSelectsAssetWhenBinaryConfigured(): void + { + $repository = $this->repoWithAsset('rr-linux-amd64.tar.gz', OperatingSystem::Linux, Architecture::X86_64); + + $result = $this->run([$repository], self::software(binary: true)); + + Assert::same($result->version->string, 'v1.0.0'); + } + + #[Test] + public function strictFilteringFailsWhenNoAssetMatchesAllCriteria(): void + { + // Binary config forces strict filtering; a Windows/arm64 asset matches neither OS nor arch + $repository = $this->repoWithAsset('rr-windows-arm64.zip', OperatingSystem::Windows, Architecture::ARM_64); + + try { + $this->run([$repository], self::software(binary: true)); + Assert::fail('DownloadFailed is expected when strict filtering matches nothing.'); + } catch (DownloadFailed $e) { + Assert::string($e->report)->contains('no asset matches OS'); + } + } + + #[Test] + public function versionConstraintSelectsMatchingRelease(): void + { + $repository = new RepositoryStub('owner/repo'); + $v1 = $this->assetRelease($repository, 'v1.5.0'); + $v2 = $this->assetRelease($repository, 'v2.0.0'); + $repository = new RepositoryStub('owner/repo', ReleasesCollection::create([$v1, $v2])); + + $config = DownloadConfig::fromSoftwareId('rr'); + $config->version = '^1.0'; + + $result = $this->run([$repository], self::software(), $config); + + Assert::same($result->version->string, 'v1.5.0'); + } + + #[Test] + public function noRelevantReleaseReportsTheAvailableOnes(): void + { + // More releases than the report lists: fetchReleaseNames must stop at its limit + $repository = new RepositoryStub('owner/repo'); + $releases = []; + for ($i = 1; $i <= 12; ++$i) { + $releases[] = $this->assetRelease($repository, 'v1.0.' . $i); + } + $repository = new RepositoryStub('owner/repo', ReleasesCollection::create($releases)); + + $config = DownloadConfig::fromSoftwareId('rr'); + $config->version = '^9.9'; + + try { + $this->run([$repository], self::software(), $config); + Assert::fail('DownloadFailed is expected when no release satisfies the constraint.'); + } catch (DownloadFailed $e) { + Assert::string($e->report)->contains('Releases available in the repository'); + Assert::string($e->report)->contains('v1.0.1'); + } + } + + #[Test] + public function pharTypeWithoutPharAssetReportsThePharRequirement(): void + { + $repository = $this->repoWithAsset('rr-linux-amd64.tar.gz', OperatingSystem::Linux, Architecture::X86_64); + + $config = DownloadConfig::fromSoftwareId('rr'); + $config->type = Type::Phar; + + try { + $this->run([$repository], self::software(), $config); + Assert::fail('DownloadFailed is expected when no phar asset is present.'); + } catch (DownloadFailed $e) { + Assert::string($e->report)->contains('phar'); + } + } + + #[Test] + public function archiveTypeWithoutArchiveAssetReportsTheArchiveRequirement(): void + { + $repository = $this->repoWithAsset('rr-linux-amd64', OperatingSystem::Linux, Architecture::X86_64); + + $config = DownloadConfig::fromSoftwareId('rr'); + $config->type = Type::Archive; + + try { + $this->run([$repository], self::software(), $config); + Assert::fail('DownloadFailed is expected when no archive asset is present.'); + } catch (DownloadFailed $e) { + Assert::string($e->report)->contains('archive extensions'); + } + } + + #[Test] + public function rateLimitedRepositoryFallsBackToTheNextOne(): void + { + $limited = $this->repoWithAsset( + 'rr-linux-amd64.tar.gz', + OperatingSystem::Linux, + Architecture::X86_64, + new RateLimitException('API rate limit exceeded', 'owner/repo'), + ); + $working = $this->repoWithAsset( + 'rr-linux-amd64.tar.gz', + OperatingSystem::Linux, + Architecture::X86_64, + tag: 'v3.0.0', + ); + + $result = $this->run( + new SequenceRepositoryFactoryStub([$limited, $working]), + self::software(repositories: 2), + ); + + Assert::same($result->version->string, 'v3.0.0'); + } + + #[Test] + public function unexpectedErrorFromRepositoryIsRethrown(): void + { + $repository = new ThrowingRepositoryStub('owner/repo', new \RuntimeException('unexpected failure')); + + Expect::exception(\RuntimeException::class)->withMessage('unexpected failure'); + + $this->run([$repository], self::software()); + } + + #[Test] + public function tmpDirPointingAtAFileIsRejected(): void + { + \is_dir($this->tempDir) or \mkdir($this->tempDir, recursive: true); + $file = $this->tempDir . '/not-a-directory'; + \file_put_contents($file, 'x'); + + $config = new DownloaderConfig(); + $config->tmpDir = $file; + $downloader = $this->makeDownloader(new SequenceRepositoryFactoryStub([]), $config); + + Expect::exception(\LogicException::class); + + $downloader->download(self::software(), DownloadConfig::fromSoftwareId('rr'), static fn(): null => null); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->tempDir = \sys_get_temp_dir() . '/dload-downloader-' . \bin2hex(\random_bytes(6)); + $this->registry = new RecordingRegistry(); + } + + #[AfterTest] + protected function cleanup(): void + { + \is_dir($this->tempDir) and FS::removeDir(Path::create($this->tempDir)); + } + + /** + * @param non-empty-string $tag + * @param bool $assets `true` for a downloadable asset, `false` for one that is gone. + */ + private static function release(RepositoryStub $repository, string $tag, bool $assets): ReleaseStub + { + $release = new ReleaseStub($repository, $tag, Version::fromVersionString($tag)); + $release->setAssets([ + $assets + ? new AssetStub($release, 'rr-linux-amd64.tar.gz', 'https://x/' . $tag . '/rr.tar.gz') + : new GoneAssetStub($release, 'rr-linux-amd64.tar.gz'), + ]); + + return $release; + } + + /** + * @param positive-int $repositories Number of repository configs on the software definition. + */ + private static function software(bool $binary = false, int $repositories = 1): Software + { + $repos = []; + for ($i = 1; $i <= $repositories; ++$i) { + $repos[] = ['type' => 'github', 'uri' => $i === 1 ? 'owner/repo' : 'owner/repo' . $i]; + } + + $data = ['name' => 'rr', 'repositories' => $repos]; + $binary and $data['binary'] = ['name' => 'rr']; + + return Software::fromArray($data); + } + + /** + * @param list|SequenceRepositoryFactoryStub $repositories + * @param non-empty-string $uri Repository URI as written in the config. + */ + private function download(array|SequenceRepositoryFactoryStub $repositories, string $uri = 'owner/repo'): DownloadResult + { + $factory = $repositories instanceof SequenceRepositoryFactoryStub + ? $repositories + : new SequenceRepositoryFactoryStub($repositories); + + $config = new DownloaderConfig(); + $config->tmpDir = $this->tempDir; + + $downloader = new Downloader( + config: $config, + logger: new Logger(), + repositoryProvider: (new RepositoryProvider())->addRepositoryFactory($factory), + architecture: Architecture::tryFromString('amd64') ?? throw new \LogicException(), + operatingSystem: OperatingSystem::tryFromString('linux') ?? throw new \LogicException(), + stability: Stability::Stable, + archiveService: new ArchiveFactory(), + registry: $this->registry, + ); + + $software = Software::fromArray([ + 'name' => 'rr', + 'repositories' => [['type' => 'github', 'uri' => $uri]], + ]); + $task = $downloader->download($software, DownloadConfig::fromSoftwareId('rr'), static fn(): null => null); + + /** @var DownloadResult */ + return await(($task->handler)()); + } + + /** + * @param list|SequenceRepositoryFactoryStub $repositories + */ + private function run( + array|SequenceRepositoryFactoryStub $repositories, + Software $software, + ?DownloadConfig $config = null, + ): DownloadResult { + $factory = $repositories instanceof SequenceRepositoryFactoryStub + ? $repositories + : new SequenceRepositoryFactoryStub($repositories); + + $downloaderConfig = new DownloaderConfig(); + $downloaderConfig->tmpDir = $this->tempDir; + + $task = $this->makeDownloader($factory, $downloaderConfig)->download( + $software, + $config ?? DownloadConfig::fromSoftwareId('rr'), + static fn(): null => null, + ); + + /** @var DownloadResult */ + return await(($task->handler)()); + } + + private function makeDownloader(SequenceRepositoryFactoryStub $factory, DownloaderConfig $config): Downloader + { + return new Downloader( + config: $config, + logger: new Logger(), + repositoryProvider: (new RepositoryProvider())->addRepositoryFactory($factory), + architecture: Architecture::tryFromString('amd64') ?? throw new \LogicException(), + operatingSystem: OperatingSystem::tryFromString('linux') ?? throw new \LogicException(), + stability: Stability::Stable, + archiveService: new ArchiveFactory(), + registry: $this->registry, + ); + } + + /** + * @param non-empty-string $name Asset file name. + * @param non-empty-string $tag Release tag. + */ + private function repoWithAsset( + string $name, + ?OperatingSystem $os = null, + ?Architecture $arch = null, + ?\Throwable $failure = null, + string $tag = 'v1.0.0', + ): RepositoryStub { + $repository = new RepositoryStub('owner/repo'); + $release = new ReleaseStub($repository, $tag, Version::fromVersionString($tag)); + $release->setAssets([new ConfigurableAssetStub($release, $name, $os, $arch, $failure)]); + + return new RepositoryStub('owner/repo', ReleasesCollection::create([$release])); + } + + /** + * @param non-empty-string $tag Release tag; the asset always matches the Linux/amd64 target. + */ + private function assetRelease(RepositoryStub $repository, string $tag): ReleaseStub + { + $release = new ReleaseStub($repository, $tag, Version::fromVersionString($tag)); + $release->setAssets([ + new ConfigurableAssetStub($release, 'rr-linux-amd64.tar.gz', OperatingSystem::Linux, Architecture::X86_64), + ]); + + return $release; + } +} diff --git a/tests/Unit/Module/Downloader/Stub/ConfigurableAssetStub.php b/tests/Unit/Module/Downloader/Stub/ConfigurableAssetStub.php new file mode 100644 index 0000000..de530a7 --- /dev/null +++ b/tests/Unit/Module/Downloader/Stub/ConfigurableAssetStub.php @@ -0,0 +1,66 @@ +release; + } + + public function getName(): string + { + return $this->name; + } + + public function getUri(): string + { + return 'https://x/' . $this->name; + } + + public function getOperatingSystem(): ?OperatingSystem + { + return $this->operatingSystem; + } + + public function getArchitecture(): ?Architecture + { + return $this->architecture; + } + + public function download(): \Traversable + { + if ($this->failure !== null) { + throw $this->failure; + + /** @psalm-suppress UnevaluatedCode */ + yield ''; + } + + yield 'Mock download content for ' . $this->name; + } +} diff --git a/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php b/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php new file mode 100644 index 0000000..aca547d --- /dev/null +++ b/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php @@ -0,0 +1,60 @@ +release; + } + + public function getName(): string + { + return $this->name; + } + + public function getUri(): string + { + return 'https://github.com/owner/repo/releases/download/' . $this->release->getName() . '/' . $this->name; + } + + public function getOperatingSystem(): ?OperatingSystem + { + return null; + } + + public function getArchitecture(): ?Architecture + { + return null; + } + + public function download(): \Traversable + { + throw $this->failure ?? new AssetNotFoundException('GitHub asset is no longer available: HTTP 404 for ' . $this->getUri(), 'owner/repo'); + + /** @psalm-suppress UnevaluatedCode */ + yield ''; + } +} diff --git a/tests/Unit/Module/Downloader/Stub/SequenceRepositoryFactoryStub.php b/tests/Unit/Module/Downloader/Stub/SequenceRepositoryFactoryStub.php new file mode 100644 index 0000000..a89bb6c --- /dev/null +++ b/tests/Unit/Module/Downloader/Stub/SequenceRepositoryFactoryStub.php @@ -0,0 +1,39 @@ + Number of repositories created so far. */ + public int $created = 0; + + /** + * @param list $repositories Repositories to return, in order; the last one repeats. + */ + public function __construct( + private readonly array $repositories, + ) {} + + public function supports(RepositoryConfig $config): bool + { + return true; + } + + public function create(RepositoryConfig $config): Repository + { + $repository = $this->repositories[\min($this->created, \count($this->repositories) - 1)]; + ++$this->created; + + return $repository; + } +} diff --git a/tests/Unit/Module/Downloader/Stub/ThrowingRepositoryStub.php b/tests/Unit/Module/Downloader/Stub/ThrowingRepositoryStub.php new file mode 100644 index 0000000..d09d4be --- /dev/null +++ b/tests/Unit/Module/Downloader/Stub/ThrowingRepositoryStub.php @@ -0,0 +1,29 @@ +name; + } + + public function getReleases(): ReleasesCollection + { + throw $this->failure; + } +} diff --git a/tests/Unit/Module/Registry/CacheDirectoryTest.php b/tests/Unit/Module/Registry/CacheDirectoryTest.php new file mode 100644 index 0000000..746aec7 --- /dev/null +++ b/tests/Unit/Module/Registry/CacheDirectoryTest.php @@ -0,0 +1,68 @@ + '/var/cache/', 'HOME' => '/home/u', 'LOCALAPPDATA' => 'C:\\x'], windows: true); + + Assert::same((string) $dir, (string) Path::create('/var/cache')->join('dload')); + } + + #[Test] + public function localAppDataIsUsedOnWindowsOnly(): void + { + $env = ['LOCALAPPDATA' => 'C:\\Users\\u\\AppData\\Local', 'HOME' => '/home/u']; + + Assert::same( + (string) CacheDirectory::resolve($env, windows: true), + (string) Path::create('C:\\Users\\u\\AppData\\Local')->join('dload', 'cache'), + ); + Assert::same( + (string) CacheDirectory::resolve($env, windows: false), + (string) Path::create('/home/u')->join('.cache', 'dload'), + ); + } + + #[Test] + public function homeIsUsedWhenNothingElseIsSet(): void + { + $dir = CacheDirectory::resolve(['HOME' => '/home/u', 'XDG_CACHE_HOME' => ' '], windows: false); + + Assert::same((string) $dir, (string) Path::create('/home/u')->join('.cache', 'dload')); + } + + #[Test] + public function userProfileStandsInForHome(): void + { + $dir = CacheDirectory::resolve(['USERPROFILE' => 'C:\\Users\\u'], windows: true); + + Assert::same((string) $dir, (string) Path::create('C:\\Users\\u')->join('.cache', 'dload')); + } + + #[Test] + public function fallsBackToAPerUserTemporaryDirectory(): void + { + $dir = CacheDirectory::resolve(['USER' => 'j doe'], windows: false); + + Assert::same((string) $dir, (string) Path::create(\sys_get_temp_dir())->join('dload-cache-j_doe')); + + // Without a user name in the environment the process owner keeps the directories apart + $anonymous = (string) CacheDirectory::resolve([], windows: false); + $prefix = (string) Path::create(\sys_get_temp_dir())->join('dload-cache-'); + Assert::true(\str_starts_with($anonymous, $prefix)); + Assert::true(\strlen($anonymous) > \strlen($prefix)); + } +} diff --git a/tests/Unit/Module/Registry/FileRegistryStorageTest.php b/tests/Unit/Module/Registry/FileRegistryStorageTest.php new file mode 100644 index 0000000..adfb58f --- /dev/null +++ b/tests/Unit/Module/Registry/FileRegistryStorageTest.php @@ -0,0 +1,356 @@ +storage(); + + $storage->save(self::record('github', 'roadrunner-server/roadrunner', self::range(150, 1))); + $storage->save(self::record('gitlab', 'group/sub/project', ['v2'])); + + $repo = $this->directory . '/repositories/github/roadrunner-server/roadrunner'; + Assert::true(\is_file($repo . '/index.json')); + Assert::true(\is_file($repo . '/releases-0001.json')); + Assert::true(\is_file($repo . '/releases-0002.json')); + Assert::true(\is_file($this->directory . '/repositories/gitlab/group/sub/project/index.json')); + + $loaded = $storage->load(new RepositoryId('github', 'roadrunner-server/roadrunner')); + Assert::same($loaded->count(), 150); + Assert::same($loaded->releases()[0]->tag, 'v150'); + Assert::same($loaded->releases()[149]->tag, 'v1'); + Assert::same($loaded->software, ['rr']); + } + + #[Test] + public function hiddenFlagSurvivesTheRoundTrip(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', 'owner/repo'); + + $storage->save((new RepositoryRecord($id, checkedAt: 1_000))->withHead([ + new ReleaseRecord('draft', 'draft', hidden: true), + new ReleaseRecord('v1', 'v1'), + ])); + + $releases = $storage->load($id)?->releases() ?? []; + Assert::true($releases[0]->hidden); + Assert::false($releases[1]->hidden); + } + + #[Test] + public function segmentsAreReadWhenReachedAndOnlyDirtyOnesAreWritten(): void + { + $storage = $this->storage(); + $storage->save(self::record('github', 'owner/repo', self::range(150, 1))); + $repo = $this->directory . '/repositories/github/owner/repo'; + + // A marker in the first segment shows whether the file is rewritten + $first = \file_get_contents($repo . '/releases-0001.json'); + \file_put_contents($repo . '/releases-0001.json', \str_replace('"v150"', '"v150" ', $first)); + + $loaded = $storage->load(new RepositoryId('github', 'owner/repo')); + Assert::same($loaded->count(), 150); + + // Appending to the tail opens a new segment after the full one and rewrites nothing else + $storage->save($loaded->withTail([new ReleaseRecord('v0', 'v0')])); + + Assert::string(\file_get_contents($repo . '/releases-0001.json'))->contains('"v150" '); + Assert::string(\file_get_contents($repo . '/releases-0003.json'))->contains('"v0"'); + Assert::same($storage->load(new RepositoryId('github', 'owner/repo'))->count(), 151); + } + + #[Test] + public function replacedSegmentFilesAreRemoved(): void + { + $storage = $this->storage(); + $storage->save(self::record('github', 'owner/repo', self::range(150, 1))); + $repo = $this->directory . '/repositories/github/owner/repo'; + + // The new release and the head segment of 50 are repacked into one file; the old one is dropped + $storage->save($storage->load(new RepositoryId('github', 'owner/repo'))->withHead([new ReleaseRecord('v151', 'v151'), new ReleaseRecord('v150', 'v150')])); + + Assert::false(\is_file($repo . '/releases-0001.json')); + Assert::true(\is_file($repo . '/releases-0002.json')); + Assert::true(\is_file($repo . '/releases-0003.json')); + Assert::same($storage->load(new RepositoryId('github', 'owner/repo'))->count(), 151); + } + + #[Test] + public function missingAndCorruptedRecordsReadAsNull(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', 'owner/repo'); + + Assert::null($storage->load($id)); + + $storage->save(self::record('github', 'owner/repo', ['v1'])); + \file_put_contents($this->directory . '/repositories/github/owner/repo/index.json', '{not json'); + + Assert::null($storage->load($id)); + } + + #[Test] + public function recordWithAMissingSegmentFileReadsAsNull(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', 'owner/repo'); + $storage->save(self::record('github', 'owner/repo', ['v1'])); + + \unlink($this->directory . '/repositories/github/owner/repo/releases-0001.json'); + + Assert::null($storage->load($id)); + } + + #[Test] + public function corruptedSegmentDropsTheRecordWhenReached(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', 'owner/repo'); + $storage->save(self::record('github', 'owner/repo', ['v1'])); + \file_put_contents($this->directory . '/repositories/github/owner/repo/releases-0001.json', '[{"name": "no tag"}]'); + + $loaded = $storage->load($id); + Assert::notNull($loaded); + + try { + $loaded->releases(); + Assert::fail('An unreadable segment must be reported.'); + } catch (\RuntimeException $e) { + Assert::string($e->getMessage())->contains('unreadable'); + Assert::false(\is_dir($this->directory . '/repositories/github/owner/repo')); + } + } + + #[Test] + public function saveReportsAnUnwritableDirectory(): void + { + $storage = $this->storage(); + // A file where the repository directory should be + \mkdir($this->directory . '/repositories/github', recursive: true); + \file_put_contents($this->directory . '/repositories/github/owner', 'not a directory'); + + try { + $storage->save(self::record('github', 'owner/repo', ['v1'])); + Assert::fail('A failed write must be reported.'); + } catch (\RuntimeException $e) { + Assert::string($e->getMessage())->contains('registry'); + } + } + + #[Test] + public function staleTemporaryFilesAreCleanedUp(): void + { + $storage = $this->storage(); + $storage->save(self::record('github', 'owner/repo', ['v1'])); + $repo = $this->directory . '/repositories/github/owner/repo'; + + \file_put_contents($repo . '/index.json.123.tmp', '{}'); + \touch($repo . '/index.json.123.tmp', \time() - 7200); + \file_put_contents($repo . '/index.json.456.tmp', '{}'); + + $storage->save($storage->load(new RepositoryId('github', 'owner/repo'))->withSoftware('rr2')); + + Assert::false(\is_file($repo . '/index.json.123.tmp')); + Assert::true(\is_file($repo . '/index.json.456.tmp')); + } + + #[Test] + public function saveWaitsForTheLockOfAnotherRunAndGivesUp(): void + { + $storage = $this->storage(lockTimeout: 0.2); + $storage->save(self::record('github', 'owner/repo', ['v1'])); + + // Another run holds the repository + $lock = \fopen($this->directory . '/locks/github_owner_repo.lock', 'c'); + \flock($lock, \LOCK_EX); + + $started = \microtime(true); + try { + $storage->save(self::record('github', 'owner/repo', ['v2', 'v1'])); + Assert::fail('A lock held by another run must be reported.'); + } catch (\RuntimeException $e) { + Assert::string($e->getMessage())->contains('Another run holds'); + Assert::true(\microtime(true) - $started >= 0.2); + } + + // The record was left untouched, and the lock is taken as soon as it is released + Assert::same($storage->load(new RepositoryId('github', 'owner/repo'))?->count(), 1); + \flock($lock, \LOCK_UN); + \fclose($lock); + $storage->save(self::record('github', 'owner/repo', ['v2', 'v1'])); + Assert::same($storage->load(new RepositoryId('github', 'owner/repo'))?->count(), 2); + } + + #[Test] + public function listsRemovesAndClears(): void + { + $storage = $this->storage(); + $storage->save(self::record('github', 'a/b', ['v1'])); + $storage->save(self::record('github', 'c/d', ['v1'])); + + Assert::count(\iterator_to_array($storage->all(), false), 2); + + $storage->remove(new RepositoryId('github', 'a/b')); + Assert::count(\iterator_to_array($storage->all(), false), 1); + Assert::false(\is_dir($this->directory . '/repositories/github/a')); + Assert::true(\is_dir($this->directory . '/repositories/github/c/d')); + + $storage->clear(); + Assert::count(\iterator_to_array($storage->all(), false), 0); + Assert::null($storage->load(new RepositoryId('github', 'c/d'))); + Assert::false(\is_dir($this->directory . '/locks')); + } + + #[Test] + public function unsafePathSegmentsAreSanitized(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', '../owner/re po:x'); + + $storage->save(new RepositoryRecord($id)); + + Assert::false(\is_dir(\dirname($this->directory) . '/owner')); + Assert::true(\is_file($this->directory . '/repositories/github/_/owner/re_po_x/index.json')); + Assert::true($storage->load($id)?->id->equals($id) ?? false); + } + + #[Test] + public function recordOfAnotherRepositoryInTheSameDirectoryIsIgnored(): void + { + // Two identities sanitize to one directory name + $storage = $this->storage(); + $storage->save(self::record('github', 'owner/re?po', ['v1'])); + + Assert::null($storage->load(new RepositoryId('github', 'owner/re*po'))); + Assert::notNull($storage->load(new RepositoryId('github', 'owner/re?po'))); + } + + #[Test] + public function windowsDeviceNamesAreEscaped(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', 'nul/com1'); + + $storage->save(self::record('github', 'nul/com1', ['v1'])); + + Assert::true(\is_file($this->directory . '/repositories/github/_nul/_com1/index.json')); + Assert::same($storage->load($id)?->releases()[0]->tag, 'v1'); + } + + #[Test] + public function removingAnUnknownRepositoryIsANoOp(): void + { + $storage = $this->storage(); + $storage->save(self::record('github', 'owner/repo', ['v1'])); + + $storage->remove(new RepositoryId('github', 'ghost/repo')); + + Assert::notNull($storage->load(new RepositoryId('github', 'owner/repo'))); + Assert::null($storage->load(new RepositoryId('github', 'ghost/repo'))); + Assert::false(\is_dir($this->directory . '/repositories/github/ghost')); + } + + #[Test] + public function saveReportsAFailedTemporaryWrite(): void + { + $storage = $this->storage(); + $repo = $this->directory . '/repositories/github/owner/repo'; + \mkdir($repo, recursive: true); + + // A directory sitting at the exact temporary path a write uses makes file_put_contents fail + \mkdir($repo . '/releases-0001.json.' . \getmypid() . '.tmp'); + + try { + $storage->save(self::record('github', 'owner/repo', ['v1'])); + Assert::fail('A failed temporary write must be reported.'); + } catch (\RuntimeException $e) { + Assert::string($e->getMessage())->contains('Failed to write registry file'); + } + } + + #[Test] + public function saveReportsAFailedRenameIntoPlace(): void + { + $storage = $this->storage(); + $repo = $this->directory . '/repositories/github/owner/repo'; + \mkdir($repo, recursive: true); + + // A directory where the index file belongs makes the rename into place fail once it cannot be + // emptied: Windows refuses to delete an open file, POSIX a file in a directory without write + // permission (root ignores permissions, so the test cannot be run as root) + \DIRECTORY_SEPARATOR === '\\' || !\function_exists('posix_geteuid') || \posix_geteuid() !== 0 + or throw new SkipTest('Root can remove any directory.'); + \mkdir($repo . '/index.json/locked', recursive: true); + $pin = \fopen($repo . '/index.json/locked/pin', 'w'); + \chmod($repo . '/index.json/locked', 0555); + + try { + $storage->save(new RepositoryRecord(new RepositoryId('github', 'owner/repo'))); + Assert::fail('A failed rename into place must be reported.'); + } catch (\RuntimeException $e) { + Assert::string($e->getMessage())->contains('Failed to store registry file'); + } finally { + \fclose($pin); + \chmod($repo . '/index.json/locked', 0755); + } + } + + #[BeforeTest] + protected function prepare(): void + { + $this->directory = \sys_get_temp_dir() . '/dload-registry-' . \bin2hex(\random_bytes(6)); + } + + #[AfterTest] + protected function cleanup(): void + { + \is_dir($this->directory) and FS::removeDir(Path::create($this->directory)); + } + + /** + * @param non-empty-string $type + * @param non-empty-string $uri + * @param list $tags Newest first. + */ + private static function record(string $type, string $uri, array $tags): RepositoryRecord + { + return (new RepositoryRecord(id: new RepositoryId($type, $uri), checkedAt: 1_000, software: ['rr'])) + ->withHead(\array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags)); + } + + /** + * @return list `v` down to `v`. + */ + private static function range(int $from, int $to): array + { + return \array_map(static fn(int $i): string => 'v' . $i, \range($from, $to)); + } + + private function storage(float $lockTimeout = 10.0): FileRegistryStorage + { + return new FileRegistryStorage(Path::create($this->directory), new Logger(), $lockTimeout); + } +} diff --git a/tests/Unit/Module/Registry/PassThroughRegistryTest.php b/tests/Unit/Module/Registry/PassThroughRegistryTest.php new file mode 100644 index 0000000..3dc10a1 --- /dev/null +++ b/tests/Unit/Module/Registry/PassThroughRegistryTest.php @@ -0,0 +1,37 @@ +attach($id, 'rr'); + $first = \iterator_to_array($registry->releases($id, $source), false); + $registry->forget($id, 'v3'); + $second = \iterator_to_array($registry->releases($id, $source), false); + + Assert::same($source->served, [0, 2, 0, 2]); + Assert::same($first, $second); + Assert::same( + \array_map(static fn(ReleaseRecord $release): string => $release->tag, \array_merge(...$first)), + ['v3', 'v2', 'v1'], + ); + } +} diff --git a/tests/Unit/Module/Registry/RepositoryRecordTest.php b/tests/Unit/Module/Registry/RepositoryRecordTest.php new file mode 100644 index 0000000..18e4aec --- /dev/null +++ b/tests/Unit/Module/Registry/RepositoryRecordTest.php @@ -0,0 +1,451 @@ + 'old v2', 'v1' => 'v1']); + + $updated = $record->withHead([new ReleaseRecord('v3', 'v3'), new ReleaseRecord('v2', 'new v2')]); + + Assert::same(self::tags($updated), ['v3', 'v2', 'v1']); + Assert::same($updated->releases()[1]->name, 'new v2'); + } + + #[Test] + public function headDropsStoredReleasesMissingFromTheFetchedSpan(): void + { + // `v3` was deleted upstream: the fresh head reaches `v2`, and `v3` is not in it + $record = self::record(['v4', 'v3', 'v2', 'v1']); + + $updated = $record->withHead(self::releases(['v5', 'v4', 'v2'])); + + Assert::same(self::tags($updated), ['v5', 'v4', 'v2', 'v1']); + Assert::same($updated->count(), 4); + } + + #[Test] + public function headDropsTheStoredReleaseRightAfterTheOnlyOneItReaches(): void + { + // `v5` was deleted: the fresh page reads `v6, v4`, so `v5` no longer follows `v6` + $record = self::record(['v6', 'v5']); + + $updated = $record->withHead(self::releases(['v6', 'v4'])); + + Assert::same(self::tags($updated), ['v6', 'v4']); + } + + #[Test] + public function headReachingNoStoredReleaseIsTheWholeListing(): void + { + $updated = self::record(['v1'])->withHead(self::releases(['v2'])); + + Assert::same(self::tags($updated), ['v2']); + } + + #[Test] + public function headKeepsUntouchedSegmentsAsTheyAre(): void + { + // Three full segments; the head reaches the first one only + $record = self::record(self::range(300, 1)); + $untouched = \array_slice($record->segments, 1); + + $updated = $record->withHead(self::releases(['v301', 'v300', 'v299'])); + + Assert::same($updated->count(), 301); + Assert::same(self::sizes($updated), [1, 100, 100, 100]); + Assert::same(\array_slice($updated->segments, -2), $untouched); + Assert::false($updated->segments[3]->dirty); + Assert::true($updated->segments[0]->dirty); + Assert::same($updated->segments[0]->tags, ['v301']); + } + + #[Test] + public function partialSegmentSitsAtTheHeadAndGrowsWithEveryCheck(): void + { + $record = self::record(self::range(150, 1)); + Assert::same(self::sizes($record), [50, 100]); + + // New releases join the head segment; the full one behind it is untouched + $updated = $record->withHead(self::releases(['v152', 'v151', 'v150'])); + Assert::same(self::sizes($updated), [52, 100]); + Assert::false($updated->segments[1]->dirty); + Assert::same(\array_slice(self::tags($updated), 0, 4), ['v152', 'v151', 'v150', 'v149']); + } + + #[Test] + public function shortHeadJoinsTheNextSegmentWhileBothFitIntoOne(): void + { + $record = self::record(self::range(101, 1)); + Assert::same(self::sizes($record), [1, 100]); + + // `v101` opens its segment, so nothing is split; the single new release joins it + $updated = $record->withHead(self::releases(['v102', 'v101'])); + + Assert::same(self::sizes($updated), [2, 100]); + Assert::false($updated->segments[1]->dirty); + Assert::same($updated->segments[0]->tags, ['v102', 'v101']); + } + + #[Test] + public function headAbsorbsAShortFollowingSegmentThatFits(): void + { + // A full head segment and a short tail of 10 kept in its own segment + $record = self::record(self::range(100, 1))->withTail(self::releases(self::range(-1, -10))); + Assert::same(self::sizes($record), [100, 10]); + + // `v100` is reached, so the head segment is repacked; its 1-release remainder plus the + // 10-release tail segment fit into one, so the tail is pulled in instead of left behind + $updated = $record->withHead(self::releases(['v101', 'v100'])); + + Assert::same($updated->count(), 111); + Assert::same(self::sizes($updated), [11, 100]); + Assert::true($updated->has('v-10')); + Assert::same(self::tags($updated)[0], 'v101'); + } + + #[Test] + public function emptyHeadClearsTheListing(): void + { + // No fetched releases means the source lists none, so the whole stored listing is dropped + $updated = self::record(['v2', 'v1'])->withHead([]); + + Assert::same($updated->count(), 0); + Assert::same($updated->releases(), []); + } + + #[Test] + public function tailFillsTheLastSegmentBeforeStartingANewOne(): void + { + $record = self::record(self::range(150, 1)); + $older = \array_map(static fn(int $i): string => 'o' . $i, \range(1, 70)); + + // The last segment is full: the older releases open a new one + $extended = $record->withTail(self::releases(\array_slice($older, 0, 62)))->persisted(); + Assert::same(self::sizes($extended), [50, 100, 62]); + Assert::same(self::keys($extended), ['0001', '0002', '0003']); + + // The next page fills that segment up under the same key and starts another + $more = $extended->withTail(self::releases(\array_slice($older, 62)))->withTail(self::releases(self::range(0, -35))); + Assert::same(self::sizes($more), [50, 100, 100, 6]); + Assert::same(self::keys($more), ['0001', '0002', '0003', '0004']); + Assert::false($more->segments[1]->dirty); + Assert::true($more->segments[2]->dirty); + Assert::same($more->segments[2]->tags[99], 'v-29'); + } + + #[Test] + public function tailIgnoresKnownReleases(): void + { + $record = self::record(['v2']); + + $updated = $record->withTail([new ReleaseRecord('v2', 'dup'), new ReleaseRecord('v1', 'v1')]); + + Assert::same(self::tags($updated), ['v2', 'v1']); + Assert::same($updated->releases()[0]->name, 'v2'); + Assert::same($record->withTail([new ReleaseRecord('v2', 'dup')]), $record); + } + + #[Test] + public function segmentKeysAreNeverReused(): void + { + $record = self::record(self::range(200, 1)); + Assert::same(self::keys($record), ['0001', '0002']); + + // The head repacks the first segment under fresh keys; the old key is gone + // `v200` was deleted: the first segment is repacked under a fresh key, the old key is gone + $updated = $record->withHead(self::releases(['v201', 'v199'])); + Assert::same(self::keys($updated), ['0003', '0002']); + Assert::same(self::sizes($updated), [100, 100]); + + // The tail appends after the greatest key ever used + $extended = $updated->withTail(self::releases(['v0'])); + Assert::same(self::keys($extended), ['0003', '0002', '0004']); + } + + #[Test] + public function releaseCanBeDroppedAndTheCheckForgotten(): void + { + $record = (new RepositoryRecord(self::id(), checkedAt: 1_000))->withHead(self::releases(['v2', 'v1'])); + + $dropped = $record->withoutRelease('v2'); + + Assert::same(self::tags($dropped), ['v1']); + Assert::same($dropped->checkedAt, 1_000); + Assert::same($record->withoutRelease('v9'), $record); + + $stale = $dropped->withoutCheck(); + Assert::null($stale->checkedAt); + Assert::same(self::tags($stale), ['v1']); + Assert::true($stale->isStale(1_000, 600)); + } + + #[Test] + public function droppingTheLastReleaseOfASegmentDropsTheSegment(): void + { + $record = self::record(self::range(101, 1)); + Assert::same(self::sizes($record), [1, 100]); + + $updated = $record->withoutRelease('v101'); + + Assert::same(self::sizes($updated), [100]); + Assert::false($updated->has('v101')); + Assert::false($updated->segments[0]->dirty); + } + + #[Test] + public function stalenessDependsOnTheLastCheck(): void + { + $never = RepositoryRecord::empty(self::id()); + $checked = $never->withCheckedAt(1_000); + + Assert::true($never->isStale(1_000, 600)); + Assert::false($checked->isStale(1_600, 600)); + Assert::true($checked->isStale(1_601, 600)); + } + + #[Test] + public function completenessFlagIsToggled(): void + { + $record = RepositoryRecord::empty(self::id()); + Assert::false($record->complete); + + Assert::true($record->withComplete(true)->complete); + Assert::false($record->withComplete(true)->withComplete(false)->complete); + } + + #[Test] + public function softwareIsAttachedOnce(): void + { + $record = RepositoryRecord::empty(self::id())->withSoftware('rr'); + + Assert::same($record->withSoftware('rr'), $record); + Assert::same($record->withSoftware('temporal')->software, ['rr', 'temporal']); + } + + #[Test] + public function survivesTheArrayRoundTrip(): void + { + $record = (new RepositoryRecord(id: self::id(), checkedAt: 1_000, complete: true, software: ['rr'])) + ->withHead([ + new ReleaseRecord( + tag: 'v2.0.0', + name: 'Release 2', + publishedAt: new \DateTimeImmutable('2024-01-02T03:04:05+00:00'), + prerelease: true, + assets: [new AssetRecord('rr-linux-amd64.tar.gz', 'https://x/rr.tar.gz', 42, 'application/gzip', 'sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08')], + ), + new ReleaseRecord('v1.0.0', 'v1.0.0'), + ]); + + // The index travels as JSON; the segments are handed back through the loader + $segments = []; + foreach ($record->segments as $segment) { + $segments[$segment->key] = \json_decode(\json_encode(\array_map( + static fn(ReleaseRecord $release): array => $release->toArray(), + $segment->releases(), + )), true); + } + $restored = RepositoryRecord::fromArray( + \json_decode(\json_encode($record->toArray()), true), + static fn(string $key): array => \array_map(ReleaseRecord::fromArray(...), $segments[$key]), + ); + + Assert::same($restored->toArray(), $record->toArray()); + Assert::true($restored->id->equals(self::id())); + Assert::same($restored->count(), 2); + Assert::false($restored->segments[0]->dirty); + Assert::same($restored->releases()[0]->assets[0]->size, 42); + Assert::same($restored->releases()[0]->assets[0]->digest, 'sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'); + Assert::same($restored->releases()[0]->publishedAt?->format(\DATE_ATOM), '2024-01-02T03:04:05+00:00'); + Assert::null($restored->releases()[1]->publishedAt); + Assert::null(AssetRecord::fromArray(['name' => 'x', 'uri' => 'https://x', 'digest' => ''])->digest); + } + + #[Test] + public function segmentsAreReadOnlyWhenReached(): void + { + $loaded = []; + $old = self::range(100, 1); + $record = RepositoryRecord::fromArray( + [ + 'version' => RepositoryRecord::FORMAT_VERSION, + 'repository' => ['type' => 'github', 'uri' => 'owner/repo'], + 'segments' => [['key' => 'a', 'tags' => ['v102', 'v101']], ['key' => 'b', 'tags' => $old]], + ], + static function (string $key) use (&$loaded, $old): array { + $loaded[] = $key; + + return self::releases($key === 'a' ? ['v102', 'v101'] : $old); + }, + ); + + Assert::same($record->count(), 102); + Assert::true($record->has('v1')); + Assert::same($loaded, []); + + $record->pages()->current(); + Assert::same($loaded, ['a']); + + // The head touches the first segment only; the full one behind it is neither read nor rewritten + $updated = $record->withHead(self::releases(['v103', 'v102'])); + Assert::same($loaded, ['a']); + Assert::same(self::sizes($updated), [3, 100]); + Assert::false($updated->segments[1]->dirty); + } + + #[Test] + public function persistedRecordHasNoDirtySegments(): void + { + $record = RepositoryRecord::empty(self::id())->withHead(self::releases(['v1'])); + Assert::true($record->segments[0]->dirty); + + $persisted = $record->persisted(); + + Assert::false($persisted->segments[0]->dirty); + Assert::same(self::tags($persisted), ['v1']); + Assert::same($persisted->persisted()->segments, $persisted->segments); + } + + #[Test] + public function rejectsUnknownFormatVersion(): void + { + try { + RepositoryRecord::fromArray(['version' => 99, 'repository' => ['type' => 'github', 'uri' => 'a/b']], static fn(): array => []); + Assert::fail('An unknown format version must be rejected.'); + } catch (\InvalidArgumentException $e) { + Assert::same($e->getMessage(), 'Unsupported repository record format.'); + } + } + + #[Test] + public function fromArrayRejectsMissingRepositoryIdentity(): void + { + try { + RepositoryRecord::fromArray( + ['version' => RepositoryRecord::FORMAT_VERSION, 'repository' => ['type' => 'github']], + static fn(): array => [], + ); + Assert::fail('A record without a repository URI must be rejected.'); + } catch (\InvalidArgumentException $e) { + Assert::same($e->getMessage(), 'Repository record requires a repository type and URI.'); + } + } + + #[Test] + public function skipsBrokenReleasesButRequiresATag(): void + { + try { + ReleaseRecord::fromArray(['name' => 'no tag']); + Assert::fail('A release without a tag must be rejected.'); + } catch (\InvalidArgumentException $e) { + Assert::same($e->getMessage(), 'Release record requires a non-empty `tag`.'); + } + + // A broken asset makes the whole release unusable rather than silently dropping the asset + try { + ReleaseRecord::fromArray(['tag' => 'v1', 'assets' => [['name' => 'x']]]); + Assert::fail('An asset without a URI must be rejected.'); + } catch (\InvalidArgumentException) { + } + + $release = ReleaseRecord::fromArray(['tag' => 'v1', 'assets' => [['name' => 'ok', 'uri' => 'https://x']]]); + Assert::same($release->name, 'v1'); + Assert::count($release->assets, 1); + } + + #[Test] + public function repositoryIdIsNormalized(): void + { + $id = new RepositoryId('GitHub', '/Owner/Repo/'); + + Assert::same((string) $id, 'github:owner/repo'); + Assert::true($id->equals(new RepositoryId('github', 'owner/repo'))); + + try { + new RepositoryId('github', '/'); + Assert::fail('A URI without a path must be rejected.'); + } catch (\InvalidArgumentException) { + } + } + + private static function id(): RepositoryId + { + return new RepositoryId('github', 'owner/repo'); + } + + /** + * Record holding the given releases as freshly packed segments. + * + * @param array $tags Newest first; a string key is the tag and the value its name. + */ + private static function record(array $tags): RepositoryRecord + { + $releases = []; + foreach ($tags as $key => $value) { + $releases[] = \is_string($key) ? new ReleaseRecord($key, $value) : new ReleaseRecord($value, $value); + } + + return RepositoryRecord::empty(self::id())->withHead($releases)->persisted(); + } + + /** + * @param list $tags + * @return list + */ + private static function releases(array $tags): array + { + return \array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags); + } + + /** + * @return list `v` down to `v`. + */ + private static function range(int $from, int $to): array + { + return \array_map(static fn(int $i): string => 'v' . $i, \range($from, $to)); + } + + /** + * @return list + */ + private static function tags(RepositoryRecord $record): array + { + return \array_map(static fn(ReleaseRecord $release): string => $release->tag, $record->releases()); + } + + /** + * @return list + */ + private static function sizes(RepositoryRecord $record): array + { + return \array_map(static fn(ReleaseSegment $segment): int => $segment->count(), $record->segments); + } + + /** + * @return list + */ + private static function keys(RepositoryRecord $record): array + { + return \array_map(static fn(ReleaseSegment $segment): string => $segment->key, $record->segments); + } +} diff --git a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php new file mode 100644 index 0000000..45cbf5a --- /dev/null +++ b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php @@ -0,0 +1,378 @@ +registry()->releases($this->id, $source)), ['v2', 'v1']); + Assert::same($this->storage->load($this->id)?->count(), 3); + + // The stored list is served without a request and still leaves the hidden one out + Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v2', 'v1']); + Assert::same($source->served, [0, 2]); + } + + #[Test] + public function firstRunLoadsOnlyThePagesTheConsumerNeeds(): void + { + $source = ArrayReleaseSource::ofTags(['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + $registry = $this->registry(); + + $pages = $registry->releases($this->id, $source); + $first = self::tagsOf($pages->current()); + + Assert::same($first, ['v6', 'v5']); + Assert::same($source->served, [0]); + + // The record already holds what was fetched, marked as incomplete + $record = $this->storage->load($this->id); + Assert::same(self::tagsOf($record->releases()), ['v6', 'v5']); + Assert::false($record->complete); + Assert::same($record->checkedAt, $this->now); + } + + #[Test] + public function olderReleasesAreLoadedOnDemandAndPersisted(): void + { + $source = ArrayReleaseSource::ofTags(['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + $registry = $this->registry(); + + $all = self::flatten($registry->releases($this->id, $source)); + + Assert::same($all, ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($source->served, [0, 2, 4]); + + $record = $this->storage->load($this->id); + Assert::same(self::tagsOf($record->releases()), ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::true($record->complete); + } + + #[Test] + public function freshRecordIsServedWithoutAnyRequest(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + $this->now += 100; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v3', 'v2', 'v1']); + Assert::same($source->served, []); + } + + #[Test] + public function staleRecordIsCheckedWithASinglePageWhenNothingIsNew(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v3', 'v2', 'v1']); + Assert::same($source->served, [0]); + Assert::same($this->storage->load($this->id)->checkedAt, $this->now); + } + + #[Test] + public function newReleasesAreFetchedUntilAKnownOneIsReached(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + // Three releases were published: they span two pages, the second one reaches `v3` + $source->publish('v6', 'v5', 'v4'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($source->served, [0, 2]); + } + + #[Test] + public function releaseDeletedUpstreamDoesNotShiftTheTail(): void + { + $source = ArrayReleaseSource::ofTags(['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + + // The first run stores the first page only + $this->registry()->releases($this->id, $source)->current(); + $source->served = []; + + // `v5` is deleted upstream, so every older release moves up one position + $source->delete('v5'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + // The check drops `v5`; the tail continues from the true offset and `v3` is not skipped + Assert::same($again, ['v6', 'v4', 'v3', 'v2', 'v1']); + Assert::same($source->served, [0, 2, 4]); + } + + #[Test] + public function unreadableReleaseIsNotTakenForADeletedOne(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1'], perPage: 3); + self::flatten($this->registry()->releases($this->id, $source)); + + // The source fails to read `v2` this time; the stored list must keep it + $source->unreadable = ['v2']; + $source->publish('v4'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v3', 'v2', 'v1']); + + // Nothing was stored, so the next run checks again instead of waiting for the TTL + $source->unreadable = []; + $source->served = []; + $fixed = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($fixed, ['v4', 'v3', 'v2', 'v1']); + Assert::same($source->served, [0]); + } + + #[Test] + public function unreadableReleaseIsSkippedWhenNothingIsStored(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1'], perPage: 3); + $source->unreadable = ['v2']; + + Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v3', 'v1']); + } + + #[Test] + public function failedTailLoadingStillServesTheStoredReleases(): void + { + $source = ArrayReleaseSource::ofTags(['v4', 'v3', 'v2', 'v1']); + $this->registry()->releases($this->id, $source)->current(); + + $source->fail(); + $pages = $this->registry()->releases($this->id, $source); + + // The stored page comes first; only the request for the tail fails + Assert::same(self::tagsOf($pages->current()), ['v4', 'v3']); + try { + $pages->next(); + Assert::fail('The failure of the tail request must reach the caller.'); + } catch (ApiException) { + Assert::false($this->storage->load($this->id)->complete); + } + } + + #[Test] + public function rateLimitOnCheckFallsBackToStoredReleasesAndIsReportedOnce(): void + { + $other = new RepositoryId('github', 'other/repo'); + $source = ArrayReleaseSource::ofTags(['v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + self::flatten($this->registry()->releases($other, $source)); + + $source->fail(new GitHubRateLimitException('Rate limit exceeded.', 'owner/repo', null)); + $this->now += 601; + $output = new BufferedOutput(); + $registry = $this->registry(logger: new Logger($output)); + + Assert::same(self::flatten($registry->releases($this->id, $source)), ['v2', 'v1']); + Assert::same(self::flatten($registry->releases($other, $source)), ['v2', 'v1']); + + // Shown without any verbosity flag, but not for every repository of the run + Assert::same(\substr_count($output->fetch(), 'rate limit prevents checking'), 1); + } + + #[Test] + public function ordinaryCheckFailureStaysOutOfTheDefaultOutput(): void + { + $source = ArrayReleaseSource::ofTags(['v1']); + self::flatten($this->registry()->releases($this->id, $source)); + + $source->fail(); + $this->now += 601; + $output = new BufferedOutput(); + self::flatten($this->registry(logger: new Logger($output))->releases($this->id, $source)); + + Assert::same($output->fetch(), ''); + } + + #[Test] + public function firstPageOverwritesStoredReleasesOnCheck(): void + { + $source = new ArrayReleaseSource([new ReleaseRecord('v1', 'v1', assets: [])]); + self::flatten($this->registry()->releases($this->id, $source)); + + // Assets were attached after the release had been stored + $updated = new ArrayReleaseSource([ + new ReleaseRecord('v1', 'v1', assets: [new \Internal\DLoad\Module\Registry\Record\AssetRecord('rr.zip', 'https://x/rr.zip')]), + ]); + $this->now += 601; + self::flatten($this->registry()->releases($this->id, $updated)); + + Assert::count($this->storage->load($this->id)->releases()[0]->assets, 1); + } + + #[Test] + public function refreshFlagIgnoresTheTtl(): void + { + $source = ArrayReleaseSource::ofTags(['v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + $source->publish('v3'); + self::flatten($this->registry(refresh: true)->releases($this->id, $source)); + + Assert::same($source->served, [0]); + Assert::same(self::tagsOf($this->storage->load($this->id)->releases()), ['v3', 'v2', 'v1']); + } + + #[Test] + public function failedCheckFallsBackToStoredReleases(): void + { + $source = ArrayReleaseSource::ofTags(['v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + + $source->fail(); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v2', 'v1']); + } + + #[Test] + public function failedCheckWithoutStoredReleasesIsReported(): void + { + $source = ArrayReleaseSource::ofTags(['v1']); + $source->fail(); + + try { + self::flatten($this->registry()->releases($this->id, $source)); + Assert::fail('The failure of the source must reach the caller when nothing is stored.'); + } catch (ApiException) { + Assert::same($this->storage->records, []); + } + } + + #[Test] + public function storageFailureDoesNotBreakTheListing(): void + { + $this->storage->failOnSave = true; + $source = ArrayReleaseSource::ofTags(['v2', 'v1']); + + Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v2', 'v1']); + } + + #[Test] + public function forgetDropsTheReleaseAndForcesTheNextCheck(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + $registry = $this->registry(); + self::flatten($registry->releases($this->id, $source)); + $source->served = []; + + // `v3` was deleted upstream and its download failed + $registry->forget($this->id, 'v3'); + + $record = $this->storage->load($this->id); + Assert::same(self::tagsOf($record->releases()), ['v2', 'v1']); + Assert::null($record->checkedAt); + + // The record is fresh by time, yet the next listing asks the source again + $again = self::flatten($this->registry()->releases($this->id, $source)); + Assert::same($source->served, [0]); + Assert::same($again, ['v3', 'v2', 'v1']); + } + + #[Test] + public function forgetOfAnUnknownReleaseChangesNothing(): void + { + $source = ArrayReleaseSource::ofTags(['v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $saves = $this->storage->saves; + + $this->registry()->forget($this->id, 'v9'); + $this->registry()->forget(new RepositoryId('github', 'other/repo'), 'v1'); + + Assert::same($this->storage->saves, $saves); + Assert::same($this->storage->load($this->id)->checkedAt, $this->now); + } + + #[Test] + public function attachRecordsTheSoftwareOnce(): void + { + $registry = $this->registry(); + + $registry->attach($this->id, 'rr'); + $registry->attach($this->id, 'rr'); + $registry->attach($this->id, 'roadrunner'); + + Assert::same($this->storage->load($this->id)->software, ['rr', 'roadrunner']); + Assert::same($this->storage->saves, 2); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->storage = new InMemoryRegistryStorage(); + $this->id = new RepositoryId('github', 'owner/repo'); + $this->now = 1_000_000; + } + + /** + * @param iterable> $pages + * @return list + */ + private static function flatten(iterable $pages): array + { + $tags = []; + foreach ($pages as $page) { + $tags = [...$tags, ...self::tagsOf($page)]; + } + + return $tags; + } + + /** + * @param list $releases + * @return list + */ + private static function tagsOf(array $releases): array + { + return \array_map(static fn(ReleaseRecord $release): string => $release->tag, $releases); + } + + private function registry(bool $refresh = false, ?Logger $logger = null): StoredVersionRegistry + { + return new StoredVersionRegistry($this->storage, 600, $logger ?? new Logger(), $refresh, fn(): int => $this->now); + } +} diff --git a/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php new file mode 100644 index 0000000..d1e1921 --- /dev/null +++ b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php @@ -0,0 +1,105 @@ + + */ + public array $served = []; + + /** When set, every page request fails with this exception. */ + public ?\Throwable $failure = null; + + /** + * Tags of releases the source cannot read: they are left out of the pages and counted as skipped. + * + * @var list + */ + public array $unreadable = []; + + /** + * @param list $releases Newest first. + * @param int<1, max> $perPage + */ + public function __construct( + private array $releases, + private readonly int $perPage = 2, + ) {} + + /** + * @param list $tags Newest first. + */ + public static function ofTags(array $tags, int $perPage = 2): self + { + return new self(\array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags), $perPage); + } + + /** + * Publishes releases on top of the list, as a repository would between two runs. + * + * @param non-empty-string ...$tags Newest first. + */ + public function publish(string ...$tags): void + { + $this->releases = [...\array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags), ...$this->releases]; + } + + /** + * Deletes a release from the list, as a repository owner would between two runs. + * + * @param non-empty-string $tag + */ + public function delete(string $tag): void + { + $this->releases = \array_values(\array_filter( + $this->releases, + static fn(ReleaseRecord $release): bool => $release->tag !== $tag, + )); + } + + public function fail(?\Throwable $failure = null): void + { + $this->failure = $failure ?? new ApiException('API is unavailable.', 'stub/stub'); + } + + public function pages(int $offset = 0): \Generator + { + // Align with a page boundary and skip within the page, like a real paged API + $page = \intdiv($offset, $this->perPage); + $skip = $offset % $this->perPage; + + do { + $this->failure === null or throw $this->failure; + + $this->served[] = $page * $this->perPage; + $items = \array_slice($this->releases, $page * $this->perPage, $this->perPage); + $last = ($page + 1) * $this->perPage >= \count($this->releases); + $readable = \array_values(\array_filter( + $items, + fn(ReleaseRecord $release): bool => !\in_array($release->tag, $this->unreadable, true), + )); + + yield new ReleasePage(\array_slice($readable, $skip), $last, \count($items) - \count($readable)); + + $skip = 0; + ++$page; + } while (!$last); + } +} diff --git a/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php b/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php new file mode 100644 index 0000000..252ae9d --- /dev/null +++ b/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php @@ -0,0 +1,52 @@ + */ + public array $records = []; + + /** @var int<0, max> */ + public int $saves = 0; + + public bool $failOnSave = false; + + public function load(RepositoryId $id): ?RepositoryRecord + { + return $this->records[(string) $id] ?? null; + } + + public function save(RepositoryRecord $record): void + { + $this->failOnSave and throw new \RuntimeException('Storage is read-only.'); + + ++$this->saves; + $this->records[(string) $record->id] = $record->persisted(); + } + + public function all(): iterable + { + yield from \array_values($this->records); + } + + public function remove(RepositoryId $id): void + { + unset($this->records[(string) $id]); + } + + public function clear(): void + { + $this->records = []; + } +} diff --git a/tests/Unit/Module/Registry/Stub/RecordingRegistry.php b/tests/Unit/Module/Registry/Stub/RecordingRegistry.php new file mode 100644 index 0000000..158028f --- /dev/null +++ b/tests/Unit/Module/Registry/Stub/RecordingRegistry.php @@ -0,0 +1,39 @@ + Software attached, as `[software, repository id]`. */ + public array $attached = []; + + /** @var list Releases forgotten, as `[repository id, tag]`. */ + public array $forgotten = []; + + public function releases(RepositoryId $id, ReleaseSource $source): \Generator + { + foreach ($source->pages() as $page) { + yield $page->releases; + } + } + + public function attach(RepositoryId $id, string $software): void + { + $this->attached[] = [$software, (string) $id]; + } + + public function forget(RepositoryId $id, string $tag): void + { + $this->forgotten[] = [(string) $id, $tag]; + } +} diff --git a/tests/Unit/Module/Repository/Internal/CachedGeneratorTest.php b/tests/Unit/Module/Repository/Internal/CachedGeneratorTest.php index 6609518..08b091f 100644 --- a/tests/Unit/Module/Repository/Internal/CachedGeneratorTest.php +++ b/tests/Unit/Module/Repository/Internal/CachedGeneratorTest.php @@ -9,6 +9,7 @@ use Testo\Assert; use Testo\Codecov\Covers; use Testo\Data\DataProvider; +use Testo\Expect; use Testo\Test; #[Covers(CachedGenerator::class)] @@ -270,6 +271,59 @@ public function handlesLargeDatasets(): void Assert::same($totalCount, 1000); } + /** + * Tests that loaded() returns only the items already produced, without pulling more from the generator. + */ + #[Test] + public function loadedReturnsOnlyItemsProducedSoFar(): void + { + $generator = $this->createGenerator(5); + $cachedGenerator = new CachedGenerator($generator); + + $i = 0; + foreach ($cachedGenerator as $item) { + if (++$i >= 2) { + break; + } + } + + $loaded = $cachedGenerator->loaded(); + + Assert::same($loaded, [0, 1]); + } + + /** + * Tests that isEmpty() returns false once the cache holds items, without touching the generator again. + */ + #[Test] + public function isEmptyReturnsFalseWhenCacheAlreadyPopulated(): void + { + $generator = $this->createGenerator(3); + $cachedGenerator = new CachedGenerator($generator); + + $cachedGenerator->first(); + $isEmpty = $cachedGenerator->isEmpty(); + + Assert::false($isEmpty); + } + + /** + * Tests that an exception raised while advancing the generator is re-thrown to the caller. + */ + #[Test] + public function throwingGeneratorRethrowsException(): void + { + $generator = (static function () { + yield 'a'; + throw new \RuntimeException('boom'); + })(); + $cachedGenerator = new CachedGenerator($generator); + + Expect::exception(\RuntimeException::class)->withMessage('boom'); + + \iterator_to_array($cachedGenerator); + } + /** * Creates a simple generator that yields consecutive integers. * diff --git a/tests/Unit/Module/Repository/Internal/CollectionTest.php b/tests/Unit/Module/Repository/Internal/CollectionTest.php index 3791cfa..e0909db 100644 --- a/tests/Unit/Module/Repository/Internal/CollectionTest.php +++ b/tests/Unit/Module/Repository/Internal/CollectionTest.php @@ -283,4 +283,122 @@ public function emptyWithLimit(): void $emptyCollection = $testCollection::create([]); Assert::true($emptyCollection->limit(5)->empty()); } + + #[Test] + public function createFromClosure(): void + { + $testCollection = new class([]) extends Collection {}; + + // A closure is invoked once and its result converted to a collection + $collection = $testCollection::create(static fn(): array => ['a', 'b', 'c']); + + Assert::equals($collection->toArray(), ['a', 'b', 'c']); + } + + #[Test] + public function createFromUnsupportedTypeThrows(): void + { + $testCollection = new class([]) extends Collection {}; + + try { + $testCollection::create(42); + Assert::fail('Expected InvalidArgumentException for an unsupported input type'); + } catch (\InvalidArgumentException $e) { + Assert::contains(\explode(' ', $e->getMessage()), 'int.'); + } + } + + #[Test] + public function firstWithoutFilterOnArray(): void + { + $testCollection = new class([]) extends Collection {}; + $items = [(object) ['n' => 10], (object) ['n' => 20], (object) ['n' => 30]]; + $collection = $testCollection::create($items); + + // No collection filters and no argument: returns the first stored item + Assert::same($collection->first(), $items[0]); + } + + #[Test] + public function firstWithoutFilterOnEmptyArrayReturnsNull(): void + { + $testCollection = new class([]) extends Collection {}; + $collection = $testCollection::create([]); + + Assert::null($collection->first()); + } + + #[Test] + public function firstWithoutFilterOnPaginator(): void + { + $testCollection = new class([]) extends Collection {}; + + $first = (object) ['label' => 'first']; + $pageLoader = static function () use ($first): \Generator { + yield [$first, (object) ['label' => 'second']]; + yield [(object) ['label' => 'third']]; + }; + $collection = $testCollection::create(Paginator::createFromGenerator($pageLoader(), null)); + + // No filters: served straight from the cached generator's first() + Assert::same($collection->first(), $first); + } + + #[Test] + public function firstReturnsNullWhenNothingMatches(): void + { + $testCollection = new class([]) extends Collection {}; + $collection = $testCollection::create([1, 2, 3]); + + // A filter that excludes everything exhausts the iterator and yields null + $first = $collection + ->filter(static fn($item) => $item > 100) + ->first(); + + Assert::null($first); + } + + #[Test] + public function countWithoutFiltersOnPaginator(): void + { + $testCollection = new class([]) extends Collection {}; + + $pageLoader = static function (): \Generator { + yield [1, 2, 3]; + yield [4, 5]; + }; + $collection = $testCollection::create(Paginator::createFromGenerator($pageLoader(), null)); + + // No filters and no limit: counted by the cached generator across all pages + Assert::count($collection, 5); + } + + #[Test] + public function emptyWithoutFiltersOnPaginator(): void + { + $testCollection = new class([]) extends Collection {}; + + $nonEmptyLoader = static function (): \Generator { + yield ['only']; + }; + $nonEmpty = $testCollection::create(Paginator::createFromGenerator($nonEmptyLoader(), null)); + Assert::false($nonEmpty->empty()); + + $emptyLoader = static function (): \Generator { + yield []; + }; + $empty = $testCollection::create(Paginator::createFromGenerator($emptyLoader(), null)); + Assert::true($empty->empty()); + } + + #[Test] + public function emptyWithFilter(): void + { + $testCollection = new class([]) extends Collection {}; + $collection = $testCollection::create([(object) ['n' => 1], (object) ['n' => 2], (object) ['n' => 3]]); + + // With filters, emptiness is decided by first(): all excluded -> empty + Assert::true($collection->filter(static fn($item) => $item->n > 100)->empty()); + Assert::false($collection->filter(static fn($item) => $item->n > 1)->empty()); + } } diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Api/ClientTest.php b/tests/Unit/Module/Repository/Internal/GitHub/Api/ClientTest.php index b27ad59..cd7b165 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/Api/ClientTest.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/Api/ClientTest.php @@ -5,12 +5,14 @@ namespace Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitHub\Api; use Internal\DLoad\Module\Config\Schema\GitHub; +use Internal\DLoad\Module\HttpClient\Internal\NyholmFactoryImpl; use Internal\DLoad\Module\Repository\Exception\AccessDeniedException; use Internal\DLoad\Module\Repository\Exception\ApiException; use Internal\DLoad\Module\Repository\Exception\AuthenticationException; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryNotFoundException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Client; +use Internal\DLoad\Service\Logger; use Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitHub\Stub\ClientExceptionStub; use Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitHub\Stub\ClientStub; use Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitHub\Stub\GitHubConfigStub; @@ -170,6 +172,27 @@ public function requestWithAuthTokenAddsAuthorizationHeader(): void Assert::equals($result, $response); } + #[Test] + public function tokenIsSentToGitHubHostsOnly(): void + { + // Asset URLs may come from a registry file on disk, so the token must not follow them anywhere + $http = new ClientStub(); + $client = new Client(new NyholmFactoryImpl(new Logger()), $http, GitHubConfigStub::withToken('secret')); + + $client->request('GET', 'https://api.github.com/repos/owner/repo/releases'); + $client->request('GET', 'https://objects.githubusercontent.com/asset'); + $client->request('GET', 'https://GitHub.com/owner/repo/releases/download/v1/rr.tar.gz'); + $client->request('GET', 'https://evil.example/github.com/asset'); + $client->request('GET', 'https://notgithub.com/asset'); + + Assert::same($http->sent[0]->getHeaderLine('authorization'), 'Bearer secret'); + Assert::same($http->sent[1]->getHeaderLine('authorization'), 'Bearer secret'); + Assert::same($http->sent[2]->getHeaderLine('authorization'), 'Bearer secret'); + Assert::same($http->sent[3]->getHeaderLine('authorization'), ''); + Assert::same($http->sent[4]->getHeaderLine('authorization'), ''); + Assert::same($http->sent[3]->getHeaderLine('accept'), 'application/vnd.github.v3+json'); + } + #[Test] public function requestWithoutTokenDoesNotAddAuthorizationHeader(): void { diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php b/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php index 6d13d98..9010405 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php @@ -6,6 +6,7 @@ use Internal\DLoad\Module\Repository\Exception\AccessDeniedException; use Internal\DLoad\Module\Repository\Exception\ApiException; +use Internal\DLoad\Module\Repository\Exception\AssetNotFoundException; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryNotFoundException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\ResponseValidator; @@ -126,20 +127,35 @@ public function serverErrorIsReportedAsTemporaryFailure(): void } #[Test] - public function repositoryIsResolvedFromAssetDownloadUrl(): void + public function missingAssetIsNotReportedAsMissingRepository(): void { $validator = new ResponseValidator(authenticated: false); $request = new Request('GET', 'https://github.com/owner/repo/releases/download/v1.0.0/asset.zip'); - $response = new ResponseStub(404, [], \json_encode(['message' => 'Not Found'])); + $response = new ResponseStub(404, [], 'Not Found'); try { $validator->validate($request, $response); - Assert::fail('RepositoryNotFoundException is expected.'); - } catch (RepositoryNotFoundException $e) { + Assert::fail('AssetNotFoundException is expected.'); + } catch (AssetNotFoundException $e) { + // The repository is still known, but the advice about tokens and addresses would mislead Assert::same($e->repository, 'owner/repo'); + Assert::string($e->getMessage())->contains('asset is no longer available'); + Assert::string($e->getMessage())->contains('release may have been deleted'); + Assert::string($e->getMessage())->notContains('GITHUB_TOKEN'); } } + #[Test] + public function forbiddenAssetIsStillAnAccessProblem(): void + { + $validator = new ResponseValidator(authenticated: false); + $request = new Request('GET', 'https://github.com/owner/repo/releases/download/v1.0.0/asset.zip'); + + Expect::exception(AccessDeniedException::class); + + $validator->validate($request, new ResponseStub(403, [], 'Forbidden')); + } + #[Test] public function transportFailureKeepsTheOriginalError(): void { @@ -172,6 +188,24 @@ public function longApiMessageIsTruncatedWithoutBreakingUtf8(): void } } + #[Test] + public function unrecognizedUriLeavesRepositoryUnknown(): void + { + $validator = new ResponseValidator(authenticated: false); + // A URI that is neither an API call nor a github.com link exposes no owner/repo pair + $request = new Request('GET', 'https://example.com/health'); + $response = new ResponseStub(404, [], \json_encode(['message' => 'Not Found'])); + + try { + $validator->validate($request, $response); + Assert::fail('RepositoryNotFoundException is expected.'); + } catch (RepositoryNotFoundException $e) { + Assert::null($e->repository); + // With no repository, the message falls back to the raw endpoint + Assert::string($e->getMessage())->contains('GET https://example.com/health'); + } + } + private static function releasesRequest(): RequestInterface { return new Request('GET', 'https://api.github.com/repos/owner/repo/releases?page=1'); diff --git a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php new file mode 100644 index 0000000..aa837eb --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php @@ -0,0 +1,292 @@ +getReleases(), false); + + Assert::same(\count($releases), 300); + Assert::same($client->requestedPages(), [1, 2, 3]); + } + + #[Test] + public function pagesAreLoadedOnlyWhenNeeded(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + + // Consume the whole first page, but nothing beyond it + $seen = 0; + foreach ($repository->getReleases() as $release) { + unset($release); + if (++$seen === 100) { + break; + } + } + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function destroyDoesNotLoadTheRemainingPages(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + $repository->getReleases()->first(); + + $repository->destroy(); + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function releasesAreRequestedAHundredPerPage(): void + { + $client = new PagedClientStub(pages: 1); + $repository = self::createRepository($client); + + \iterator_to_array($repository->getReleases(), false); + + Assert::same($client->requests, ['page=1&per_page=100']); + } + + #[Test] + public function secondRunIsServedFromTheRegistryWithoutRequests(): void + { + $storage = new InMemoryRegistryStorage(); + + $firstClient = new PagedClientStub(pages: 2); + $firstRun = self::names(self::createRepository($firstClient, self::registry($storage))); + + // A second run in a fresh process with the registry carried over + $secondClient = new PagedClientStub(pages: 2); + $secondRun = self::names(self::createRepository($secondClient, self::registry($storage))); + + Assert::same($firstClient->requestedPages(), [1, 2]); + Assert::same($secondClient->requestedPages(), []); + Assert::same($secondRun, $firstRun); + Assert::same(\count($secondRun), 200); + } + + #[Test] + public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void + { + $storage = new InMemoryRegistryStorage(); + + // The first run needs the first page only + $firstClient = new PagedClientStub(pages: 3); + foreach (self::createRepository($firstClient, self::registry($storage))->getReleases() as $release) { + unset($release); + break; + } + + // The second run needs everything: the stored page costs nothing, the rest is fetched + $secondClient = new PagedClientStub(pages: 3); + $all = self::names(self::createRepository($secondClient, self::registry($storage))); + + Assert::same($firstClient->requestedPages(), [1]); + Assert::same($secondClient->requestedPages(), [2, 3]); + Assert::same(\count($all), 300); + } + + #[Test] + public function draftReleasesAreNeitherServedNorStored(): void + { + $storage = new InMemoryRegistryStorage(); + + $client = new PagedClientStub(pages: 1, drafts: 2); + $names = self::names(self::createRepository($client, self::registry($storage))); + + Assert::same(\count($names), 100); + Assert::false(\in_array('draft-1', $names, true)); + + // The draft keeps its listing position as a hidden placeholder, so the stored count still + // maps onto the API paging and the second page is the next request, not the first one again + Assert::same($client->requestedPages(), [1, 2]); + $stored = $storage->load(new RepositoryId(GitHubRepository::TYPE, 'owner/repo')); + Assert::same($stored?->count(), 102); + Assert::true($stored?->releases()[0]->hidden); + Assert::same($stored?->releases()[0]->assets, []); + } + + #[Test] + public function unreadableReleasesAreCountedOnThePage(): void + { + $page = (new GitHubReleaseSource(self::api(new PagedClientStub(pages: 1, broken: 2))))->pages()->current(); + + Assert::same(\count($page->releases), 98); + Assert::same($page->skipped, 2); + } + + #[Test] + public function assetDigestReportedByTheApiIsStored(): void + { + $storage = new InMemoryRegistryStorage(); + + self::createRepository(new PagedClientStub(pages: 1), self::registry($storage))->getReleases()->first(); + + $asset = $storage->load(new RepositoryId(GitHubRepository::TYPE, 'owner/repo'))?->releases()[0]->assets[0]; + Assert::same($asset?->digest, 'sha256:' . \hash('sha256', 'v1.0.1')); + } + + #[Test] + public function tailIsLoadedFromInsideAPageWhenTheStoredCountIsNotPageAligned(): void + { + // A fresh record holds the first 50 releases: the tail starts in the middle of API page 1 + $storage = new InMemoryRegistryStorage(); + $storage->save((new RepositoryRecord(new RepositoryId(GitHubRepository::TYPE, 'owner/repo'), checkedAt: \time()))->withHead(\array_map( + static fn(int $i): ReleaseRecord => new ReleaseRecord(\sprintf('v1.0.%d', $i), \sprintf('v1.0.%d', $i)), + \range(1, 50), + ))); + + $client = new PagedClientStub(pages: 2); + $all = self::names(self::createRepository($client, self::registry($storage))); + + Assert::same($client->requestedPages(), [1, 2]); + Assert::same(\count($all), 200); + Assert::same(\count(\array_unique($all)), 200); + Assert::same($all[50], 'v1.0.51'); + } + + #[Test] + public function theSameCollectionIsReturnedOnEveryCall(): void + { + $repository = self::createRepository(new PagedClientStub(pages: 1)); + + Assert::same($repository->getReleases(), $repository->getReleases()); + } + + #[Test] + public function nameIsTheOwnerAndRepository(): void + { + Assert::same(self::createRepository(new PagedClientStub(pages: 1))->getName(), 'owner/repo'); + } + + #[Test] + public function aReleaseWithAnUnparsableTagIsSkipped(): void + { + $registry = new ScriptedRegistryStub([[self::record('not-a-version'), self::record('v1.0.1')]]); + $repository = self::createRepository(new PagedClientStub(pages: 1), $registry); + + $names = self::names($repository); + + Assert::same($names, ['v1.0.1']); + } + + #[Test] + public function aFailureOfTheFirstPageReachesTheCaller(): void + { + $registry = new ScriptedRegistryStub([new \RuntimeException('missing repository')]); + $repository = self::createRepository(new PagedClientStub(pages: 1), $registry); + + Expect::exception(\RuntimeException::class)->withMessage('missing repository'); + + self::names($repository); + } + + #[Test] + public function aRateLimitOnALaterPageReachesTheCaller(): void + { + $registry = new ScriptedRegistryStub([ + [self::record('v1.0.2')], + new RateLimitException('API rate limit exceeded'), + ]); + $repository = self::createRepository(new PagedClientStub(pages: 1), $registry); + + Expect::exception(RateLimitException::class)->withMessage('API rate limit exceeded'); + + self::names($repository); + } + + #[Test] + public function anOrdinaryFailureOfALaterPageStopsPaginationAndKeepsWhatLoaded(): void + { + $registry = new ScriptedRegistryStub([ + [self::record('v1.0.2')], + new \RuntimeException('transient network error'), + ]); + $repository = self::createRepository(new PagedClientStub(pages: 1), $registry); + + // The first page is enough to keep going, so the later failure only ends the listing + Assert::same(self::names($repository), ['v1.0.2']); + } + + /** + * @param non-empty-string $tag + */ + private static function record(string $tag): ReleaseRecord + { + return new ReleaseRecord($tag, $tag); + } + + private static function createRepository( + PagedClientStub $client, + VersionRegistry $registry = new PassThroughRegistry(), + ): GitHubRepository { + return new GitHubRepository(self::api($client), 'owner', 'repo', new Logger(), $registry); + } + + private static function api(PagedClientStub $client): RepositoryApi + { + $logger = new Logger(); + $httpFactory = new NyholmFactoryImpl($logger); + + return new RepositoryApi( + new Client($httpFactory, $client, new GitHubConfig()), + $httpFactory, + 'owner', + 'repo', + $logger, + ); + } + + private static function registry(InMemoryRegistryStorage $storage): StoredVersionRegistry + { + return new StoredVersionRegistry($storage, 600, new Logger()); + } + + /** + * @return list + */ + private static function names(GitHubRepository $repository): array + { + return \array_map( + static fn(ReleaseInterface $release): string => $release->getName(), + \iterator_to_array($repository->getReleases(), false), + ); + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Stub/ClientStub.php b/tests/Unit/Module/Repository/Internal/GitHub/Stub/ClientStub.php index 86270cc..d452ec8 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/Stub/ClientStub.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/Stub/ClientStub.php @@ -17,6 +17,9 @@ */ final class ClientStub implements ClientInterface { + /** @var list Requests in the order they were sent. */ + public array $sent = []; + /** * @var array */ @@ -43,6 +46,7 @@ public function withException(RequestInterface $request, ClientExceptionInterfac public function sendRequest(RequestInterface $request): ResponseInterface { + $this->sent[] = $request; $requestKey = $this->createRequestKey($request); if (isset($this->exceptions[$requestKey])) { diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php new file mode 100644 index 0000000..d1070a1 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php @@ -0,0 +1,116 @@ + + */ + public array $requests = []; + + /** + * @param int<1, max> $pages Number of pages the list is split into when 100 releases are requested per page. + * @param int<1, max> $releasesPerPage Number of releases on every such page. + * @param int<0, max> $drafts Draft releases listed on top, as the API shows them to the token holder. + * @param int<0, max> $broken Number of the newest published releases that cannot be decoded. + */ + public function __construct( + private readonly int $pages = 1, + private readonly int $releasesPerPage = 100, + private readonly int $drafts = 0, + private readonly int $broken = 0, + ) {} + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $query = $request->getUri()->getQuery(); + $this->requests[] = $query; + + $page = self::pageOf($query); + $perPage = self::perPageOf($query); + $all = $this->allReleases(); + + // Serve the slice the real API would serve for the requested page size + $releases = \array_slice($all, ($page - 1) * $perPage, $perPage); + if ($releases === []) { + return new ResponseStub(200, [], '[]'); + } + + $headers = $page * $perPage < \count($all) + ? ['link' => [\sprintf('; rel="next"', $page + 1)]] + : []; + + return new ResponseStub(200, $headers, \json_encode($releases)); + } + + /** + * Page number of every received request, in order. + * + * @return list + */ + public function requestedPages(): array + { + return \array_map(self::pageOf(...), $this->requests); + } + + private static function pageOf(string $query): int + { + \parse_str($query, $params); + + return (int) ($params['page'] ?? 1); + } + + private static function perPageOf(string $query): int + { + \parse_str($query, $params); + + return \max(1, (int) ($params['per_page'] ?? 30)); + } + + /** + * @return list> + */ + private function allReleases(): array + { + $releases = []; + $total = $this->pages * $this->releasesPerPage; + + for ($i = 1 - $this->drafts; $i <= $total; $i++) { + $tag = $i < 1 ? \sprintf('draft-%d', 1 - $i) : \sprintf('v1.0.%d', $i); + $releases[] = [ + 'name' => $tag, + // A number where a string belongs fails the strict constructor of the response object + 'tag_name' => $i >= 1 && $i <= $this->broken ? $i : $tag, + 'published_at' => $i < 1 ? null : '2024-01-01T00:00:00Z', + 'assets' => [[ + 'name' => 'rr-linux-amd64.tar.gz', + 'browser_download_url' => 'https://github.com/owner/repo/releases/download/' . $tag . '/rr-linux-amd64.tar.gz', + 'size' => 1024, + 'content_type' => 'application/gzip', + 'digest' => 'sha256:' . \hash('sha256', $tag), + ]], + 'prerelease' => false, + 'draft' => $i < 1, + ]; + } + + return $releases; + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Stub/ScriptedRegistryStub.php b/tests/Unit/Module/Repository/Internal/GitHub/Stub/ScriptedRegistryStub.php new file mode 100644 index 0000000..78559d3 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitHub/Stub/ScriptedRegistryStub.php @@ -0,0 +1,40 @@ +|\Throwable> $steps + */ + public function __construct( + private readonly array $steps, + ) {} + + public function releases(RepositoryId $id, ReleaseSource $source): \Generator + { + foreach ($this->steps as $step) { + $step instanceof \Throwable and throw $step; + + yield $step; + } + } + + public function attach(RepositoryId $id, string $software): void {} + + public function forget(RepositoryId $id, string $tag): void {} +} diff --git a/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php b/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php index fd8e2ea..48e0f5f 100644 --- a/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php +++ b/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php @@ -4,6 +4,7 @@ namespace Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitLab\Api; +use Internal\DLoad\Module\Repository\Exception\AssetNotFoundException; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryNotFoundException; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\ResponseValidator; @@ -16,6 +17,26 @@ #[Covers(ResponseValidator::class)] final class ResponseValidatorTest { + #[Test] + public function missingAssetIsNotReportedAsMissingProject(): void + { + $validator = new ResponseValidator(authenticated: false); + $request = new Request( + 'GET', + 'https://gitlab.com/api/v4/projects/group%2Fproject/releases/v1.0.0/downloads/asset.zip', + ); + $response = new ResponseStub(404, [], \json_encode(['message' => '404 Not Found'])); + + try { + $validator->validate($request, $response); + Assert::fail('AssetNotFoundException is expected.'); + } catch (AssetNotFoundException $e) { + Assert::same($e->repository, 'group/project'); + Assert::string($e->getMessage())->contains('asset is no longer available'); + Assert::string($e->getMessage())->notContains('GITLAB_TOKEN'); + } + } + #[Test] public function projectPathIsDecodedFromApiUrl(): void { @@ -33,6 +54,24 @@ public function projectPathIsDecodedFromApiUrl(): void } } + #[Test] + public function unrecognizedUriLeavesProjectUnknown(): void + { + $validator = new ResponseValidator(authenticated: false); + // A URI without a `/projects/{id}` segment exposes no project identifier + $request = new Request('GET', 'https://gitlab.com/api/v4/version'); + $response = new ResponseStub(404, [], \json_encode(['message' => '404 Not Found'])); + + try { + $validator->validate($request, $response); + Assert::fail('RepositoryNotFoundException is expected.'); + } catch (RepositoryNotFoundException $e) { + Assert::null($e->repository); + // With no project, the message falls back to the raw endpoint + Assert::string($e->getMessage())->contains('GET https://gitlab.com/api/v4/version'); + } + } + #[Test] public function tooManyRequestsIsReportedAsRateLimit(): void { diff --git a/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php b/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php index 96b9527..8a25c34 100644 --- a/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php @@ -6,6 +6,7 @@ use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitLab as GitLabConfig; +use Internal\DLoad\Module\Registry\Internal\PassThroughRegistry; use Internal\DLoad\Module\Repository\Internal\GitLab\Factory; use Internal\DLoad\Service\Logger; use Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitLab\Stub\HttpFactoryStub; @@ -69,6 +70,11 @@ public function createDerivesTheProjectPathFromTheUri(string $uri, string $expec #[BeforeTest] protected function prepare(): void { - $this->factory = new Factory(new HttpFactoryStub(), new GitLabConfig(), new Logger()); + $this->factory = new Factory( + new HttpFactoryStub(), + new GitLabConfig(), + new Logger(), + new PassThroughRegistry(), + ); } } diff --git a/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php new file mode 100644 index 0000000..8394045 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php @@ -0,0 +1,240 @@ +getReleases(), false); + + Assert::same(\count($releases), 300); + Assert::same($client->requestedPages(), [1, 2, 3]); + } + + #[Test] + public function pagesAreLoadedOnlyWhenNeeded(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + + $seen = 0; + foreach ($repository->getReleases() as $release) { + unset($release); + if (++$seen === 100) { + break; + } + } + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function destroyDoesNotLoadTheRemainingPages(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + $repository->getReleases()->first(); + + $repository->destroy(); + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function releasesAreRequestedAHundredPerPage(): void + { + $client = new PagedClientStub(pages: 1); + $repository = self::createRepository($client); + + \iterator_to_array($repository->getReleases(), false); + + Assert::same($client->requests, ['page=1&per_page=100']); + } + + #[Test] + public function secondRunIsServedFromTheRegistryWithoutRequests(): void + { + $storage = new InMemoryRegistryStorage(); + + $firstClient = new PagedClientStub(pages: 2); + \iterator_to_array(self::createRepository($firstClient, self::registry($storage))->getReleases(), false); + + $secondClient = new PagedClientStub(pages: 2); + $secondRun = \iterator_to_array(self::createRepository($secondClient, self::registry($storage))->getReleases(), false); + + Assert::same($firstClient->requestedPages(), [1, 2]); + Assert::same($secondClient->requestedPages(), []); + Assert::same(\count($secondRun), 200); + } + + #[Test] + public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void + { + $storage = new InMemoryRegistryStorage(); + + $firstClient = new PagedClientStub(pages: 3); + foreach (self::createRepository($firstClient, self::registry($storage))->getReleases() as $release) { + unset($release); + break; + } + + $secondClient = new PagedClientStub(pages: 3); + $all = self::names(self::createRepository($secondClient, self::registry($storage))); + + Assert::same($firstClient->requestedPages(), [1]); + Assert::same($secondClient->requestedPages(), [2, 3]); + Assert::same(\count($all), 300); + } + + #[Test] + public function tailIsLoadedFromInsideAPageWhenTheStoredCountIsNotPageAligned(): void + { + $storage = new InMemoryRegistryStorage(); + $storage->save((new RepositoryRecord(new RepositoryId(GitLabRepository::TYPE, 'group/project'), checkedAt: \time()))->withHead(\array_map( + static fn(int $i): ReleaseRecord => new ReleaseRecord(\sprintf('v1.0.%d', $i), \sprintf('v1.0.%d', $i)), + \range(1, 50), + ))); + + $client = new PagedClientStub(pages: 2); + $all = self::names(self::createRepository($client, self::registry($storage))); + + Assert::same($client->requestedPages(), [1, 2]); + Assert::same(\count($all), 200); + Assert::same(\count(\array_unique($all)), 200); + Assert::same($all[50], 'v1.0.51'); + } + + #[Test] + public function theSameCollectionIsReturnedOnEveryCall(): void + { + $repository = self::createRepository(new PagedClientStub(pages: 1)); + + Assert::same($repository->getReleases(), $repository->getReleases()); + } + + #[Test] + public function nameIsTheGroupAndProject(): void + { + Assert::same(self::createRepository(new PagedClientStub(pages: 1))->getName(), 'group/project'); + } + + #[Test] + public function aReleaseWithAnUnparsableTagIsSkipped(): void + { + $registry = new ScriptedRegistryStub([[self::record('not-a-version'), self::record('v1.0.1')]]); + $repository = self::createRepository(new PagedClientStub(pages: 1), $registry); + + $names = self::names($repository); + + Assert::same($names, ['v1.0.1']); + } + + #[Test] + public function aFailureOfTheFirstPageReachesTheCaller(): void + { + $registry = new ScriptedRegistryStub([new \RuntimeException('missing project')]); + $repository = self::createRepository(new PagedClientStub(pages: 1), $registry); + + Expect::exception(\RuntimeException::class)->withMessage('missing project'); + + self::names($repository); + } + + #[Test] + public function aRateLimitOnALaterPageReachesTheCaller(): void + { + $registry = new ScriptedRegistryStub([ + [self::record('v1.0.2')], + new RateLimitException('API rate limit exceeded'), + ]); + $repository = self::createRepository(new PagedClientStub(pages: 1), $registry); + + Expect::exception(RateLimitException::class)->withMessage('API rate limit exceeded'); + + self::names($repository); + } + + #[Test] + public function anOrdinaryFailureOfALaterPageStopsPaginationAndKeepsWhatLoaded(): void + { + $registry = new ScriptedRegistryStub([ + [self::record('v1.0.2')], + new \RuntimeException('transient network error'), + ]); + $repository = self::createRepository(new PagedClientStub(pages: 1), $registry); + + // The first page is enough to keep going, so the later failure only ends the listing + Assert::same(self::names($repository), ['v1.0.2']); + } + + /** + * @param non-empty-string $tag + */ + private static function record(string $tag): ReleaseRecord + { + return new ReleaseRecord($tag, $tag); + } + + private static function createRepository( + PagedClientStub $client, + VersionRegistry $registry = new PassThroughRegistry(), + ): GitLabRepository { + $logger = new Logger(); + $httpFactory = new NyholmFactoryImpl($logger); + $api = new RepositoryApi( + new Client($httpFactory, $client, new GitLabConfig()), + $httpFactory, + 'group/project', + ); + + return new GitLabRepository($api, 'group/project', $logger, $registry); + } + + private static function registry(InMemoryRegistryStorage $storage): StoredVersionRegistry + { + return new StoredVersionRegistry($storage, 600, new Logger()); + } + + /** + * @return list + */ + private static function names(GitLabRepository $repository): array + { + return \array_map( + static fn(ReleaseInterface $release): string => $release->getName(), + \iterator_to_array($repository->getReleases(), false), + ); + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php new file mode 100644 index 0000000..eca2b51 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php @@ -0,0 +1,106 @@ + + */ + public array $requests = []; + + /** + * @param int<1, max> $pages Number of pages the list is split into when 100 releases are requested per page. + * @param int<1, max> $releasesPerPage Number of releases on every such page. + */ + public function __construct( + private readonly int $pages = 1, + private readonly int $releasesPerPage = 100, + ) {} + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $query = $request->getUri()->getQuery(); + $this->requests[] = $query; + + $page = self::pageOf($query); + $perPage = self::perPageOf($query); + $all = $this->allReleases(); + + // Serve the slice the real API would serve for the requested page size + $releases = \array_slice($all, ($page - 1) * $perPage, $perPage); + if ($releases === []) { + return new ResponseStub(200, [], '[]'); + } + + $headers = $page * $perPage < \count($all) + ? ['link' => [\sprintf('; rel="next"', $page + 1)]] + : []; + + return new ResponseStub(200, $headers, \json_encode($releases)); + } + + /** + * Page number of every received request, in order. + * + * @return list + */ + public function requestedPages(): array + { + return \array_map(self::pageOf(...), $this->requests); + } + + private static function pageOf(string $query): int + { + \parse_str($query, $params); + + return (int) ($params['page'] ?? 1); + } + + private static function perPageOf(string $query): int + { + \parse_str($query, $params); + + return \max(1, (int) ($params['per_page'] ?? 20)); + } + + /** + * @return list> + */ + private function allReleases(): array + { + $releases = []; + $total = $this->pages * $this->releasesPerPage; + + for ($i = 1; $i <= $total; $i++) { + $tag = \sprintf('v1.0.%d', $i); + $releases[] = [ + 'name' => $tag, + 'tag_name' => $tag, + 'description' => 'Release ' . $tag, + 'created_at' => '2024-01-01T00:00:00Z', + 'released_at' => '2024-01-01T00:00:00Z', + 'assets' => ['links' => []], + 'upcoming_release' => false, + ]; + } + + return $releases; + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitLab/Stub/ScriptedRegistryStub.php b/tests/Unit/Module/Repository/Internal/GitLab/Stub/ScriptedRegistryStub.php new file mode 100644 index 0000000..09daf45 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitLab/Stub/ScriptedRegistryStub.php @@ -0,0 +1,40 @@ +|\Throwable> $steps + */ + public function __construct( + private readonly array $steps, + ) {} + + public function releases(RepositoryId $id, ReleaseSource $source): \Generator + { + foreach ($this->steps as $step) { + $step instanceof \Throwable and throw $step; + + yield $step; + } + } + + public function attach(RepositoryId $id, string $software): void {} + + public function forget(RepositoryId $id, string $tag): void {} +} diff --git a/tests/Unit/Module/Repository/Internal/ResponseValidatorTest.php b/tests/Unit/Module/Repository/Internal/ResponseValidatorTest.php new file mode 100644 index 0000000..63a6358 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/ResponseValidatorTest.php @@ -0,0 +1,168 @@ + 'Bad credentials']), 'Unauthorized'); + + try { + $validator->validate(self::githubRequest(), $response); + Assert::fail('AuthenticationException is expected.'); + } catch (AuthenticationException $e) { + Assert::same($e->repository, 'owner/repo'); + Assert::string($e->getMessage()) + ->contains('rejected the credentials') + ->contains('Bad credentials') + ->contains('invalid, expired or revoked') + ->contains('GITHUB_TOKEN'); + } + } + + #[Test] + public function unauthorizedAnonymousResponseSuggestsConfiguringAToken(): void + { + $validator = new GitHubValidator(authenticated: false); + $response = new ResponseStub(401, [], \json_encode(['message' => 'Requires authentication']), 'Unauthorized'); + + try { + $validator->validate(self::githubRequest(), $response); + Assert::fail('AuthenticationException is expected.'); + } catch (AuthenticationException $e) { + Assert::same($e->repository, 'owner/repo'); + Assert::string($e->getMessage()) + ->contains('rejected the credentials') + ->contains('Requires authentication') + ->contains('No API token is configured') + ->contains('the request was anonymous') + ->contains('GITHUB_TOKEN') + ->notContains('invalid, expired or revoked'); + } + } + + #[Test] + public function notFoundWithTokenExplainsPrivateRepositoryAccess(): void + { + $validator = new GitHubValidator(authenticated: true); + $response = new ResponseStub(404, [], \json_encode(['message' => 'Not Found'])); + + try { + $validator->validate(self::githubRequest(), $response); + Assert::fail('RepositoryNotFoundException is expected.'); + } catch (RepositoryNotFoundException $e) { + Assert::string($e->getMessage()) + ->contains('repository `owner/repo`') + ->contains('a 404 is also returned instead of 403') + ->contains('GITHUB_TOKEN'); + } + } + + #[Test] + public function anonymousRateLimitOmitsLimitCountWhenNoneIsKnown(): void + { + // GitLab exposes no numeric rate limit, so neither a header nor a fallback count is available + $validator = new GitLabValidator(authenticated: false); + $request = new Request('GET', 'https://gitlab.com/api/v4/projects/group%2Fproject/releases'); + $response = new ResponseStub(429, [], ''); + + try { + $validator->validate($request, $response); + Assert::fail('RateLimitException is expected.'); + } catch (RateLimitException $e) { + Assert::string($e->getMessage()) + ->contains('GitLab API rate limit exceeded') + ->contains('No API token is configured') + ->contains('GITLAB_TOKEN') + ->notContains('requests per hour'); + } + } + + #[Test] + #[DataSet([30, 'sec'], 'reset within a minute is counted in seconds')] + #[DataSet([-10, 'a moment'], 'a past reset time reads as a moment')] + public function rateLimitResetCountdownIsHumanized(int $offsetSeconds, string $expected): void + { + $validator = new GitHubValidator(authenticated: true); + $response = new ResponseStub( + 403, + [ + 'x-ratelimit-remaining' => ['0'], + 'x-ratelimit-reset' => [(string) (\time() + $offsetSeconds)], + ], + \json_encode(['message' => 'API rate limit exceeded']), + ); + + try { + $validator->validate(self::githubRequest(), $response); + Assert::fail('RateLimitException is expected.'); + } catch (RateLimitException $e) { + Assert::string($e->getMessage()) + ->contains('The limit resets at') + ->contains($expected) + ->notContains(' min '); + } + } + + #[Test] + #[DataSet(['', false, ''], 'empty body carries no API message')] + #[DataSet(['123', false, ''], 'a JSON scalar is not treated as a message')] + #[DataSet( + ['{"error":["Access forbidden.","Come back later."]}', true, 'Access forbidden. Come back later.'], + 'a list of error strings is joined', + )] + #[DataSet( + ['["Legacy rate limit message.","https://docs"]', true, 'Legacy rate limit message.'], + 'the first item of a plain list is used', + )] + public function apiMessageIsExtractedFromBodyShapes(string $body, bool $hasMessage, string $expected): void + { + $validator = new GitHubValidator(authenticated: false); + $response = new ResponseStub(503, [], $body, 'Service Unavailable'); + + try { + $validator->validate(self::githubRequest(), $response); + Assert::fail('ApiException is expected.'); + } catch (ApiException $e) { + $message = $e->getMessage(); + + if ($hasMessage) { + Assert::string($message)->contains($expected); + return; + } + + // A null message leaves nothing between the endpoint and the "Try again later." suffix + Assert::string($message)->contains('). Try again later.'); + $body === '' or Assert::string($message)->notContains($body); + } + } + + private static function githubRequest(): RequestInterface + { + return new Request('GET', 'https://api.github.com/repos/owner/repo/releases?page=1'); + } +}