From 42c4440edaaab6aa5171140898725473234895a6 Mon Sep 17 00:00:00 2001 From: Joel Wurtz Date: Tue, 6 Oct 2026 13:44:56 +0200 Subject: [PATCH] Push every build cache and back the registry one with the GitHub Actions cache --- CHANGELOG.md | 10 +++++ doc/tasks.md | 35 +++++++++++++++- phpunit.xml.dist | 5 +++ src/Service/Builder/BuildBuilder.php | 18 ++++++++ src/tasks.php | 51 ++++++++++++++-------- tests/Integration/DockerPushTest.php | 56 ++++++++++++++++++++++--- tests/Unit/Builder/BuildBuilderTest.php | 48 +++++++++++++++++++++ tests/fixtures/push-project/castor.php | 10 +++++ 8 files changed, 210 insertions(+), 23 deletions(-) create mode 100644 tests/Unit/Builder/BuildBuilderTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index b065938..db44a93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## Unreleased + +### Added + +* On GitHub Actions, every build also reads and writes the [GitHub Actions cache](tasks.md#on-github-actions), and `castor docker:push` pushes to it. + +### Changed + +* `castor docker:push` pushes to every cache a service lists in its `cache_from`, not only the first one. + ## 0.8.1 - 2026-09-29 ### Added diff --git a/doc/tasks.md b/doc/tasks.md index de5baf9..bb0429a 100644 --- a/doc/tasks.md +++ b/doc/tasks.md @@ -246,7 +246,9 @@ built, and `docker buildx bake` reads the compose file itself, so what it builds is exactly what `castor docker:build` builds. Each service lands in one repository, holding its cache under the `cache` tag -and its image under `latest` — `--tag` publishes it under another name. The +and its image under `latest` — `--tag` publishes it under another name. A +service listing several caches in its `cache_from` pushes to all of them, and +its image to each registry among them. The image carries `org.opencontainers.image.source`, which is what [attaches the package to your repository](#publishing-to-ghcr-io). @@ -258,6 +260,37 @@ castor docker:push --tag "$(git rev-parse --short HEAD)" castor docker:push --dry-run ``` +#### On GitHub Actions + +When `GITHUB_ACTIONS` is set, every registry cache gets a +[GitHub Actions cache](https://docs.docker.com/build/cache/backends/gha/) next +to it, scoped by the cache name: every build reads both and writes the GitHub +one, `castor docker:push` writes both. BuildKit sometimes misses layers of a +registry cache without a word, the second one makes up for it. + +What a pull request writes stays with it: its next runs reuse it, the default +branch never sees it. The layers it shares with the default branch are not +stored twice. + +Buildx reaches that cache with variables Actions only hands to JavaScript +actions, so every job that builds has to expose them first, reading the cache +needing them as much as writing it: + +```yaml +- uses: actions/github-script@v9 + with: + script: | + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN) + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL) + core.exportVariable('ACTIONS_CACHE_SERVICE_V2', process.env.ACTIONS_CACHE_SERVICE_V2 || 'true') + +- run: castor docker:build +``` + +Without them buildx skips the cache silently, and the build only uses the +registry. A matrix building several variants of the same service has to give +each one its own cache name, or the jobs overwrite each other's cache. + #### Publishing to ghcr.io GitHub attaches a package to a repository in two cases only: a push from a diff --git a/phpunit.xml.dist b/phpunit.xml.dist index ab06bc6..7123d00 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -6,6 +6,11 @@ cacheDirectory=".phpunit.cache" failOnRisky="true" > + + + + + tests/Unit diff --git a/src/Service/Builder/BuildBuilder.php b/src/Service/Builder/BuildBuilder.php index 68c31fd..38b4f2b 100644 --- a/src/Service/Builder/BuildBuilder.php +++ b/src/Service/Builder/BuildBuilder.php @@ -26,6 +26,8 @@ final class BuildBuilder private array $additionalContexts = []; /** @var array */ private array $cacheFrom = []; + /** @var array */ + private array $cacheTo = []; public function __construct(private readonly ServiceBuilder $serviceBuilder) {} @@ -38,6 +40,7 @@ public function clone(ServiceBuilder $serviceBuilder): self $new->args = $this->args; $new->additionalContexts = $this->additionalContexts; $new->cacheFrom = $this->cacheFrom; + $new->cacheTo = $this->cacheTo; return $new; } @@ -98,6 +101,7 @@ public function cacheFrom(string $image): self public function noCacheFrom(): self { $this->cacheFrom = []; + $this->cacheTo = []; return $this; } @@ -105,6 +109,16 @@ public function noCacheFrom(): self public function withRegistryCache(string $image): self { $this->cacheFrom = ['type=registry,ref=${REGISTRY:-}/' . $image . ':cache']; + $this->cacheTo = []; + + // BuildKit silently misses layers of a registry cache on CI + // (jolicode/docker-starter#430), the GitHub Actions one backs it up. + // Every build writes it: GitHub keeps what a pull request writes to + // that pull request, whose next runs reuse it. + if ('true' === getenv('GITHUB_ACTIONS')) { + $this->cacheFrom[] = 'type=gha,scope=' . $image; + $this->cacheTo = ['type=gha,scope=' . $image . ',mode=max,ignore-error=true']; + } return $this; } @@ -145,6 +159,10 @@ public function toArray(): array $build['cache_from'] = $this->cacheFrom; } + if (!empty($this->cacheTo)) { + $build['cache_to'] = $this->cacheTo; + } + return $build; } } diff --git a/src/tasks.php b/src/tasks.php index 2003225..bb6af59 100644 --- a/src/tasks.php +++ b/src/tasks.php @@ -697,14 +697,14 @@ function push( } // Only a service declaring a cache_from has somewhere to push its build - // cache back to. + // cache back to, and it goes back to every one of them. $targets = []; foreach (get_services() as $service => $config) { - $cacheFrom = $config['build']['cache_from'][0] ?? null; + $cacheFrom = $config['build']['cache_from'] ?? []; - if (null !== $cacheFrom) { - $targets[$service] = normalize_cache_entry($cacheFrom); + if ($cacheFrom) { + $targets[$service] = array_map(normalize_cache_entry(...), $cacheFrom); } } @@ -730,26 +730,43 @@ function push( // apply: bake sees every service that has a "build". $command = ['docker', 'buildx', 'bake', '-f', $c->workingDirectory . '/compose.yaml']; - foreach ($targets as $service => $cacheTo) { - $command[] = '--set'; - $command[] = \sprintf('%s.cache-to=%s,mode=max', $service, $cacheTo); + foreach ($targets as $service => $cacheEntries) { + $images = []; - $cacheRef = get_cache_reference($cacheTo); + foreach ($cacheEntries as $cacheTo) { + $cacheRef = get_cache_reference($cacheTo); - // "type=gha" or "type=local" names no repository to publish an image - // to, so that service pushes its cache and nothing else. - if (null === $cacheRef) { - continue; + $command[] = '--set'; + // A GitHub Actions cache only backs the registry one up: its + // outage must not keep the images from being pushed. + $command[] = \sprintf('%s.cache-to=%s,mode=max%s', $service, $cacheTo, str_contains($cacheTo, 'type=gha') ? ',ignore-error=true' : ''); + + // "type=gha" or "type=local" names no repository to publish an + // image to. + if (null === $cacheRef) { + continue; + } + + $image = get_image_reference($cacheRef, $tag); + + if ($image === $cacheRef) { + throw new \RuntimeException(\sprintf('Pushing "%s" under the tag "%s" would overwrite the build cache of "%s". Pick another --tag.', $image, $tag, $service)); + } + + $images[] = $image; } - $image = get_image_reference($cacheRef, $tag); + // A service whose caches all live outside of a registry pushes its + // cache and nothing else. + if (!$images) { + continue; + } - if ($image === $cacheRef) { - throw new \RuntimeException(\sprintf('Pushing "%s" under the tag "%s" would overwrite the build cache of "%s". Pick another --tag.', $image, $tag, $service)); + foreach ($images as $image) { + $command[] = '--set'; + $command[] = \sprintf('%s.tags=%s', $service, $image); } - $command[] = '--set'; - $command[] = \sprintf('%s.tags=%s', $service, $image); // Per target, rather than a global "--push": the services whose cache // is not a registry one must not be pushed anywhere. $command[] = '--set'; diff --git a/tests/Integration/DockerPushTest.php b/tests/Integration/DockerPushTest.php index 3d836cb..63a4958 100644 --- a/tests/Integration/DockerPushTest.php +++ b/tests/Integration/DockerPushTest.php @@ -12,9 +12,9 @@ * Nothing but a real registry says whether the cache export "docker:push" asks * bake for actually happened, so this starts a throwaway one and pushes to it. * - * tests/fixtures/push-project declares the three cases the task has to tell - * apart: a full "type=registry,ref=..." cache, the bare reference compose also - * accepts, and a service that builds with no cache at all. + * tests/fixtures/push-project declares the cases the task has to tell apart: a + * full "type=registry,ref=..." cache, the bare reference compose also accepts, + * a cache in two registries, and a service that builds with no cache at all. * * Requires a castor binary (CASTOR_BINARY env var, or "castor" in PATH) and a * running Docker daemon. @@ -47,7 +47,7 @@ public function testOnlyTheServicesWithACacheAreBuilt(): void $targets = $plan['group']['default']['targets']; sort($targets); - static::assertSame(['cached', 'shorthand'], $targets, '"uncached" builds but has no cache to push, it has no business being built here.'); + static::assertSame(['cached', 'mirrored', 'shorthand'], $targets, '"uncached" builds but has no cache to push, it has no business being built here.'); // The bare reference of "shorthand" is what compose accepts and buildx // does not: unspelled, this is where the whole push would fail. @@ -57,6 +57,52 @@ public function testOnlyTheServicesWithACacheAreBuilt(): void ); } + public function testEveryCacheOfAServiceIsPushed(): void + { + $castor = $this->castorOrSkip(); + + $push = $this->castor($castor, ['docker:push', '--dry-run'], ['CASTOR_DOCKER_TEST_REGISTRY' => 'registry.invalid/ns']); + + static::assertTrue($push->isSuccessful(), "castor docker:push --dry-run failed:\n" . $push->getOutput() . $push->getErrorOutput()); + + $target = json_decode($push->getOutput(), true)['target']['mirrored']; + + static::assertSame([ + ['mode' => 'max', 'ref' => 'registry.invalid/ns/mirrored:cache', 'type' => 'registry'], + ['mode' => 'max', 'ref' => 'registry.invalid/ns/mirror:cache', 'type' => 'registry'], + ], $target['cache-to']); + static::assertSame(['registry.invalid/ns/mirrored:latest', 'registry.invalid/ns/mirror:latest'], $target['tags']); + } + + /** + * The GitHub Actions cache comes on top of the registry one, which the + * images still go to. The cache_to every build writes the GitHub one with + * gives way to the one exporting both. + */ + public function testOnGithubActionsTheCacheIsAlsoPushedToGithub(): void + { + $castor = $this->castorOrSkip(); + + $push = $this->castor($castor, ['docker:push', '--dry-run'], [ + 'CASTOR_DOCKER_TEST_REGISTRY' => 'registry.invalid/ns', + 'GITHUB_ACTIONS' => 'true', + ]); + + static::assertTrue($push->isSuccessful(), "castor docker:push --dry-run failed:\n" . $push->getOutput() . $push->getErrorOutput()); + + $target = json_decode($push->getOutput(), true)['target']['cached']; + + static::assertSame([ + ['ref' => 'registry.invalid/ns/cached:cache', 'type' => 'registry'], + ['scope' => 'cached', 'type' => 'gha'], + ], $target['cache-from']); + static::assertSame([ + ['mode' => 'max', 'ref' => 'registry.invalid/ns/cached:cache', 'type' => 'registry'], + ['ignore-error' => 'true', 'mode' => 'max', 'scope' => 'cached', 'type' => 'gha'], + ], $target['cache-to']); + static::assertSame(['registry.invalid/ns/cached:latest'], $target['tags']); + } + /** * The cache alone carries no label, so a registry reading one to attach a * package to a repository — ghcr.io — would file every push under nobody. @@ -147,7 +193,7 @@ public function testTheBuildCacheLandsInTheRegistry(): void unlink($config); } - foreach (['cached', 'shorthand'] as $service) { + foreach (['cached', 'shorthand', 'mirrored', 'mirror'] as $service) { $tags = json_decode((string) file_get_contents("{$api}{$namespace}/{$service}/tags/list"), true); static::assertContains('cache', $tags['tags'] ?? [], "docker:push pushed no cache for \"{$service}\"."); diff --git a/tests/Unit/Builder/BuildBuilderTest.php b/tests/Unit/Builder/BuildBuilderTest.php new file mode 100644 index 0000000..fa282db --- /dev/null +++ b/tests/Unit/Builder/BuildBuilderTest.php @@ -0,0 +1,48 @@ + ['type=registry,ref=${REGISTRY:-}/app:cache']], + $this->buildWithRegistryCache(), + ); + } + + public function testOnGithubActionsEveryBuildReadsAndWritesTheGithubCache(): void + { + putenv('GITHUB_ACTIONS=true'); + + static::assertSame( + [ + 'cache_from' => ['type=registry,ref=${REGISTRY:-}/app:cache', 'type=gha,scope=app'], + 'cache_to' => ['type=gha,scope=app,mode=max,ignore-error=true'], + ], + $this->buildWithRegistryCache(), + ); + } + + /** + * @return array + */ + private function buildWithRegistryCache(): array + { + $builder = new ComposeBuilder(); + $builder->service('app')->build()->withRegistryCache('app'); + + return $builder->toArray()['services']['app']['build']; + } +} diff --git a/tests/fixtures/push-project/castor.php b/tests/fixtures/push-project/castor.php index 8509f98..0b263b0 100644 --- a/tests/fixtures/push-project/castor.php +++ b/tests/fixtures/push-project/castor.php @@ -10,6 +10,7 @@ * "type=registry,ref=..." entry; * - "shorthand", whose cache_from is the bare image reference compose also * accepts, and buildx does not accept in a cache-to; + * - "mirrored", whose cache lives in two registries at once; * - "uncached", which builds but has nowhere to push a cache, and must stay * out of what bake is asked to build. * @@ -62,6 +63,15 @@ function add_builds(ComposeBuilder $builder, Context $context): void ->end() ->command(['sleep', 'infinity']) ->end() + ->service('mirrored') + ->build() + ->context(__DIR__ . '/app') + ->arg('greeting', 'mirrored') + ->withRegistryCache('mirrored') + ->cacheFrom("type=registry,ref={$registry}/mirror:cache") + ->end() + ->command(['sleep', 'infinity']) + ->end() ->service('uncached') ->build() ->context(__DIR__ . '/app')