Share cache between e2e tests

.circleci/config.yml

@@ -9,12 +9,17 @@ jobs:
        PLZ_ARGS: "--profile ci --profile alpine --exclude no-musl"
      steps:
        - checkout
+       - run:
+           name: E2E cache key
+           command: ./tools/misc/e2e_cache_key.sh > /tmp/e2e-cache-key
        - restore_cache:
            key: go-mod-alpine-v9-{{ checksum "go.mod" }}
        - restore_cache:
            key: go-alpine-main-v9-{{ checksum "third_party/go/BUILD" }}
        - restore_cache:
            key: python-alpine-main-v3-{{ checksum "third_party/python/BUILD" }}
+       - restore_cache:
+           key: e2e-alpine-v1-{{ checksum "/tmp/e2e-cache-key" }}
        - run:
            name: Bootstrap & Build
            command: ./bootstrap.sh --test_results_file plz-out/results/please/test_results.xml
@@ -39,6 +44,9 @@ jobs:
        - save_cache:
            key: python-alpine-main-v3-{{ checksum "third_party/python/BUILD" }}
            paths: [ ".plz-cache/third_party/python" ]
+       - save_cache:
+           key: e2e-alpine-v1-{{ checksum "/tmp/e2e-cache-key" }}
+           paths: [ "plz-out/e2e_cache" ]
 
    build-linux:
      working_directory: ~/please
@@ -49,12 +57,17 @@ jobs:
        PLZ_ARGS: "--profile ci"
      steps:
        - checkout
+       - run:
+           name: E2E cache key
+           command: ./tools/misc/e2e_cache_key.sh > /tmp/e2e-cache-key
        - restore_cache:
            key: go-mod-linux-v9-{{ checksum "go.mod" }}
        - restore_cache:
            key: go-linux-main-v9-{{ checksum "third_party/go/BUILD" }}
        - restore_cache:
            key: python-linux-main-v3-{{ checksum "third_party/python/BUILD" }}
+       - restore_cache:
+           key: e2e-linux-v1-{{ checksum "/tmp/e2e-cache-key" }}
        - run:
            name: Bootstrap & Build
            command: ./bootstrap.sh --test_results_file plz-out/results/please/test_results.xml
@@ -87,6 +100,9 @@ jobs:
        - save_cache:
            key: python-linux-main-v3-{{ checksum "third_party/python/BUILD" }}
            paths: [ ".plz-cache/third_party/python" ]
+       - save_cache:
+           key: e2e-linux-v1-{{ checksum "/tmp/e2e-cache-key" }}
+           paths: [ "plz-out/e2e_cache" ]
 
    build-linux-alt:
      working_directory: ~/please
@@ -98,12 +114,17 @@ jobs:
        PLZ_COVER: "cover --nocoverage_report"
      steps:
        - checkout
+       - run:
+           name: E2E cache key
+           command: ./tools/misc/e2e_cache_key.sh > /tmp/e2e-cache-key
        - restore_cache:
            key: go-mod-linux-alt-v7-{{ checksum "go.mod" }}
        - restore_cache:
            key: go-linux-alt-v7-{{ checksum "third_party/go/BUILD" }}
        - restore_cache:
            key: python-linux-alt-v3-{{ checksum "third_party/python/BUILD" }}
+       - restore_cache:
+           key: e2e-linux-alt-v1-{{ checksum "/tmp/e2e-cache-key" }}
        - run:
            name: Bootstrap & Build
            command: ./bootstrap.sh --test_results_file plz-out/results/please/test_results.xml
@@ -121,6 +142,9 @@ jobs:
        - save_cache:
            key: python-linux-alt-v3-{{ checksum "third_party/python/BUILD" }}
            paths: [ ".plz-cache/third_party/python" ]
+       - save_cache:
+           key: e2e-linux-alt-v1-{{ checksum "/tmp/e2e-cache-key" }}
+           paths: [ "plz-out/e2e_cache" ]
    build-darwin-amd64:
      working_directory: ~/please
      macos:
@@ -211,12 +235,19 @@ jobs:
         PLZ_ARGS: "--profile ci --exclude pip --exclude embed"
       steps:
        - checkout
+       - run:
+           name: E2E cache key
+           command: ./tools/misc/e2e_cache_key.sh > /tmp/e2e-cache-key
        - restore_cache:
            key: go-mod-darwin-arm64-v8-{{ checksum "go.mod" }}
        - restore_cache:
            key: go-darwin-arm64-go123-v1-{{ checksum "third_party/go/BUILD" }}
        - restore_cache:
            key: python-darwin-arm64-v3-{{ checksum "third_party/python/BUILD" }}
+       # Keyed separately from the Linux jobs: the cache holds darwin_arm64 artifacts, so sharing a
+       # key with them would just mean a cache full of things this job can't use.
+       - restore_cache:
+           key: e2e-darwin-arm64-v1-{{ checksum "/tmp/e2e-cache-key" }}
        - run:
            name: Install deps
            command: ./.circleci/setup_osx.sh
@@ -255,6 +286,9 @@ jobs:
        - save_cache:
            key: python-darwin-arm64-v3-{{ checksum "third_party/python/BUILD" }}
            paths: [ ".plz-cache/third_party/python" ]
+       - save_cache:
+           key: e2e-darwin-arm64-v1-{{ checksum "/tmp/e2e-cache-key" }}
+           paths: [ "plz-out/e2e_cache" ]
 
    test-rex:
      working_directory: ~/please

src/core/lock_test.go

@@ -42,9 +42,10 @@ func TestAcquireSharedRepoRoot(t *testing.T) {
 
 	assert.IsType(t, &os.File{}, repoLockFile)
 
+	// We don't record the pid under a shared lock; other holders would be racing to write it too.
 	contents, err := os.ReadFile(repoLockFile.Name())
 	require.NoError(t, err)
-	assert.Empty(t, contents)
+	assert.Empty(t, string(contents))
 }
 
 func TestAcquireExclusiveRepoRoot(t *testing.T) {
@@ -54,8 +55,8 @@ func TestAcquireExclusiveRepoRoot(t *testing.T) {
 	assert.IsType(t, &os.File{}, repoLockFile)
 
 	contents, err := os.ReadFile(repoLockFile.Name())
-	assert.Equal(t, strconv.Itoa(os.Getpid()), string(contents))
 	assert.NoError(t, err)
+	assert.Equal(t, strconv.Itoa(os.Getpid()), string(contents))
 }
 
 func TestAcquireRepoRootOverride(t *testing.T) {

test/audit/BUILD

@@ -4,4 +4,6 @@ please_repo_e2e_test(
     name = "audit_test",
     repo = "test_repo",
     plz_command = "./test_audit.sh",
+    # Asserts that the plugin download is audited, which only happens if we actually download it.
+    shared_cache = False,
 )

test/build_defs/test.build_defs

@@ -1,3 +1,16 @@
+# Every e2e test runs in its own copy of a repo with its own $HOME, so by default they each get an
+# empty dir cache and repeat a lot of expensive work (fetching plugins, building the Go stdlib etc).
+# The dir cache is content-addressed and safe to use concurrently, so we point them all at a single
+# cache instead. $TMP_DIR is <repo root>/plz-out/tmp/..., so trimming from the last /plz-out/ gets us
+# back to the root of the repo we're being run from.
+#
+# Cleaning is off: the suite needs about 1.5G and 50k files of this, and the cleaner walks the whole
+# cache on startup, so leaving it on has every test in the suite doing that at once. It all lives
+# under plz-out, so `plz clean` takes it away with everything else.
+SHARED_CACHE_DIR = '${TMP_DIR%/plz-out/*}/plz-out/e2e_cache'
+# The same settings as command-line flags, for tests that can't have a config file written into them.
+SHARED_CACHE_FLAG = f'-o "cache.dir:{SHARED_CACHE_DIR}" -o cache.dirclean:false'
+
 # Runs e2e tests against please in a specified repo
 def please_repo_e2e_test(
         name: str,
@@ -12,12 +25,25 @@ def please_repo_e2e_test(
         expect_output_contains: dict = {},
         expect_output_doesnt_contain: dict = {},
         labels: list = [],
+        shared_cache: bool = True,
 ):
     if expected_failure:
         plz_command += "; [ ! $? -eq 0 ]"
 
     test_cmd = [
         "cp $DATA_BASE_CONFIG $DATA_REPO",
+    ]
+
+    if shared_cache:
+        # See SHARED_CACHE_DIR above. This goes in .plzconfig.local rather than on the command line
+        # so that it applies to nested plz invocations too (some tests run a script rather than plz
+        # directly), and rather than in $PLZ_OVERRIDES because a test passing its own -o flags makes
+        # go-flags ignore that env var in its entirety.
+        # The path goes through printf's arguments rather than its format string so that a repo root
+        # containing a % can't be read as a format specifier.
+        test_cmd.append(f'printf "[cache]\\ndir = %s\\ndirclean = false\\n" "{SHARED_CACHE_DIR}" > $DATA_REPO/.plzconfig.local')
+
+    test_cmd += [
         "cd $DATA_REPO",
         plz_command,
     ]

test/cli/BUILD

@@ -7,4 +7,6 @@ please_repo_e2e_test(
     },
     plz_command = "plz test --show_all_output //package:wibble 2>&1 | grep 'wibble wibble wibble' > wibble_test.txt && plz test --rerun --show_all_output //package:wibble 2>&1 | grep 'wibble wibble wibble' >> wibble_test.txt",
     repo = "test_repo",
+    # Asserts on the test's own output, which we'd never see if its results came from a cache.
+    shared_cache = False,
 )

test/entry_point/test_repo/plugins/BUILD_FILE

@@ -7,5 +7,5 @@ plugin_repo(
 plugin_repo(
     name = "go",
     plugin = "go-rules",
-    revision = "v1.29.0",
+    revision = "v1.31.1",
 )

test/export/please_export_e2e_test.build_defs

@@ -53,7 +53,11 @@ def please_export_e2e_test(
         f'plz --repo_root="{exported_repo}" build //...',
     ] + [f'plz --repo_root="{exported_repo}" {cmd}' for cmd in cmd_on_export]
 
-    test_cmd = [cmd.replace("plz ", "$TOOLS_PLEASE ") for cmd in test_cmd]
+    # Share the dir cache with the rest of the e2e tests; these repos build a Go toolchain and stdlib
+    # between them, which is by far the most expensive thing either of them does. It goes on the
+    # command line rather than in a config file because both repos here get diffed against golden
+    # masters, so we can't write anything into them.
+    test_cmd = [cmd.replace("plz ", f"$TOOLS_PLEASE {SHARED_CACHE_FLAG} ") for cmd in test_cmd]
     test_cmd = " && ".join(test_cmd)
 
     data["SOURCE_REPO"] = [source_repo]

test/export/test_go_bin/expected_repo/plugins/BUILD_FILE

@@ -1,5 +1,5 @@
 plugin_repo(
     name = "go",
     plugin = "go-rules",
-    revision = "v1.21.5",
+    revision = "v1.31.1",
 )

test/export/test_go_bin/expected_repo/third_party/go/BUILD_FILE

@@ -4,15 +4,8 @@ package(default_visibility = ["PUBLIC"])
 
 go_toolchain(
     name = "toolchain",
-    hashes = [
-        "fde05d84f7f64c8d01564f299ea1897fe94457d20d8d9054200ac1f8ae1c2bc3",  # go1.24.10.darwin-amd64.tar.gz
-        "71c70841bcdadf4b5d2f7c0f099952907969f25235663622a47d6f2233ad39aa",  # go1.24.10.darwin-arm64.tar.gz
-        "cb917b64aa4a407ed3310b397cc4dca10f0a3e2b0dd184ed74164ceaeab2625e",  # go1.24.10.freebsd-amd64.tar.gz
-        "dd52b974e3d9c5a7bbfb222c685806def6be5d6f7efd10f9caa9ca1fa2f47955",  # go1.24.10.linux-amd64.tar.gz
-        "94a99dae43dab8a3fe337485bbb89214b524285ec53ea02040514b0c2a9c3f94",  # go1.24.10.linux-arm64.tar.gz
-    ],
     install_std = False,
-    version = "1.24.10",
+    version = "1.26.5",
 )
 
 go_stdlib(

test/export/test_go_bin/source_repo/plugins/BUILD_FILE

@@ -1,5 +1,5 @@
 plugin_repo(
     name = "go",
     plugin = "go-rules",
-    revision = "v1.21.5",
+    revision = "v1.31.1",
 )

test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE

@@ -4,15 +4,8 @@ package(default_visibility = ["PUBLIC"])
 
 go_toolchain(
     name = "toolchain",
-    hashes = [
-        "fde05d84f7f64c8d01564f299ea1897fe94457d20d8d9054200ac1f8ae1c2bc3",  # go1.24.10.darwin-amd64.tar.gz
-        "71c70841bcdadf4b5d2f7c0f099952907969f25235663622a47d6f2233ad39aa",  # go1.24.10.darwin-arm64.tar.gz
-        "cb917b64aa4a407ed3310b397cc4dca10f0a3e2b0dd184ed74164ceaeab2625e",  # go1.24.10.freebsd-amd64.tar.gz
-        "dd52b974e3d9c5a7bbfb222c685806def6be5d6f7efd10f9caa9ca1fa2f47955",  # go1.24.10.linux-amd64.tar.gz
-        "94a99dae43dab8a3fe337485bbb89214b524285ec53ea02040514b0c2a9c3f94",  # go1.24.10.linux-arm64.tar.gz
-    ],
     install_std = False,
-    version = "1.24.10",
+    version = "1.26.5",
 )
 
 go_stdlib(

test/export/test_native_target_with_go_dep/expected_repo/plugins/BUILD_FILE

@@ -1,5 +1,5 @@
 plugin_repo(
     name = "go",
     plugin = "go-rules",
-    revision = "v1.21.5",
+    revision = "v1.31.1",
 )

test/export/test_native_target_with_go_dep/expected_repo/third_party/go/BUILD_FILE

@@ -4,15 +4,8 @@ package(default_visibility = ["PUBLIC"])
 
 go_toolchain(
     name = "toolchain",
-    hashes = [
-        "fde05d84f7f64c8d01564f299ea1897fe94457d20d8d9054200ac1f8ae1c2bc3",  # go1.24.10.darwin-amd64.tar.gz
-        "71c70841bcdadf4b5d2f7c0f099952907969f25235663622a47d6f2233ad39aa",  # go1.24.10.darwin-arm64.tar.gz
-        "cb917b64aa4a407ed3310b397cc4dca10f0a3e2b0dd184ed74164ceaeab2625e",  # go1.24.10.freebsd-amd64.tar.gz
-        "dd52b974e3d9c5a7bbfb222c685806def6be5d6f7efd10f9caa9ca1fa2f47955",  # go1.24.10.linux-amd64.tar.gz
-        "94a99dae43dab8a3fe337485bbb89214b524285ec53ea02040514b0c2a9c3f94",  # go1.24.10.linux-arm64.tar.gz
-    ],
     install_std = False,
-    version = "1.24.10",
+    version = "1.26.5",
 )
 
 go_stdlib(

test/export/test_native_target_with_go_dep/source_repo/plugins/BUILD_FILE

@@ -1,5 +1,5 @@
 plugin_repo(
     name = "go",
     plugin = "go-rules",
-    revision = "v1.21.5",
+    revision = "v1.31.1",
 )

test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE

@@ -4,15 +4,8 @@ package(default_visibility = ["PUBLIC"])
 
 go_toolchain(
     name = "toolchain",
-    hashes = [
-        "fde05d84f7f64c8d01564f299ea1897fe94457d20d8d9054200ac1f8ae1c2bc3",  # go1.24.10.darwin-amd64.tar.gz
-        "71c70841bcdadf4b5d2f7c0f099952907969f25235663622a47d6f2233ad39aa",  # go1.24.10.darwin-arm64.tar.gz
-        "cb917b64aa4a407ed3310b397cc4dca10f0a3e2b0dd184ed74164ceaeab2625e",  # go1.24.10.freebsd-amd64.tar.gz
-        "dd52b974e3d9c5a7bbfb222c685806def6be5d6f7efd10f9caa9ca1fa2f47955",  # go1.24.10.linux-amd64.tar.gz
-        "94a99dae43dab8a3fe337485bbb89214b524285ec53ea02040514b0c2a9c3f94",  # go1.24.10.linux-arm64.tar.gz
-    ],
     install_std = False,
-    version = "1.24.10",
+    version = "1.26.5",
 )
 
 go_stdlib(

test/plugins/test_repo/plugins/BUILD_FILE

@@ -6,5 +6,5 @@ plugin_repo(
 plugin_repo(
     name = "go",
     plugin = "go-rules",
-    revision = "v1.29.0",
+    revision = "v1.31.1",
 )

test/plz_exec/test_repo/plugins/BUILD_FILE

@@ -1,10 +1,11 @@
 plugin_repo(
     name = "shell",
-    revision = "v0.1.1",
+    plugin = "shell-rules",
+    revision = "v0.2.1",
 )
 
 plugin_repo(
     name = "go",
     plugin = "go-rules",
-    revision = "v1.29.0",
+    revision = "v1.31.1",
 )

test/proto_plugin/test_repo/plugins/BUILD_FILE

@@ -6,7 +6,7 @@ plugin_repo(
 plugin_repo(
     name = "go",
     plugin = "go-rules",
-    revision = "v1.29.0",
+    revision = "v1.31.1",
 )
 
 plugin_repo(

tools/misc/e2e_cache_key.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Prints the contents of every file that determines what ends up in the e2e tests' shared dir cache
+# (see SHARED_CACHE_DIR in //test/build_defs:test.build_defs), for CI to checksum as a cache key.
+#
+# In practice that's the plugin revisions and Go toolchains the test repos pin: those account for
+# almost all of the cache, and everything else in it is cheap enough to rebuild. Entries are
+# content-addressed, so a key that's too coarse only costs us some dead weight in the cache, which
+# the dir cache's own water marks take care of.
+#
+# This globs rather than listing files so that a newly added test repo is picked up automatically.
+
+set -euo pipefail
+
+cd "$(dirname "$0")/../.."
+
+# cksum prints the checksum, the size and the name, so a file being renamed changes the output too.
+# Sorted so the ordering doesn't depend on how the filesystem hands them back.
+find test \( -path '*/plugins/BUILD_FILE' -o -path '*/third_party/go/BUILD_FILE' \) -type f \
+    | LC_ALL=C sort \
+    | xargs cksum