From e2b499ec42d60afbfbc5bcd6301c3ed139682e30 Mon Sep 17 00:00:00 2001 From: Blake Gentry Date: Sun, 25 Oct 2026 08:58:11 -0500 Subject: [PATCH] [go] keep CI build caches warm Verification and test jobs share an immutable dependency-keyed cache. Small verification builds can freeze that archive without the artifacts needed by the full suites, so later jobs repeat downloads and compilation. Separate shared module downloads from workload-specific build caches. Include workspace and module metadata in dependency keys, and refresh build archives after Go, SQL, fixture, and CI input changes. Retain exact toolchain, runner, database version, and race scopes when restoring older builds. Let only the primary full suite publish module downloads. Use the documented setup action across Go jobs and port fixture helpers. Check out sources before setup so local action loading and dependency hashing work. Include the action in filtered workflows and the existing monthly Dependabot coverage. --- .github/actions/setup-go-ci/action.yml | 127 +++++++++++++++++++++++++ .github/dependabot.yml | 2 +- .github/workflows/ci.yaml | 58 ++++++----- .github/workflows/conformance.yaml | 4 +- .github/workflows/java.yaml | 8 +- .github/workflows/js.yaml | 6 +- .github/workflows/ruby.yaml | 6 +- .github/workflows/rust.yaml | 6 +- 8 files changed, 184 insertions(+), 33 deletions(-) create mode 100644 .github/actions/setup-go-ci/action.yml diff --git a/.github/actions/setup-go-ci/action.yml b/.github/actions/setup-go-ci/action.yml new file mode 100644 index 000000000..f06e5fa46 --- /dev/null +++ b/.github/actions/setup-go-ci/action.yml @@ -0,0 +1,127 @@ +# Jobs with the same Go version and dependencies share setup-go's default cache. +# If a migration job saves it first, the archive lacks builds needed by tests. +# Later test jobs cannot update that saved archive, so they rebuild each run. +# Share downloaded modules, but give each workload a build cache that can refresh +# after source changes. Keep this policy here rather than repeating it in jobs. +name: Set up Go for CI +description: Install Go with shared module downloads and workload-specific build caches + +# Composite action inputs are strings, unlike typed workflow_call inputs. Pass +# "true" or "false" for the boolean-like flags below; quote literals in callers. +inputs: + build-cache-scope: + description: >- + Free-form workload identifier, such as "postgres-18-race" or "sqlite-standard"; + defaults to the workflow name and job ID. Test scopes must include backend/version + and race mode because Go also caches test results. + required: false + check-latest: + description: >- + String flag ("true" or "false"): "true" checks for the latest matching Go + version; "false" prefers an already installed matching version. + default: "false" + required: false + go-version: + description: Go version or selector, such as "1.27" or "stable"; takes precedence over go-version-file + required: false + go-version-file: + description: Repository-relative go.mod or go.work file to read when go-version is omitted + required: false + save-module-cache: + description: >- + String flag ("true" or "false"): "true" also saves downloads after a successful + job; "false" only restores them. Enable for a full-suite job, or the only Go + workload on a platform, so a smaller job cannot freeze a partial archive. + default: "false" + required: false + +runs: + using: composite + steps: + - name: Install Go + id: go + uses: actions/setup-go@v7 + with: + # This action manages module and build caches separately below. + cache: false + check-latest: ${{ inputs.check-latest }} + go-version: ${{ inputs.go-version }} + go-version-file: ${{ inputs.go-version-file }} + + # Keep key components in step outputs so they do not leak into later steps. + # actions/cache remembers the primary key in its own private post-job state, + # but reads its path input again when saving. Only paths use GITHUB_ENV: nested + # composite outputs are unavailable when post inputs are resolved. These two + # names are job-wide, so prefix them and invoke this action once per job. + - name: Prepare cache paths and keys + id: cache-config + shell: bash + env: + BUILD_CACHE_SCOPE: ${{ inputs.build-cache-scope || format('{0}-{1}', github.workflow, github.job) }} + DEPENDENCY_HASH: ${{ hashFiles('**/go.mod', '**/go.sum', 'go.work', 'go.work.sum') }} + GO_VERSION: ${{ steps.go.outputs.go-version }} + SOURCE_HASH: ${{ hashFiles('**/*.go', '**/*.sql', '**/testdata/**', 'Makefile', '.github/actions/setup-go-ci/**', '.github/workflows/*.yaml') }} + run: | + module_cache_prefix="go-mod-v1-${RUNNER_OS}-${RUNNER_ARCH}" + build_cache_prefix="go-build-v1-${RUNNER_OS}-${RUNNER_ARCH}-${ImageOS:-$RUNNER_OS}-${GO_VERSION}-${BUILD_CACHE_SCOPE}" + + { + echo "RIVER_GO_BUILD_CACHE=$(go env GOCACHE)" + echo "RIVER_GO_MOD_CACHE=$(go env GOMODCACHE)" + } >> "$GITHUB_ENV" + + { + echo "build-prefix=$build_cache_prefix" + echo "dependency-hash=$DEPENDENCY_HASH" + echo "module-prefix=$module_cache_prefix" + echo "source-hash=$SOURCE_HASH" + } >> "$GITHUB_OUTPUT" + + # Module caches contain downloads, not compiled code, so Go versions and race + # modes can share them. Hash every module's go.mod AND go.sum: version selection + # can change while go.sum still contains all the old checksums. Include workspace + # metadata too: go.work can select different local modules or replacements. + # The v1 prefix is our cache-layout version, independent of the action version. + # + # Full-suite jobs save this immutable archive; Windows has only the CLI job, + # so it saves its own downloads. A small Linux CLI/migration job would freeze + # an incomplete graph. Separate save/restore steps + # are deliberate: actions/cache saves in a post step; cache/restore never saves. + - name: Restore and save module cache + if: inputs.save-module-cache == 'true' + uses: actions/cache@v5 + with: + path: ${{ env.RIVER_GO_MOD_CACHE }} + key: ${{ steps.cache-config.outputs.module-prefix }}-${{ steps.cache-config.outputs.dependency-hash }} + # Reuse older downloads after dependency changes; Go still selects the + # requested versions and verifies checksums, fetching anything missing. + restore-keys: | + ${{ steps.cache-config.outputs.module-prefix }}- + + - name: Restore module cache + if: inputs.save-module-cache != 'true' + uses: actions/cache/restore@v5 + with: + path: ${{ env.RIVER_GO_MOD_CACHE }} + key: ${{ steps.cache-config.outputs.module-prefix }}-${{ steps.cache-config.outputs.dependency-hash }} + restore-keys: | + ${{ steps.cache-config.outputs.module-prefix }}- + + # Builds are scoped by OS, architecture, runner image, exact Go version, and + # workload. Separate race/non-race scopes keep both warm; Go itself can safely + # distinguish their artifacts. Database-version scopes also isolate cached + # test results because Go cannot detect changes to an external database. + - name: Restore and save build cache + uses: actions/cache@v5 + with: + path: ${{ env.RIVER_GO_BUILD_CACHE }} + # Archives cannot be updated in place. A new Go/CI input hash lets the job + # save its fresh artifacts. SQL/testdata hashes cover embedded migrations and + # fixtures too; unrelated port implementation/docs edits keep the same key. + key: ${{ steps.cache-config.outputs.build-prefix }}-${{ steps.cache-config.outputs.dependency-hash }}-${{ steps.cache-config.outputs.source-hash }} + # Search from most to least specific: first drop the source hash, then + # the dependency hash. Both fallbacks retain toolchain/workload scope; + # Go validates its own inputs and rebuilds anything that no longer matches. + restore-keys: | + ${{ steps.cache-config.outputs.build-prefix }}-${{ steps.cache-config.outputs.dependency-hash }}- + ${{ steps.cache-config.outputs.build-prefix }}- diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cff587eff..9650362cf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -28,7 +28,7 @@ updates: patterns: - "*" - package-ecosystem: "github-actions" - # The JavaScript workflows' composite actions live under .github/actions. + # Scan every local composite action as well as the workflows. directories: - "/" - "/.github/actions/*" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index df8118c14..b57f15a21 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,9 +52,10 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Setup Go ${{ matrix.go-version }} - uses: actions/setup-go@v7 + - name: Set up Go ${{ matrix.go-version }} for CI + uses: ./.github/actions/setup-go-ci with: + build-cache-scope: benchmark-postgres-${{ matrix.postgres-version }} go-version: ${{ matrix.go-version }} - name: Display Go version @@ -123,9 +124,15 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Setup Go ${{ matrix.go-version }} - uses: actions/setup-go@v7 + - name: Set up Go ${{ matrix.go-version }} for CI + uses: ./.github/actions/setup-go-ci with: + # Go's test-result cache cannot detect a different Postgres server. + # Keep each database version and race mode in its own build archive. + build-cache-scope: postgres-${{ matrix.postgres-version }}-${{ matrix.primary && 'race' || 'standard' }} + # Only the full primary suite may publish the shared module archive; + # verification/CLI jobs download only a subset of the dependencies. + save-module-cache: "${{ matrix.primary == true }}" go-version: ${{ matrix.go-version }} - name: Display Go version @@ -162,9 +169,10 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Setup Go ${{ matrix.go-version }} - uses: actions/setup-go@v7 + - name: Set up Go ${{ matrix.go-version }} for CI + uses: ./.github/actions/setup-go-ci with: + build-cache-scope: sqlite-${{ matrix.primary && 'race' || 'standard' }} go-version: ${{ matrix.go-version }} - name: Test with race detector @@ -184,6 +192,10 @@ jobs: timeout-minutes: 10 steps: + # Load the local action and dependency files before restoring caches. + - name: Checkout + uses: actions/checkout@v7 + # CLI tests use each runner's bundled Postgres version; the database # test matrix above covers supported Postgres versions separately. # GitHub's Linux image includes Postgres with its service disabled. @@ -228,10 +240,13 @@ jobs: # adds the bundled client tools to PATH for all subsequent steps. $env:PGBIN | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: + # Windows has no full-suite job, so its only Go workload must save + # its own module downloads. Linux uses the primary suite's archive. + save-module-cache: "${{ runner.os == 'Windows' }}" go-version: "stable" - check-latest: true + check-latest: "true" - name: Setup GOBIN (Linux) if: runner.os == 'Linux' @@ -248,9 +263,6 @@ jobs: echo $gobin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append shell: pwsh - - name: Checkout - uses: actions/checkout@v7 - - name: export RIVER_CMD_DIR run: | if [ "$RUNNER_OS" == "Linux" ]; then @@ -377,14 +389,15 @@ jobs: pull-requests: read steps: - - uses: actions/setup-go@v7 - with: - go-version: "stable" - check-latest: true - + # Load the local action and dependency files before restoring caches. - name: Checkout uses: actions/checkout@v7 + - uses: ./.github/actions/setup-go-ci + with: + go-version: "stable" + check-latest: "true" + - name: Lint uses: golangci/golangci-lint-action@v9 with: @@ -409,7 +422,7 @@ jobs: - name: Checkout uses: actions/checkout@v7 - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: go-version: "1.27" @@ -435,14 +448,15 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v7 - with: - go-version: "stable" - check-latest: true - + # Load the local action and dependency files before restoring caches. - name: Checkout uses: actions/checkout@v7 + - uses: ./.github/actions/setup-go-ci + with: + go-version: "stable" + check-latest: "true" + - name: Check all go/toolchain directives match run: CHECK=true make update-mod-go diff --git a/.github/workflows/conformance.yaml b/.github/workflows/conformance.yaml index 4f55e6def..a674d975b 100644 --- a/.github/workflows/conformance.yaml +++ b/.github/workflows/conformance.yaml @@ -10,6 +10,7 @@ on: branches: - master paths: + - ".github/actions/setup-go-ci/**" - ".github/workflows/conformance.yaml" - "Makefile" - "**.go" @@ -20,6 +21,7 @@ on: - "riverdriver/**/*.sql" pull_request: paths: + - ".github/actions/setup-go-ci/**" - ".github/workflows/conformance.yaml" - "Makefile" - "**.go" @@ -51,7 +53,7 @@ jobs: with: persist-credentials: false - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: go-version-file: go.work diff --git a/.github/workflows/java.yaml b/.github/workflows/java.yaml index 007f59dc1..61f1ab706 100644 --- a/.github/workflows/java.yaml +++ b/.github/workflows/java.yaml @@ -6,6 +6,7 @@ on: branches: - master paths: + - ".github/actions/setup-go-ci/**" - ".github/actions/setup-java/**" - ".github/workflows/java.yaml" - "Makefile" @@ -15,6 +16,7 @@ on: - "riverdriver/riversqlite/migration/**" pull_request: paths: + - ".github/actions/setup-go-ci/**" - ".github/actions/setup-java/**" - ".github/workflows/java.yaml" - "Makefile" @@ -44,7 +46,7 @@ jobs: - uses: ./.github/actions/setup-java - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: go-version-file: go.work @@ -85,7 +87,7 @@ jobs: with: java-version: ${{ matrix.java-version }} - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: go-version-file: go.work @@ -124,7 +126,7 @@ jobs: - uses: ./.github/actions/setup-java - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: go-version-file: go.work diff --git a/.github/workflows/js.yaml b/.github/workflows/js.yaml index 4c7ff727b..ca0e31b6f 100644 --- a/.github/workflows/js.yaml +++ b/.github/workflows/js.yaml @@ -7,6 +7,7 @@ on: branches: - master paths: + - ".github/actions/setup-go-ci/**" - ".github/actions/setup-js/**" - ".github/workflows/js.yaml" - "Makefile" @@ -16,6 +17,7 @@ on: tags: ["js/v*"] pull_request: paths: + - ".github/actions/setup-go-ci/**" - ".github/actions/setup-js/**" - ".github/workflows/js.yaml" - "Makefile" @@ -192,7 +194,7 @@ jobs: # Some unit tests read fixtures generated from River's Go # implementation. - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: go-version-file: go.work @@ -246,7 +248,7 @@ jobs: node-version: ${{ matrix.node-version }} # The resumable integration scenarios read Go-generated fixtures. - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: go-version-file: go.work diff --git a/.github/workflows/ruby.yaml b/.github/workflows/ruby.yaml index 794389382..226221a65 100644 --- a/.github/workflows/ruby.yaml +++ b/.github/workflows/ruby.yaml @@ -6,6 +6,7 @@ on: push: branches: [master] paths: + - ".github/actions/setup-go-ci/**" - ".github/workflows/ruby.yaml" - "Makefile" - "conformance/**" @@ -14,6 +15,7 @@ on: tags: ["ruby/v*"] pull_request: paths: + - ".github/actions/setup-go-ci/**" - ".github/workflows/ruby.yaml" - "Makefile" - "conformance/**" @@ -106,7 +108,7 @@ jobs: ruby-version: ${{ matrix.ruby }} working-directory: ruby bundler-cache: true - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: go-version-file: go.work - run: make generate/fixtures @@ -142,7 +144,7 @@ jobs: ruby-version: "4.0" working-directory: ruby bundler-cache: true - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: go-version-file: go.work - run: make generate/fixtures diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index f5915c1c8..466107542 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -7,6 +7,7 @@ on: branches: - master paths: + - ".github/actions/setup-go-ci/**" - ".github/workflows/rust.yaml" - "Makefile" - "conformance/**" @@ -16,6 +17,7 @@ on: - "rust/**" pull_request: paths: + - ".github/actions/setup-go-ci/**" - ".github/workflows/rust.yaml" - "Makefile" - "conformance/**" @@ -157,7 +159,7 @@ jobs: save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} workspaces: rust -> target # Tests read fixtures generated from River's Go implementation. - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: go-version-file: go.work - name: Check every target and feature @@ -199,7 +201,7 @@ jobs: save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && matrix.postgres-version == 18 }} workspaces: rust -> target # Tests read fixtures generated from River's Go implementation. - - uses: actions/setup-go@v7 + - uses: ./.github/actions/setup-go-ci with: go-version-file: go.work - name: Create test database