diff --git a/.github/taskgate-allow.txt b/.github/taskgate-allow.txt index 2aa0f8d..99bb2f8 100644 --- a/.github/taskgate-allow.txt +++ b/.github/taskgate-allow.txt @@ -8,3 +8,4 @@ # would be in the module graph of every repository that imports it. .github/workflows/ci.yml/gate/Task's pin # read before Task is installed .github/workflows/pages.yml/build/Task's pin # read before Task is installed +.github/workflows/ci.yml/release/Task's pin # read before Task is installed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e5d480..e31eb36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,3 +111,44 @@ jobs: archive: false retention-days: 7 if-no-files-found: ignore + + # Main's commit, once the gate has passed on it, as the binaries a repository's `go tool task` + # runs (cmd/task): every command for each platform, attested as built here from this commit, and + # published as the release bin- that a versions.yaml pins by sha256. + release: + name: Release + needs: gate + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write + id-token: write + attestations: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + + - name: Task's pin + run: go run ./cmd/versions env task >> "$GITHUB_ENV" + + - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 + with: + version: ${{ env.TASK_VERSION }} + checksum: ${{ env.TASK_SHA256 }} + + - name: The binaries + run: task release:build + + - name: Their provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: _output/release/*.tar.gz + + - name: The release + env: + GH_TOKEN: ${{ github.token }} + run: task release:publish diff --git a/README.md b/README.md index 2e36006..10deb76 100644 --- a/README.md +++ b/README.md @@ -21,15 +21,18 @@ their `go.mod`; each fails CI with a specific finding and says how to resolve it | [`linkweight`](https://spin-stack.github.io/go-tools/linkweight/) | what each binary links and why: the bytes every module puts in it, the function that stops the linker dropping unused methods, the import whose cut drops the most, and a capability - exec, listen, mount, setuid - no line of the allowlist explains | | [`proven`](https://spin-stack.github.io/go-tools/proven/) | what CI already proved about a commit, by its tree: the run that answers for it, what a run on main is of, whether it may be released, each test's time, which failed tests are flaky | | [`exposure`](https://spin-stack.github.io/go-tools/exposure/) | what a repository would show if made public: secrets and private values in every ref's commits, pull request heads included, and in its issues, comments and releases | +| [`task`](https://spin-stack.github.io/go-tools/start/) | not a check: Task at the version `versions.yaml` pins, with every other go-tools command on its `PATH`, both downloaded as released binaries - the one command a repository's `go.mod` takes, so neither Task's modules nor the tools' are compiled or cached | | [`gencache`](https://spin-stack.github.io/go-tools/gencache/) | not a check: `go generate` with each mockgen directive answered from a cache when nothing its mock is made from has changed | | [`ghatrace`](https://spin-stack.github.io/go-tools/ghatrace/) | not a check: where a GitHub Actions run's time went - the critical path, each job's wait for a runner, its steps and the spans inside them - as the run's summary, an HTML page and a Perfetto trace, with nothing hosted | | [`provenance`, `sbom`, `attestgate`, `trustroot`](https://spin-stack.github.io/go-tools/supply-chain/) | releases published without SLSA provenance or an SBOM | ```sh -go get -tool github.com/spin-stack/go-tools/cmd/mutate -go tool mutate -base origin/main +go get -tool github.com/spin-stack/go-tools/cmd/task # and task + go-tools in versions.yaml +go tool task mutate # a task that runs `mutate -base origin/main` ``` +[Getting started](https://spin-stack.github.io/go-tools/start/) has the two `versions.yaml` entries. + `versions`, `versions/upstream`, `versions/gate`, `provenance`, `attestgate`, `sbom`, `ghatrace` and `trustroot` are also packages another repository imports; the rest is `internal/`. diff --git a/Taskfile.yml b/Taskfile.yml index b5205e2..278859d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -115,6 +115,46 @@ tasks: cmds: - go run ./cmd/ghatrace run -spans "${RUNNER_TEMP:-}" + release:build: + desc: Every command, for each platform a repository runs them on, as _output/release/go-tools__.tar.gz. + # What a repository's `go tool task` (cmd/task) unpacks onto Task's PATH, so that no repository + # compiles these commands or takes their modules. Stripped and without cgo: a program that runs + # on any machine of its platform, the same bytes from the same commit. The archive's members + # are sorted, owned by nobody and dated by the commit, for the same reason. cmd/task is not in + # it: it is the command a repository compiles, and its directory is first on Task's PATH, where + # a second task would stand in front of Task. + vars: + PLATFORMS: linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 + COMMANDS: + sh: go list ./cmd/... | grep -v '/cmd/task$' | tr '\n' ' ' + cmds: + - rm -rf {{.OUT}}/release + - for: {var: PLATFORMS, split: ' '} + cmd: >- + CGO_ENABLED=0 GOOS={{index (splitList "/" .ITEM) 0}} GOARCH={{index (splitList "/" .ITEM) 1}} + go build -trimpath -ldflags='-s -w' -o {{.OUT}}/release/{{replace "/" "_" .ITEM}}/ {{.COMMANDS}} + - for: {var: PLATFORMS, split: ' '} + cmd: >- + tar --sort=name --owner=0 --group=0 --numeric-owner --mtime="@$(git show -s --format=%ct HEAD)" + -C {{.OUT}}/release/{{replace "/" "_" .ITEM}} -czf {{.OUT}}/release/go-tools_{{replace "/" "_" .ITEM}}.tar.gz . + + release:publish: + desc: Publish _output/release's archives as the GitHub release bin-, the version a versions.yaml pins (GH_TOKEN, GITHUB_SHA). + # Named by the commit's UTC time, which is also the time in the pseudo-version a go.mod requires + # this commit at: versions/gate holds a repository's go-tools entry to its go.mod's. A tag of + # that shape is no module version, so `go get` never takes it for one. A run that is retried + # publishes the release it made the first time, and fails rather than replace another's files. + cmds: + - | + set -euo pipefail + tag="bin-$(TZ=UTC git show -s --format=%cd --date=format-local:%Y%m%d.%H%M%S "${GITHUB_SHA}")" + if gh release view "$tag" >/dev/null 2>&1; then + echo "$tag is published already" + exit 0 + fi + gh release create "$tag" {{.OUT}}/release/*.tar.gz --target "${GITHUB_SHA}" --title "$tag" \ + --notes "Every go-tools command at ${GITHUB_SHA}, for a repository's versions.yaml: \`go tool versions bump go-tools\`." + site: desc: Build the documentation site (site/, Astro Starlight) into site/dist; task site:dev serves it with live reload. cmds: diff --git a/cmd/proven/main_test.go b/cmd/proven/main_test.go index 2e27365..eede35a 100644 --- a/cmd/proven/main_test.go +++ b/cmd/proven/main_test.go @@ -382,10 +382,10 @@ func TestFlakesAreTheTestsThatPassedOnTheSameTree(t *testing.T) { require.NoError(t, err) assert.Contains(t, out, "1 failed tests in 4 runs; none flaky more than once\n") - // An artifact that is not a zip is said with its name. - f.archive = []byte("nope") + // An artifact uploaded unarchived - a run's trace.json - is its one file: read, not refused. + f.archive = []byte(`{"traceEvents":[]}`) err = run(t.Context(), []string{"flakes", "-also", ""}) - assert.ErrorContains(t, err, "run 6: artifact ci-lanes-short-6: the artifact is not a zip") + require.NoError(t, err) } // What is refused before the API is asked: no command, an unknown one, a flag it has not, no diff --git a/cmd/task/main.go b/cmd/task/main.go new file mode 100644 index 0000000..c95a031 --- /dev/null +++ b/cmd/task/main.go @@ -0,0 +1,102 @@ +// Command task is Task at the version a repository's versions.yaml pins, with go-tools' commands +// on its PATH: `go tool task lint` runs the released Task, and a Taskfile says `mutate` where it +// would say `go tool mutate`. +// +// go tool task [Task's arguments] +// +// A repository takes this one command in its go.mod instead of Task's and go-tools' own: compiling +// those is about 450 modules and a minute or more on a runner, for programs their authors already +// release. This one links versions.yaml's reader and a download, and it puts versions.yaml's `task` +// entry - and its `go-tools` entry, when it has one - in the user's cache, each held to its sha256 +// and unpacked once by it (internal/toolcache), then becomes Task: the same process, so a signal, +// the terminal and the exit status are Task's. +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/spin-stack/go-tools/internal/toolcache" + "github.com/spin-stack/go-tools/versions" +) + +func main() { + err := run(context.Background(), os.Args[1:], syscall.Exec) // mutate-exempt: the process's own arguments; run is what the test drives + fmt.Fprintln(os.Stderr, "task:", err) //nolint:errcheck // the error is the exit status's reason; stderr failing changes no status + os.Exit(2) // mutate-exempt: main is reached only when the exec failed, which no test makes it do; run is what the tests drive +} + +// execFunc replaces the process with argv[0]'s program: syscall.Exec, and a recorder in a test. +type execFunc func(path string, argv []string, env []string) error + +// run finds the repository's versions.yaml, installs what it pins, and execs Task with args. It +// returns only when that fails: on success the process is Task's. +func run(ctx context.Context, args []string, exec execFunc) error { + wd, err := os.Getwd() + if err != nil { + return err + } + file, err := find(wd) + if err != nil { + return err + } + v, err := versions.Load(file) + if err != nil { + return err + } + cache, err := os.UserCacheDir() + if err != nil { + return err + } + root := filepath.Join(cache, "spin-stack", "tools") + task, err := v.Get("task") + if err != nil { + return fmt.Errorf("%w: `go tool task` runs the Task it pins", err) + } + taskDir, err := toolcache.Install(ctx, task, root) + if err != nil { + return err + } + path := os.Getenv("PATH") + if tools, err := v.Get("go-tools"); err == nil { + dir, err := toolcache.Install(ctx, tools, root) + if err != nil { + return err + } + path = dir + string(os.PathListSeparator) + path + } + bin := filepath.Join(taskDir, "task") + return exec(bin, append([]string{bin}, args...), withPath(os.Environ(), path)) +} + +// withPath is env with PATH set to path: replaced where it is, since a program that reads the +// environment from the front would find a second PATH appended after the first only by luck. +func withPath(env []string, path string) []string { + var out []string + for _, kv := range env { + if !strings.HasPrefix(kv, "PATH=") { + out = append(out, kv) + } + } + return append(out, "PATH="+path) +} + +// find is versions.yaml in dir or the nearest directory above it, as Task finds its Taskfile. +func find(dir string) (string, error) { + for { + p := filepath.Join(dir, versions.File) + if _, err := os.Stat(p); err == nil { + return p, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", errors.New("no versions.yaml here or above: `go tool task` runs the Task a repository's versions.yaml pins") + } + dir = parent + } +} diff --git a/cmd/task/main_test.go b/cmd/task/main_test.go new file mode 100644 index 0000000..efc3023 --- /dev/null +++ b/cmd/task/main_test.go @@ -0,0 +1,178 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" +) + +// archive is a .tar.gz of one executable named name, and its sha256. +func archive(t *testing.T, name string) ([]byte, string) { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o755, Size: 2, Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte("x\n")); err != nil { + t.Fatal(err) + } + if err := errors.Join(tw.Close(), gz.Close()); err != nil { + t.Fatal(err) + } + s := sha256.Sum256(buf.Bytes()) + return buf.Bytes(), hex.EncodeToString(s[:]) +} + +// repository is a directory whose versions.yaml pins task, and go-tools when withTools, at a TLS +// server the default client trusts for the test; the user cache is a directory of the test's. +func repository(t *testing.T, withTools bool) string { + t.Helper() + platform := runtime.GOOS + "_" + runtime.GOARCH + taskArchive, taskPin := archive(t, "task") + toolsArchive, toolsPin := archive(t, "mutate") + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/task/v3.54.0/task_" + platform + ".tar.gz": + _, _ = w.Write(taskArchive) + case "/tools/bin-1.2/go-tools_" + platform + ".tar.gz": + _, _ = w.Write(toolsArchive) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + before := http.DefaultClient.Transport + http.DefaultClient.Transport = srv.Client().Transport + t.Cleanup(func() { http.DefaultClient.Transport = before }) + + p := runtime.GOOS + "/" + runtime.GOARCH + file := `- name: task + kind: download + source: ` + srv.URL + `/task/v{version}/task_{os}_{arch}.tar.gz + version: 3.54.0 + pins: + ` + p + `: ` + taskPin + ` + track: github-release https://github.com/go-task/task +` + if withTools { + file += `- name: go-tools + kind: download + source: ` + srv.URL + `/tools/bin-{version}/go-tools_{os}_{arch}.tar.gz + version: "1.2" + pins: + ` + p + `: ` + toolsPin + ` + track: github-release https://github.com/spin-stack/go-tools bin- +` + } + repo := t.TempDir() + if err := os.WriteFile(filepath.Join(repo, "versions.yaml"), []byte(file), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("XDG_CACHE_HOME", t.TempDir()) + t.Setenv("HOME", t.TempDir()) + return repo +} + +type call struct { + path string + argv, env []string + calledWith bool +} + +func (c *call) exec(path string, argv, env []string) error { + c.path, c.argv, c.env, c.calledWith = path, argv, env, true + return nil +} + +func pathOf(env []string) []string { + var paths []string + for _, kv := range env { + if v, ok := strings.CutPrefix(kv, "PATH="); ok { + paths = append(paths, v) + } + } + return paths +} + +// From anywhere under the repository, `go tool task` is the pinned Task, given the arguments it +// was given, with go-tools' commands first on a PATH that is there once. +func TestTaskIsThePinnedTaskWithTheToolsOnItsPath(t *testing.T) { + repo := repository(t, true) + sub := filepath.Join(repo, "internal", "x") + if err := os.MkdirAll(sub, 0o750); err != nil { + t.Fatal(err) + } + t.Chdir(sub) + t.Setenv("PATH", "/usr/bin") + var c call + if err := run(t.Context(), []string{"lint", "--", "-v"}, c.exec); err != nil { + t.Fatal(err) + } + if filepath.Base(c.path) != "task" || !slices.Equal(c.argv, []string{c.path, "lint", "--", "-v"}) { + t.Errorf("exec'd %s %v", c.path, c.argv) + } + if st, err := os.Stat(c.path); err != nil || st.Mode().Perm()&0o100 == 0 { + t.Errorf("Task is not an executable there: %v, %v", st, err) + } + paths := pathOf(c.env) + if len(paths) != 1 { + t.Fatalf("PATH is in the environment %d times: %v", len(paths), paths) + } + first, rest, _ := strings.Cut(paths[0], string(os.PathListSeparator)) + if _, err := os.Stat(filepath.Join(first, "mutate")); err != nil || rest != "/usr/bin" { + t.Errorf("PATH = %s: go-tools' directory is not first, or the rest is not the caller's", paths[0]) + } +} + +// A repository that pins no go-tools still runs its Task, with the caller's PATH as it was. +func TestTaskWithoutGoToolsKeepsThePath(t *testing.T) { + t.Chdir(repository(t, false)) + t.Setenv("PATH", "/usr/bin") + var c call + if err := run(t.Context(), nil, c.exec); err != nil { + t.Fatal(err) + } + if paths := pathOf(c.env); !slices.Equal(paths, []string{"/usr/bin"}) { + t.Errorf("PATH = %v", paths) + } +} + +// What `go tool task` cannot run is said, and nothing is exec'd: no versions.yaml above, one that +// pins no Task, one that does not read. +func TestWhatCannotRunTaskIsSaid(t *testing.T) { + for _, tc := range []struct { + name, file, says string + }{ + {"no versions.yaml", "", "no versions.yaml here or above"}, + {"no task in it", "- name: go\n kind: date\n version: \"1\"\n track: today\n", "runs the Task it pins"}, + {"a file that does not read", "- name: [\n", "versions"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if tc.file != "" { + if err := os.WriteFile(filepath.Join(dir, "versions.yaml"), []byte(tc.file), 0o600); err != nil { + t.Fatal(err) + } + } + t.Chdir(dir) + var c call + err := run(t.Context(), nil, c.exec) + if err == nil || !strings.Contains(err.Error(), tc.says) || c.calledWith { + t.Errorf("err = %v, exec'd %v; want %q", err, c.calledWith, tc.says) + } + }) + } +} diff --git a/internal/proven/flakes.go b/internal/proven/flakes.go index ab555b9..c1c973a 100644 --- a/internal/proven/flakes.go +++ b/internal/proven/flakes.go @@ -30,12 +30,16 @@ func Failed(log string) []string { return out } -// FailedIn is every top-level test the text files of a zip archive say failed - a lane's logs, -// kept as a run's artifact when it fails. +// FailedIn is every top-level test the text files of an artifact say failed - a lane's logs, kept +// as a run's artifact when it fails. An artifact is a zip of its files, or the one file itself +// when it was uploaded unarchived (upload-artifact's archive: false, as a run's trace is). func FailedIn(archive []byte) ([]string, error) { + if !bytes.HasPrefix(archive, []byte("PK\x03\x04")) { + return Failed(string(archive)), nil + } r, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive))) if err != nil { - return nil, fmt.Errorf("the artifact is not a zip: %w", err) + return nil, fmt.Errorf("the artifact is a broken zip: %w", err) } var out []string seen := map[string]bool{} diff --git a/internal/proven/flakes_test.go b/internal/proven/flakes_test.go index 59ade40..bab1ac7 100644 --- a/internal/proven/flakes_test.go +++ b/internal/proven/flakes_test.go @@ -44,8 +44,13 @@ func TestFailedInReadsEveryFileOfTheArtifact(t *testing.T) { })) require.NoError(t, err) assert.ElementsMatch(t, []string{"TestA", "TestB"}, got) - _, err = proven.FailedIn([]byte("not a zip")) - assert.ErrorContains(t, err, "is not a zip") + // An artifact uploaded unarchived is its one file. + got, err = proven.FailedIn([]byte("ok \tx\n--- FAIL: TestC (2s)\n")) + require.NoError(t, err) + assert.Equal(t, []string{"TestC"}, got) + // A zip cut short is said, not read as text. + _, err = proven.FailedIn([]byte("PK\x03\x04 cut short")) + assert.ErrorContains(t, err, "the artifact is a broken zip") } // A test is flaky when another finished run on a tree it failed on did not fail it - passed, or diff --git a/internal/toolcache/toolcache.go b/internal/toolcache/toolcache.go new file mode 100644 index 0000000..123ab79 --- /dev/null +++ b/internal/toolcache/toolcache.go @@ -0,0 +1,147 @@ +// Package toolcache puts a released program where a command can run it: the archive versions.yaml +// pins for this platform, downloaded, held to its sha256, and unpacked once into a directory named +// by that sha256, so a second run, and another repository that pins the same file, finds it there. +// +// A directory there is complete or absent: an archive is unpacked beside it and renamed into +// place, which is also what makes two processes installing the same pin at once both right. +package toolcache + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/spin-stack/go-tools/internal/fetch" + "github.com/spin-stack/go-tools/versions" +) + +// Install is the directory e's archive for this platform is unpacked in, under root: there already +// when an earlier run unpacked it, downloaded and unpacked otherwise. +func Install(ctx context.Context, e versions.Entry, root string) (string, error) { + return install(ctx, e, root, runtime.GOOS, runtime.GOARCH) +} + +func install(ctx context.Context, e versions.Entry, root, goos, goarch string) (string, error) { + if e.Kind != versions.Download { + return "", fmt.Errorf("toolcache: %s is a %s, not a download", e.Name, e.Kind) + } + e, err := e.For(goos, goarch) + if err != nil { + return "", err + } + if !strings.HasSuffix(e.URL(), ".tar.gz") { + return "", fmt.Errorf("toolcache: %s is %s, and only a .tar.gz is unpacked", e.Name, e.URL()) + } + dir := filepath.Join(root, e.Name, e.Pin) + if _, err := os.Stat(dir); err == nil { + return dir, nil + } + if err := os.MkdirAll(filepath.Dir(dir), 0o750); err != nil { + return "", fmt.Errorf("toolcache: %w", err) + } + archive, err := os.CreateTemp(filepath.Dir(dir), e.Pin+".*.tar.gz") + if err != nil { + return "", fmt.Errorf("toolcache: %w", err) + } + defer func() { _ = os.Remove(archive.Name()) }() // the archive is not kept: the directory is the cache + err = fetch.File(ctx, e.URL(), archive) + if cerr := archive.Close(); err == nil { + err = cerr + } + if err != nil { + return "", fmt.Errorf("toolcache: %s: %w", e.Name, err) + } + if got, err := sum(archive.Name()); err != nil || got != e.Pin { + return "", errors.Join(err, fmt.Errorf("toolcache: %s is not what versions.yaml pins: its SHA-256 is %s", e.URL(), got)) + } + part, err := os.MkdirTemp(filepath.Dir(dir), e.Pin+".*.part") + if err != nil { + return "", fmt.Errorf("toolcache: %w", err) + } + defer func() { _ = os.RemoveAll(part) }() // gone once renamed; otherwise what was cut short + if err := unpack(archive.Name(), part); err != nil { + return "", fmt.Errorf("toolcache: %s: %w", e.Name, err) + } + renamed := os.Rename(part, dir) + // Renamed into place here, or put there first by another process installing the same pin, + // whose directory is this one's: either way it is there. + if _, err := os.Stat(dir); err == nil { + return dir, nil + } + return "", fmt.Errorf("toolcache: %w", renamed) +} + +// unpack writes the regular files and directories of the gzipped tar at archive into dir, each +// with its mode's permission bits. A name that would land outside dir is refused, and a link +// is not followed or made: what the archive holds is programs, and a link is a way out of dir. +func unpack(archive, dir string) error { + f, err := os.Open(archive) //nolint:gosec // the archive this package just downloaded and checked + if err != nil { + return err + } + defer func() { _ = f.Close() }() // read-only + gz, err := gzip.NewReader(f) + if err != nil { + return err + } + tr := tar.NewReader(gz) + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + name := filepath.Clean(h.Name) + if !filepath.IsLocal(name) { + return fmt.Errorf("%s would be written outside the directory", h.Name) + } + target := filepath.Join(dir, name) + switch h.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0o750); err != nil { + return err + } + case tar.TypeReg: + if err := write(target, tr, fs.FileMode(h.Mode).Perm()); err != nil { + return err + } + } + } +} + +func write(target string, r io.Reader, mode fs.FileMode) error { + if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { + return err + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode) //nolint:gosec // inside the directory unpack checked + if err != nil { + return err + } + _, err = io.Copy(out, r) //nolint:gosec // an archive this package checked against its pin, not one a stranger sent + return errors.Join(err, out.Close()) +} + +func sum(p string) (string, error) { + f, err := os.Open(p) //nolint:gosec // the archive this package just downloaded + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() // read-only + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/toolcache/toolcache_test.go b/internal/toolcache/toolcache_test.go new file mode 100644 index 0000000..7c92f2e --- /dev/null +++ b/internal/toolcache/toolcache_test.go @@ -0,0 +1,198 @@ +package toolcache + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/spin-stack/go-tools/versions" +) + +type member struct { + name, body string + mode int64 + kind byte +} + +// tarball is a gzipped tar of members, and its sha256. +func tarball(t *testing.T, members ...member) ([]byte, string) { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for _, m := range members { + h := &tar.Header{Name: m.name, Mode: m.mode, Size: int64(len(m.body)), Typeflag: m.kind} + if m.kind != tar.TypeReg { + h.Size = 0 + h.Linkname = m.body + } + if err := tw.WriteHeader(h); err != nil { + t.Fatal(err) + } + if m.kind == tar.TypeReg { + if _, err := tw.Write([]byte(m.body)); err != nil { + t.Fatal(err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + s := sha256.Sum256(buf.Bytes()) + return buf.Bytes(), hex.EncodeToString(s[:]) +} + +// release serves body at /v1/tool__.tar.gz and counts the downloads. +func release(t *testing.T, body []byte) (string, *atomic.Int32) { + t.Helper() + var n atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/tool_linux_amd64.tar.gz" { + http.NotFound(w, r) + return + } + n.Add(1) + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return srv.URL + "/v{version}/tool_{os}_{arch}.tar.gz", &n +} + +func entry(source, pin string) versions.Entry { + return versions.Entry{Name: "tool", Kind: versions.Download, Version: "1", Source: source, + Pins: map[string]string{"linux/amd64": pin}} +} + +// A release is unpacked once, by its pin: its programs keep their modes, a second install is the +// directory already there, and the archive is not kept beside it. +func TestAReleaseIsUnpackedOnceByItsPin(t *testing.T) { + body, pin := tarball(t, + member{name: "bin/", kind: tar.TypeDir, mode: 0o755}, + member{name: "bin/task", body: "#!/bin/sh\n", mode: 0o755, kind: tar.TypeReg}, + member{name: "LICENSE", body: "MIT", mode: 0o644, kind: tar.TypeReg}, + member{name: "bin/link", body: "/etc/passwd", kind: tar.TypeSymlink}) + source, downloads := release(t, body) + root := t.TempDir() + dir, err := install(t.Context(), entry(source, pin), root, "linux", "amd64") + if err != nil { + t.Fatal(err) + } + if dir != filepath.Join(root, "tool", pin) { + t.Errorf("installed in %s", dir) + } + st, err := os.Stat(filepath.Join(dir, "bin", "task")) + if err != nil || st.Mode().Perm() != 0o755 { + t.Errorf("bin/task: %v, %v", st, err) + } + if body, err := os.ReadFile(filepath.Join(dir, "bin", "task")); err != nil || string(body) != "#!/bin/sh\n" { + t.Errorf("bin/task holds %q, %v", body, err) + } + if st, err := os.Stat(filepath.Join(dir, "LICENSE")); err != nil || st.Mode().Perm() != 0o644 { + t.Errorf("LICENSE: %v, %v", st, err) + } + if _, err := os.Lstat(filepath.Join(dir, "bin", "link")); err == nil { + t.Error("a link in the archive was made") + } + if again, err := install(t.Context(), entry(source, pin), root, "linux", "amd64"); err != nil || again != dir { + t.Errorf("again: %s, %v", again, err) + } + if downloads.Load() != 1 { + t.Errorf("downloaded %d times", downloads.Load()) + } + left, _ := os.ReadDir(filepath.Join(root, "tool")) + if len(left) != 1 { + t.Errorf("beside the directory: %v", left) + } +} + +// What is not the release versions.yaml pins, or would write outside its directory, leaves nothing +// in the cache; neither does an entry that is no archive of this platform. +func TestWhatIsNotThePinnedReleaseLeavesNothing(t *testing.T) { + good, pin := tarball(t, member{name: "task", body: "x", mode: 0o755, kind: tar.TypeReg}) + escape, escapePin := tarball(t, member{name: "../outside", body: "x", mode: 0o644, kind: tar.TypeReg}) + notGzip := []byte("not an archive") + sumOf := func(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } + goodSource, _ := release(t, good) + escapeSource, _ := release(t, escape) + notGzipSource, _ := release(t, notGzip) + for _, tc := range []struct { + name string + e versions.Entry + says string + }{ + {"another sha256", entry(goodSource, strings.Repeat("0", 64)), "is not what versions.yaml pins"}, + {"a name outside the directory", entry(escapeSource, escapePin), "would be written outside the directory"}, + {"no gzip", entry(notGzipSource, sumOf(notGzip)), "gzip"}, + {"a platform not pinned", versions.Entry{Name: "tool", Kind: versions.Download, Source: goodSource, Version: "1", + Pins: map[string]string{"darwin/arm64": pin}}, "pins no file for linux/amd64"}, + {"no archive", versions.Entry{Name: "tool", Kind: versions.Download, Source: "http://x/{version}/tool.zip", Version: "1", Pin: pin}, + "only a .tar.gz is unpacked"}, + {"no download", versions.Entry{Name: "tool", Kind: versions.Git, Version: "1"}, "not a download"}, + {"nothing there", entry(strings.Replace(goodSource, "/v{version}/", "/gone-{version}/", 1), pin), "404"}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + _, err := install(t.Context(), tc.e, root, "linux", "amd64") + if err == nil || !strings.Contains(err.Error(), tc.says) { + t.Fatalf("err = %v, want %q", err, tc.says) + } + left, _ := os.ReadDir(filepath.Join(root, "tool")) + if len(left) != 0 { + t.Errorf("left in the cache: %v", left) + } + if _, err := os.Stat(filepath.Join(root, "outside")); err == nil { + t.Error("a file was written outside the directory") + } + }) + } +} + +// Install is this platform's: on another, the pin it reads is another's. +func TestInstallIsThisPlatforms(t *testing.T) { + e := versions.Entry{Name: "tool", Kind: versions.Download, Source: "http://x/{version}/tool_{os}_{arch}.tar.gz", Version: "1", + Pins: map[string]string{"plan9/mips": strings.Repeat("0", 64)}} + if _, err := Install(t.Context(), e, t.TempDir()); err == nil || !strings.Contains(err.Error(), "pins no file for") { + t.Errorf("err = %v", err) + } +} + +// Two processes installing the same pin at once are both right: the one that renames second finds +// the other's directory in place, and answers it. +func TestAnInstallThatLosesTheRaceAnswersTheWinnersDirectory(t *testing.T) { + body, pin := tarball(t, member{name: "task", body: "mine", mode: 0o755, kind: tar.TypeReg}) + root := t.TempDir() + winner := filepath.Join(root, "tool", pin) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // While this one downloads, another puts the same pin in place. + if err := os.MkdirAll(winner, 0o750); err != nil { + t.Error(err) + } + if err := os.WriteFile(filepath.Join(winner, "task"), []byte("theirs"), 0o600); err != nil { + t.Error(err) + } + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + dir, err := install(t.Context(), entry(srv.URL+"/v{version}/tool_{os}_{arch}.tar.gz", pin), root, "linux", "amd64") + if err != nil || dir != winner { + t.Fatalf("installed %s, %v", dir, err) + } + if got, _ := os.ReadFile(filepath.Join(dir, "task")); string(got) != "theirs" { + t.Errorf("the winner's directory holds %q", got) + } + if left, _ := os.ReadDir(filepath.Join(root, "tool")); len(left) != 1 { + t.Errorf("beside it: %v", left) + } +} diff --git a/site/src/content/docs/start.md b/site/src/content/docs/start.md index a2585a7..bab7677 100644 --- a/site/src/content/docs/start.md +++ b/site/src/content/docs/start.md @@ -3,25 +3,56 @@ title: Getting started description: Add go-tools to a repository as Go tools, run the gates from a Taskfile, and wire them into CI. --- -Every tool here is a command under `cmd/` that a repository runs with `go tool`, at the version -its `go.mod` pins. There is nothing to install globally, and nothing to keep in sync between -machines. +Every tool here is a command under `cmd/`, released as binaries for main's head each time the gate +passes on a push to it (`bin-`), and a +repository runs them through Task at the versions its `versions.yaml` pins. There is nothing to +install globally, and nothing to compile. -## Add the tools you want +## Take Task and the tools + +One command in `go.mod`: ```sh -go get -tool github.com/spin-stack/go-tools/cmd/mutate -go get -tool github.com/spin-stack/go-tools/cmd/testquality -go get -tool github.com/spin-stack/go-tools/cmd/versions +go get -tool github.com/spin-stack/go-tools/cmd/task +``` + +and two entries in `versions.yaml`, each pinned by the sha256 of every platform's archive: + +```yaml title="versions.yaml" +- name: task + kind: download + source: https://github.com/go-task/task/releases/download/v{version}/task_{os}_{arch}.tar.gz + version: 3.54.0 + pins: + linux/amd64: 680859dbb4d881a9c72d4d9a8f510825450849af8567deacd7302c01124416fb + darwin/arm64: ... + track: github-release https://github.com/go-task/task +- name: go-tools + kind: download + source: https://github.com/spin-stack/go-tools/releases/download/bin-{version}/go-tools_{os}_{arch}.tar.gz + version: "20261109.193146" + pins: + linux/amd64: ... + track: github-release https://github.com/spin-stack/go-tools bin- ``` -Each adds a `tool` line to `go.mod`, and `go tool ` runs it. Moving to a newer go-tools is -one command, reviewed like any other dependency bump: +`go tool task lint` then downloads both for this machine, checks them against the pins, unpacks +them once into the user's cache, and runs Task with every go-tools command on its `PATH`: a +Taskfile says `mutate -base origin/main`. The first run on a machine takes a second or two; Task's +and the tools' own modules - about 450 for Task and the tools together - are in no `go.mod` and no +CI cache. + +Moving to a newer go-tools is a bump of both halves to the same commit, which `versions/gate` +checks: the packages a repository imports from `go.mod`, the commands from `versions.yaml`. ```sh go get github.com/spin-stack/go-tools@ +versions bump go-tools ``` +`versions` is one of the commands on Task's `PATH`, so the second line is a task's: the +spin-stack repositories have `task bump NAME=go-tools VERSION=...`. + ## Run them where developers already run checks The spin-stack repositories put the quick gates in `task lint` and the test-driven ones after diff --git a/site/src/content/docs/versions/reference.md b/site/src/content/docs/versions/reference.md index 601ac78..8dae789 100644 --- a/site/src/content/docs/versions/reference.md +++ b/site/src/content/docs/versions/reference.md @@ -12,6 +12,7 @@ description: Every kind of entry versions.yaml takes, the build arguments each b | `source` | where it comes from: an image repository, a URL with `{version}`, a git repository, a Go package path | | `version` | the version a person reads | | `pin` | what makes it exact: a digest, a sha256, a commit | +| `pins` | for a `download` with one file per platform: the sha256 of each, by `os/arch` as Go names them, in place of `pin` | | `archive` | for `git`: the sha256 of `git archive` at the commit, bumped by hand | | `track` | how `check` and `bump` find a newer version | | `note` | what a bump has to be checked with | @@ -23,6 +24,7 @@ description: Every kind of entry versions.yaml takes, the build arguments each b | `image` | the tag | its digest | `NAME_IMAGE` (the reference by digest) | | `git` | a tag or a branch | the commit; `archive:` the sha256 of `git archive` | `NAME_VERSION`, `NAME_COMMIT`, `NAME_ARCHIVE_SHA256` | | `download` | put into `source`'s `{version}` (and `{major}`) | the file's sha256 | `NAME_VERSION`, `NAME_SHA256` | +| `download` per platform | `source` also says `{os}` and `{arch}`, and `pins:` has a sha256 for each `os/arch` | each platform's sha256; `fetch` takes this machine's, and a bump hashes every one | `NAME_VERSION`, `NAME_SHA256_LINUX_AMD64`, ... | | `date` | a point in time, such as the day a package archive is read as of | none | `NAME` | | `module` | a Go program's version; `source` its package path, for `go install @` | none: the checksum database holds it | `NAME_VERSION` | | `pypi` | the package's version; `source` its name | none: for a test's tool only | `NAME_VERSION` | @@ -31,7 +33,7 @@ description: Every kind of entry versions.yaml takes, the build arguments each b | track | the newest is | |---|---| -| `github-release [repo]` | the repository's latest release (`source`'s by default), with a `v` only when the version has one | +| `github-release [repo] [prefix]` | the repository's latest release (`source`'s by default), with a `v` only when the version has one; with a prefix, a release is `` (go-tools' `bin-20261109.193146`) | | `tags ` | the highest release-numbered tag of a git repository; for a download, the highest whose file is published | | `commit ` | the commit a branch is at, for a download whose version is a commit: behind only when the file changed | | `go` | the newest stable Go, spelled as the tag is (`1.27.1-trixie`) | @@ -45,3 +47,20 @@ description: Every kind of entry versions.yaml takes, the build arguments each b The format only grows. Every repository that uses go-tools reads the same file format, so a field, a kind or a track is added, never renamed. + +## A download per platform + +```yaml title="versions.yaml" +- name: task + kind: download + source: https://github.com/go-task/task/releases/download/v{version}/task_{os}_{arch}.tar.gz + version: 3.54.0 + pins: + linux/amd64: 680859dbb4d881a9c72d4d9a8f510825450849af8567deacd7302c01124416fb + darwin/arm64: ... + track: github-release https://github.com/go-task/task +``` + +A bump downloads every platform's file of the new version and rewrites each pin where it is +written; a release that lacks one of them is refused whole. The platforms are the ones the entry +lists: adding one is an edit, not a bump. diff --git a/versions/gate/gate.go b/versions/gate/gate.go index bc64a7b..3926510 100644 --- a/versions/gate/gate.go +++ b/versions/gate/gate.go @@ -96,6 +96,7 @@ func (g Gate) Check() error { } } } + errs = append(errs, taskCommand(v, files["go.mod"], used)...) for _, e := range v.Entries { if !used[e.Name] { errs = append(errs, fmt.Errorf("%s pins %s, and nothing reads it", versions.File, e.Name)) @@ -104,6 +105,43 @@ func (g Gate) Check() error { return errors.Join(errs...) } +// TaskCommand is the go-tools command a go.mod takes as `go tool task`: Task at versions.yaml's +// `task`, with versions.yaml's `go-tools` on its PATH. +const TaskCommand = "github.com/spin-stack/go-tools/cmd/task" + +// goTools is go.mod's go-tools requirement at a commit: v0.0.0--, whose time is +// the name of that commit's release (bin-YYYYMMDD.hhmmss). +var goTools = regexp.MustCompile(`(?m)^\s*(?:require\s+)?github\.com/spin-stack/go-tools v0\.0\.0-(\d{8})(\d{6})-[0-9a-f]{12}\b`) + +// taskCommand is what a go.mod that takes go-tools' task command owes versions.yaml: the `task` it +// runs, and the `go-tools` it puts on Task's PATH at the commit go.mod's library is - one commit for +// the packages a repository imports and the commands its Taskfile runs. Both entries are read by +// the command, which no line of the repository spells as a read, so they are marked used here. +func taskCommand(v *versions.Versions, gomod string, used map[string]bool) []error { + if !slices.ContainsFunc(strings.Fields(gomod), func(f string) bool { return f == TaskCommand }) { + return nil + } + var errs []error + if _, err := v.Get("task"); err != nil { + errs = append(errs, fmt.Errorf("go.mod takes %s, which runs the Task %s pins: %w", TaskCommand, versions.File, err)) + } + used["task"] = true + tools, err := v.Get("go-tools") + if err != nil { + return errs + } + used["go-tools"] = true + m := goTools.FindStringSubmatch(gomod) + if m == nil { + return errs + } + if want := m[1] + "." + m[2]; tools.Version != want { + errs = append(errs, fmt.Errorf("go.mod requires go-tools at %s and %s's go-tools is %s: one commit for the packages and the commands (versions bump go-tools %s)", + want, versions.File, tools.Version, want)) + } + return errs +} + // pins is what the file rel breaks of the gate by what it writes: a digest or a commit, or a // Dockerfile's pin that is not one of args, the build arguments versions.yaml hands in, or has a // default. diff --git a/versions/gate/gate_test.go b/versions/gate/gate_test.go index 4dec4b2..412a9d6 100644 --- a/versions/gate/gate_test.go +++ b/versions/gate/gate_test.go @@ -138,3 +138,52 @@ func TestAGateOutsideARepositorySaysWhatGitSaid(t *testing.T) { t.Fatalf("err = %v, want git's word that this is no repository", err) } } + +// A go.mod that takes go-tools' task command reads versions.yaml's task and go-tools through it, +// and holds go-tools to the commit the go.mod requires; one without it reads neither. +func TestTheTaskCommandReadsTaskAndGoToolsAtGoModsCommit(t *testing.T) { + pins := func(tools string) string { + file := "- name: task\n kind: download\n source: https://x/v{version}/task_{os}_{arch}.tar.gz\n version: 3.54.0\n pins:\n linux/amd64: " + + strings.Repeat("a", 64) + "\n track: github-release https://github.com/go-task/task\n" + if tools != "" { + file += "- name: go-tools\n kind: download\n source: https://x/bin-{version}/go-tools_{os}_{arch}.tar.gz\n version: \"" + tools + + "\"\n pins:\n linux/amd64: " + strings.Repeat("b", 64) + "\n track: github-release https://github.com/spin-stack/go-tools bin-\n" + } + return file + } + const tool = "\ntool (\n\tgithub.com/spin-stack/go-tools/cmd/task\n)\n" + const require = "module x\n\ngo 1.27\n\nrequire github.com/spin-stack/go-tools v0.0.0-20261109193146-29e3e436a905\n" + for _, tc := range []struct { + name, gomod, file string + says []string + }{ + {"the command, and go-tools at go.mod's commit", require + tool, pins("20261109.193146"), nil}, + {"the command, and go-tools at another commit", require + tool, pins("20261108.000000"), + []string{"go.mod requires go-tools at 20261109.193146 and versions.yaml's go-tools is 20261108.000000"}}, + {"the command, with go-tools not required as a library", "module x\n" + tool, pins("20261108.000000"), nil}, + {"the command and no go-tools", "module x\n" + tool, pins(""), nil}, + {"the command and no task", "module x\n" + tool, "- name: go\n kind: date\n version: \"1\"\n track: today\n", + []string{"go.mod takes " + TaskCommand + ", which runs the Task versions.yaml pins", "pins go, and nothing reads it"}}, + {"the command on a tool line of its own", "module x\n\ntool " + TaskCommand + "\n", pins(""), nil}, + {"no command: nothing reads them", "module x\n", pins("20261109.193146"), + []string{"pins task, and nothing reads it", "pins go-tools, and nothing reads it"}}, + } { + t.Run(tc.name, func(t *testing.T) { + err := Gate{Root: repo(t, map[string]string{versions.File: tc.file, "go.mod": tc.gomod})}.Check() + if len(tc.says) == 0 { + if err != nil { + t.Fatal(err) + } + return + } + if err == nil { + t.Fatalf("passed; want %q", tc.says) + } + for _, s := range tc.says { + if !strings.Contains(err.Error(), s) { + t.Errorf("does not say %q:\n%v", s, err) + } + } + }) + } +} diff --git a/versions/testdata/versions.yaml b/versions/testdata/versions.yaml index 957e609..d766660 100644 --- a/versions/testdata/versions.yaml +++ b/versions/testdata/versions.yaml @@ -50,3 +50,12 @@ source: google-crc32c version: "1.7.1" track: pypi + +- name: task + kind: download + source: https://github.com/go-task/task/releases/download/v{version}/task_{os}_{arch}.tar.gz + version: 3.54.0 + pins: + linux/amd64: 680859dbb4d881a9c72d4d9a8f510825450849af8567deacd7302c01124416fb # the runners' + darwin/arm64: "1111111111111111111111111111111111111111111111111111111111111111" + track: github-release https://github.com/go-task/task diff --git a/versions/upstream/upstream.go b/versions/upstream/upstream.go index a07c3b5..640fe9a 100644 --- a/versions/upstream/upstream.go +++ b/versions/upstream/upstream.go @@ -18,6 +18,7 @@ import ( "path" "path/filepath" "regexp" + "runtime" "slices" "strconv" "strings" @@ -43,16 +44,15 @@ func newest(ctx context.Context, e versions.Entry) (string, error) { kind, arg, _ := strings.Cut(e.Track, " ") switch kind { case "github-release": - repo := e.Source - if arg != "" { - repo = arg + repo, prefix, _ := strings.Cut(arg, " ") + if repo == "" { + repo = e.Source } tag, err := githubRelease(ctx, repo) - // Spelled as the pin is: a download's URL may add the v its tags have. - if !strings.HasPrefix(e.Version, "v") { - tag = strings.TrimPrefix(tag, "v") + if err != nil { + return "", err } - return tag, err + return releaseVersion(tag, prefix, strings.HasPrefix(e.Version, "v")) case "go": return newestGo(ctx, e.Version) case "kernel": @@ -155,6 +155,9 @@ func Bump(ctx context.Context, v *versions.Versions, name, version string) (vers return versions.Entry{}, nil, fmt.Errorf("%w; name the version", err) } } + if len(e.Pins) > 0 { + return bumpPins(ctx, v, e, version) + } pin, err := resolve(ctx, e, version) if err != nil { return versions.Entry{}, nil, err @@ -167,9 +170,37 @@ func Bump(ctx context.Context, v *versions.Versions, name, version string) (vers return e, out, nil } +// bumpPins is Bump for a download pinned per platform: each platform's file at version, hashed. +// One the release does not have refuses the bump whole, since a pin left at the old version beside +// new ones would be a file of another release. +func bumpPins(ctx context.Context, v *versions.Versions, e versions.Entry, version string) (versions.Entry, []byte, error) { + e.Version = version + pins := map[string]string{} + for p := range e.Pins { + goos, goarch, _ := strings.Cut(p, "/") + one, err := e.For(goos, goarch) + if err != nil { // mutate-exempt: p is one of e's own platforms, which For always has + return versions.Entry{}, nil, err + } + if pins[p], err = sha256Of(ctx, one.URL()); err != nil { + return versions.Entry{}, nil, fmt.Errorf("versions: %s for %s: %w", e.Name, p, err) + } + } + out, err := v.SetPins(e.Name, version, pins) + if err != nil { + return versions.Entry{}, nil, err + } + e.Pins = pins + return e, out, nil +} + // Pinned is e's download, refused past limit bytes, and unless its SHA-256 is e's pin: the check -// beside the resolve that wrote the pin. +// beside the resolve that wrote the pin. A download pinned per platform is this platform's file. func Pinned(ctx context.Context, e versions.Entry, limit int64) ([]byte, error) { + e, err := e.For(runtime.GOOS, runtime.GOARCH) + if err != nil { + return nil, err + } raw, err := fetch.Bytes(ctx, e.URL(), limit) if err != nil { return nil, err @@ -191,6 +222,10 @@ func PinnedFile(ctx context.Context, e versions.Entry, dir string) (string, erro if e.Kind != versions.Download { return "", fmt.Errorf("versions: %s is a %s, not a download", e.Name, e.Kind) } + e, err := e.For(runtime.GOOS, runtime.GOARCH) + if err != nil { + return "", err + } file := filepath.Join(dir, path.Base(e.URL())) if got, err := fileSum(file); err == nil && got == e.Pin { return file, nil @@ -273,6 +308,21 @@ func sha256Of(ctx context.Context, url string) (string, error) { return sum, nil } +// releaseVersion is a release's tag as the pin spells a version: prefix taken off, for a +// repository whose releases are named prefix (go-tools' bin-20261109.193146), and the v +// its tags have only when the pin has one, since a download's URL may add it. A tag that does not +// start with prefix is no release of the kind the entry pins. +func releaseVersion(tag, prefix string, withV bool) (string, error) { + version, ok := strings.CutPrefix(tag, prefix) + if !ok { + return "", fmt.Errorf("versions: the newest release, %s, is not %s", tag, prefix) + } + if !withV { + version = strings.TrimPrefix(version, "v") + } + return version, nil +} + func githubRelease(ctx context.Context, repo string) (string, error) { path, ok := strings.CutPrefix(repo, "https://github.com/") if !ok { diff --git a/versions/upstream/upstream_test.go b/versions/upstream/upstream_test.go index 2700e35..fdb2115 100644 --- a/versions/upstream/upstream_test.go +++ b/versions/upstream/upstream_test.go @@ -5,11 +5,13 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "io" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" + "runtime" "slices" "strings" "sync/atomic" @@ -292,3 +294,119 @@ func TestAPinnedFileIsOnlyThePinnedBytes(t *testing.T) { _, err := PinnedFile(t.Context(), versions.Entry{Name: "go", Kind: versions.Image}, t.TempDir()) require.ErrorContains(t, err, "go is a image, not a download") } + +// A release's tag is the version the pin spells: a prefix the track names taken off, and a v kept +// only when the pin has one. +func TestAReleasesTagIsTheVersionThePinSpells(t *testing.T) { + for _, tc := range []struct { + tag, prefix string + withV bool + want string + }{ + {"v3.54.0", "", false, "3.54.0"}, + {"v3.54.0", "", true, "v3.54.0"}, + {"bin-20261109.193146", "bin-", false, "20261109.193146"}, + {"bin-v2", "bin-", true, "v2"}, + } { + if got, err := releaseVersion(tc.tag, tc.prefix, tc.withV); err != nil || got != tc.want { + t.Errorf("%s with %q: %q, %v; want %q", tc.tag, tc.prefix, got, err, tc.want) + } + } + if _, err := releaseVersion("v3.54.0", "bin-", false); err == nil { + t.Error("a release that is not bin- was read as one") + } +} + +type roundTrip func(*http.Request) (*http.Response, error) + +func (f roundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +// serve answers each URL in files with its body and every other with 404, as the default client's +// transport for the test. +func serve(t *testing.T, files map[string]string) { + t.Helper() + before := http.DefaultClient.Transport + http.DefaultClient.Transport = roundTrip(func(r *http.Request) (*http.Response, error) { + body, ok := files[r.URL.String()] + status := http.StatusOK + if !ok { + status = http.StatusNotFound + } + return &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(body)), Request: r, Header: http.Header{}}, nil + }) + t.Cleanup(func() { http.DefaultClient.Transport = before }) +} + +func sumOf(s string) string { h := sha256.Sum256([]byte(s)); return hex.EncodeToString(h[:]) } + +const perPlatform = `- name: go-tools + kind: download + source: https://example.test/bin-{version}/go-tools_{os}_{arch}.tar.gz + version: "1.0" + pins: + linux/amd64: ` + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ` + darwin/arm64: ` + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ` + track: github-release https://github.com/spin-stack/go-tools bin- +` + +// A bump of a download pinned per platform hashes each platform's file of the new version, and is +// refused whole when the release lacks one of them. +func TestABumpPinsEveryPlatformsFile(t *testing.T) { + serve(t, map[string]string{ + "https://example.test/bin-2.0/go-tools_linux_amd64.tar.gz": "linux", + "https://example.test/bin-2.0/go-tools_darwin_arm64.tar.gz": "darwin", + "https://example.test/bin-3.0/go-tools_linux_amd64.tar.gz": "linux 3", + }) + v, err := versions.Parse([]byte(perPlatform)) + if err != nil { + t.Fatal(err) + } + e, out, err := Bump(t.Context(), v, "go-tools", "2.0") + if err != nil { + t.Fatal(err) + } + if e.Pins["linux/amd64"] != sumOf("linux") || e.Pins["darwin/arm64"] != sumOf("darwin") || + !strings.Contains(string(out), "linux/amd64: "+sumOf("linux")+"\n") || !strings.Contains(string(out), `version: "2.0"`) { + t.Errorf("bumped: %+v\n%s", e, out) + } + if _, _, err := Bump(t.Context(), v, "go-tools", "3.0"); err == nil || !strings.Contains(err.Error(), "darwin/arm64") { + t.Errorf("a release without darwin/arm64's file was pinned: %v", err) + } +} + +// What is fetched of a download pinned per platform is this platform's file, held to its pin. +func TestAFetchIsThisPlatformsFile(t *testing.T) { + own := "https://example.test/bin-1.0/go-tools_" + runtime.GOOS + "_" + runtime.GOARCH + ".tar.gz" + serve(t, map[string]string{own: "mine"}) + e := versions.Entry{Name: "go-tools", Kind: versions.Download, Version: "1.0", + Source: "https://example.test/bin-{version}/go-tools_{os}_{arch}.tar.gz", + Pins: map[string]string{runtime.GOOS + "/" + runtime.GOARCH: sumOf("mine")}} + file, err := PinnedFile(t.Context(), e, t.TempDir()) + if err != nil || filepath.Base(file) != filepath.Base(own) { + t.Fatalf("fetched %s, %v", file, err) + } + if raw, err := Pinned(t.Context(), e, 1<<10); err != nil || string(raw) != "mine" { + t.Errorf("Pinned = %q, %v", raw, err) + } + e.Pins = map[string]string{"plan9/mips": sumOf("mine")} + if _, err := PinnedFile(t.Context(), e, t.TempDir()); err == nil { + t.Error("a platform the entry pins nothing for fetched a file") + } + if _, err := Pinned(t.Context(), e, 1<<10); err == nil { + t.Error("a platform the entry pins nothing for read a file") + } +} + +// One platform pinned under pins is still a file per platform: the bump hashes its file, not the +// source with {os} and {arch} in it. +func TestABumpOfOnePlatformsPinsHashesItsFile(t *testing.T) { + serve(t, map[string]string{"https://example.test/bin-2.0/go-tools_linux_amd64.tar.gz": "linux"}) + v, err := versions.Parse([]byte(strings.Replace(perPlatform, " darwin/arm64: "+strings.Repeat("b", 64)+"\n", "", 1))) + if err != nil { + t.Fatal(err) + } + e, _, err := Bump(t.Context(), v, "go-tools", "2.0") + if err != nil || e.Pins["linux/amd64"] != sumOf("linux") { + t.Errorf("bumped: %+v, %v", e, err) + } +} diff --git a/versions/versions.go b/versions/versions.go index db333e8..665fd55 100644 --- a/versions/versions.go +++ b/versions/versions.go @@ -12,6 +12,7 @@ import ( "cmp" "errors" "fmt" + "maps" "os" "regexp" "slices" @@ -48,6 +49,9 @@ type Entry struct { Source string `yaml:"source,omitempty"` Version string `yaml:"version"` Pin string `yaml:"pin,omitempty"` + // Pins is a download's sha256 for each platform, by os/arch as Go names them, when the file is + // one per platform: its source then says {os} and {arch}, and For is the entry for one of them. + Pins map[string]string `yaml:"pins,omitempty"` // Archive is a git entry's second check: the sha256 of what `git archive` writes for the // commit, for a build that compiles those bytes and has to know they are the ones that were // measured. Only a build can compute it, since it depends on the git that writes it. @@ -57,10 +61,11 @@ type Entry struct { } var ( - name = regexp.MustCompile(`^[a-z][a-z0-9-]*$`) - digest = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) - commit = regexp.MustCompile(`^[0-9a-f]{40}$`) - sum = regexp.MustCompile(`^[0-9a-f]{64}$`) + name = regexp.MustCompile(`^[a-z][a-z0-9-]*$`) + platform = regexp.MustCompile(`^[a-z0-9]+/[a-z0-9]+$`) + digest = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + commit = regexp.MustCompile(`^[0-9a-f]{40}$`) + sum = regexp.MustCompile(`^[0-9a-f]{64}$`) ) // validate is the entry whole and its pin in its kind's form. @@ -75,6 +80,7 @@ func (e Entry) validate() error { check(e.Version != "", "no version") check(e.Track != "", "no track: how check finds the newest") check(e.Archive == "" || e.Kind == Git, "only a git entry has an archive") + check(len(e.Pins) == 0 || e.Kind == Download, "only a download has a pin per platform") switch e.Kind { case Image: check(e.Source != "", "an image with no repository") @@ -86,7 +92,19 @@ func (e Entry) validate() error { case Download: check(strings.HasPrefix(e.Source, "https://") && strings.Contains(e.Source, "{version}"), "a download whose source is not an https URL with {version} in it") - check(sum.MatchString(e.Pin), "the pin %q is not a sha256", e.Pin) + perPlatform := strings.Contains(e.Source, "{os}") && strings.Contains(e.Source, "{arch}") + if len(e.Pins) == 0 { + check(!strings.Contains(e.Source, "{os}") && !strings.Contains(e.Source, "{arch}"), + "a source that says {os} or {arch} is a file per platform, pinned by pins") + check(sum.MatchString(e.Pin), "the pin %q is not a sha256", e.Pin) + break + } + check(e.Pin == "", "a pin and pins: one file, or one per platform") + check(perPlatform, "pins with a source that does not say both {os} and {arch}") + for _, p := range slices.Sorted(maps.Keys(e.Pins)) { + check(platform.MatchString(p), "the platform %q is not os/arch", p) + check(sum.MatchString(e.Pins[p]), "%s's pin %q is not a sha256", p, e.Pins[p]) + } case Date: check(e.Pin == "", "a date has no pin") case Module: @@ -102,9 +120,32 @@ func (e Entry) validate() error { return errors.Join(errs...) } +// equal is whether two entries say the same thing; an entry with pins is no comparable value. +func (e Entry) equal(o Entry) bool { + return e.Name == o.Name && e.Kind == o.Kind && e.Source == o.Source && e.Version == o.Version && + e.Pin == o.Pin && maps.Equal(e.Pins, o.Pins) && e.Archive == o.Archive && e.Track == o.Track && e.Note == o.Note +} + // Ref is an image's reference, by its digest: what `docker run` and FROM are given. func (e Entry) Ref() string { return e.Source + ":" + e.Version + "@" + e.Pin } +// For is the entry as one platform has it: a download pinned per platform becomes the file for +// goos/goarch and its pin, which is how everything that fetches one reads it. An entry pinned once +// is itself, for every platform; one that pins others but not this one is an error, not the file +// of another. +func (e Entry) For(goos, goarch string) (Entry, error) { + if len(e.Pins) == 0 { + return e, nil + } + pin, ok := e.Pins[goos+"/"+goarch] + if !ok { + return Entry{}, fmt.Errorf("versions: %s pins no file for %s/%s", e.Name, goos, goarch) + } + e.Source = strings.NewReplacer("{os}", goos, "{arch}", goarch).Replace(e.Source) + e.Pin, e.Pins = pin, nil + return e, nil +} + // URL is a download's, for its version. func (e Entry) URL() string { major, _, _ := strings.Cut(e.Version, ".") @@ -126,7 +167,15 @@ func (e Entry) Args() map[string]string { } return args case Download: - return map[string]string{prefix + "_VERSION": e.Version, prefix + "_SHA256": e.Pin} + args := map[string]string{prefix + "_VERSION": e.Version} + if len(e.Pins) == 0 { + args[prefix+"_SHA256"] = e.Pin + } + // Each platform's as TASK_SHA256_LINUX_AMD64, for a Dockerfile that builds for several. + for p, pin := range e.Pins { + args[prefix+"_SHA256_"+strings.ToUpper(strings.ReplaceAll(p, "/", "_"))] = pin + } + return args case PyPI, Module: return map[string]string{prefix + "_VERSION": e.Version} default: @@ -192,11 +241,32 @@ func (v *Versions) Get(name string) (Entry, error) { // blank lines, quotes - is as it was. v is left as it is; the file Set answers is what a Load of // it would read, and a value Set cannot write in place is refused rather than written otherwise. func (v *Versions) Set(name, version, pin string) ([]byte, error) { + return v.set(name, version, pin, nil) +} + +// SetPins is Set for a download pinned per platform: version, and each platform's pin. The +// platforms are the ones the file has, every one of them; adding or dropping one is a person's +// edit, not a bump's. +func (v *Versions) SetPins(name, version string, pins map[string]string) ([]byte, error) { + e, err := v.Get(name) + if err != nil { + return nil, err + } + if !slices.Equal(slices.Sorted(maps.Keys(e.Pins)), slices.Sorted(maps.Keys(pins))) { + return nil, fmt.Errorf("versions: %s pins %v, not the platforms given", name, slices.Sorted(maps.Keys(e.Pins))) + } + return v.set(name, version, "", pins) +} + +func (v *Versions) set(name, version, pin string, pins map[string]string) ([]byte, error) { e, err := v.Get(name) if err != nil { return nil, err } e.Version, e.Pin = version, pin + if pins != nil { + e.Pins = maps.Clone(pins) + } if err := e.validate(); err != nil { return nil, err } @@ -210,16 +280,26 @@ func (v *Versions) Set(name, version, pin string) ([]byte, error) { if n := field(item, "name"); n == nil || n.Value != name { continue } - for _, kv := range [][2]string{{"version", version}, {"pin", pin}} { // mutate-exempt: the pair's length; nothing reads past kv[1] - n := field(item, kv[0]) - if n == nil { + type value struct { + what string + n *yaml.Node + to string + } + values := []value{{"version", field(item, "version"), version}, {"pin", field(item, "pin"), pin}} + if m := field(item, "pins"); m != nil { + for i := 0; i+1 < len(m.Content); i += 2 { + values = append(values, value{"pins " + m.Content[i].Value, m.Content[i+1], e.Pins[m.Content[i].Value]}) + } + } + for _, val := range values { + if val.n == nil { continue } - start, end, err := span(lines[n.Line-1], n) + start, end, err := span(lines[val.n.Line-1], val.n) if err != nil { - return nil, fmt.Errorf("versions: %s's %s: %w", name, kv[0], err) + return nil, fmt.Errorf("versions: %s's %s: %w", name, val.what, err) } - edits = append(edits, edit{n.Line - 1, start, end, render(n.Style, kv[1])}) + edits = append(edits, edit{val.n.Line - 1, start, end, render(val.n.Style, val.to)}) } } // Right to left, so an edit does not move the next one on a line that has both. @@ -240,7 +320,7 @@ func (v *Versions) Set(name, version, pin string) ([]byte, error) { want[i] = e } } - if !slices.Equal(reread.Entries, want) { + if !slices.EqualFunc(reread.Entries, want, Entry.equal) { return nil, fmt.Errorf("versions: %s could not be rewritten in place", name) } return out, nil diff --git a/versions/versions_test.go b/versions/versions_test.go index 06ce741..78f7f34 100644 --- a/versions/versions_test.go +++ b/versions/versions_test.go @@ -92,6 +92,8 @@ func TestEachKindIsItsBuildArguments(t *testing.T) { {Entry{Name: "qboot", Kind: Git, Version: "master", Pin: "c", Archive: "a"}, map[string]string{"QBOOT_VERSION": "master", "QBOOT_COMMIT": "c", "QBOOT_ARCHIVE_SHA256": "a"}}, {Entry{Name: "kernel", Kind: Download, Version: "7.2.8", Pin: "s"}, map[string]string{"KERNEL_VERSION": "7.2.8", "KERNEL_SHA256": "s"}}, + {Entry{Name: "go-tools", Kind: Download, Version: "1", Pins: map[string]string{"linux/amd64": "a", "darwin/arm64": "d"}}, + map[string]string{"GO_TOOLS_VERSION": "1", "GO_TOOLS_SHA256_LINUX_AMD64": "a", "GO_TOOLS_SHA256_DARWIN_ARM64": "d"}}, {Entry{Name: "google-crc32c", Kind: PyPI, Source: "google-crc32c", Version: "1.7.1"}, map[string]string{"GOOGLE_CRC32C_VERSION": "1.7.1"}}, {Entry{Name: "debian-snapshot", Kind: Date, Version: "20260914T000000Z"}, map[string]string{"DEBIAN_SNAPSHOT": "20260914T000000Z"}}, } { @@ -176,6 +178,18 @@ func TestAnEntryOutOfItsKindsFormIsRefused(t *testing.T) { "a download whose source is not an https URL"}, {"a download with no version in its source", "kind: download\n source: https://x/latest.tgz\n pin: " + strings.Repeat("a", 64), "a download whose source is not an https URL"}, + {"a file per platform with one pin", "kind: download\n source: https://x/{version}/t_{os}_{arch}.tgz\n pin: " + strings.Repeat("a", 64), + "a source that says {os} or {arch} is a file per platform"}, + {"pins and a pin", "kind: download\n source: https://x/{version}/t_{os}_{arch}.tgz\n pin: " + strings.Repeat("a", 64) + + "\n pins: {linux/amd64: " + strings.Repeat("a", 64) + "}", "a pin and pins"}, + {"pins for a source that names no platform", "kind: download\n source: https://x/{version}/t_{os}.tgz\n pins: {linux/amd64: " + strings.Repeat("a", 64) + "}", + "pins with a source that does not say both {os} and {arch}"}, + {"a platform that is not os/arch", "kind: download\n source: https://x/{version}/t_{os}_{arch}.tgz\n pins: {linux: " + strings.Repeat("a", 64) + "}", + `the platform "linux" is not os/arch`}, + {"a platform's pin that is not a sha256", "kind: download\n source: https://x/{version}/t_{os}_{arch}.tgz\n pins: {linux/amd64: abc}", + `linux/amd64's pin "abc" is not a sha256`}, + {"pins on another kind", "kind: git\n source: https://x/y\n pin: " + strings.Repeat("a", 40) + "\n pins: {linux/amd64: " + strings.Repeat("a", 64) + "}", + "only a download has a pin per platform"}, {"a module named by a URL", "kind: module\n source: https://x/y", "a module with no package path"}, {"a module with no source", "kind: module", "a module with no package path"}, } { @@ -228,3 +242,66 @@ func TestAVersionIsNewerByItsNumbers(t *testing.T) { } } } + +// A download pinned per platform is, for each platform, the one file and pin of that platform; +// one pinned once is the same file everywhere; a platform it pins nothing for is refused. +func TestAPlatformHasItsOwnFileAndPin(t *testing.T) { + v, err := Load(fixture) + if err != nil { + t.Fatal(err) + } + task, _ := v.Get("task") + got, err := task.For("darwin", "arm64") + if err != nil { + t.Fatal(err) + } + if got.URL() != "https://github.com/go-task/task/releases/download/v3.54.0/task_darwin_arm64.tar.gz" || + got.Pin != strings.Repeat("1", 64) || got.Pins != nil { + t.Errorf("darwin/arm64 = %s %s %v", got.URL(), got.Pin, got.Pins) + } + if task.Pins["darwin/arm64"] == "" || !strings.Contains(task.Source, "{os}") { + t.Error("For changed the entry it was asked of") + } + if _, err := task.For("windows", "amd64"); err == nil || !strings.Contains(err.Error(), "pins no file for windows/amd64") { + t.Errorf("a platform with no pin: %v", err) + } + alpine, _ := v.Get("alpine") + if same, err := alpine.For("windows", "amd64"); err != nil || !same.equal(alpine) { + t.Errorf("an entry pinned once is not itself everywhere: %+v, %v", same, err) + } +} + +// A bump of a download pinned per platform writes each platform's pin where it is written, and +// only for the platforms the file already pins. +func TestABumpRewritesEachPlatformsPinInPlace(t *testing.T) { + raw, err := os.ReadFile(fixture) + if err != nil { + t.Fatal(err) + } + v, err := Parse(raw) + if err != nil { + t.Fatal(err) + } + a, d := strings.Repeat("a", 64), strings.Repeat("d", 64) + out, err := v.SetPins("task", "3.55.0", map[string]string{"linux/amd64": a, "darwin/arm64": d}) + if err != nil { + t.Fatal(err) + } + want := strings.NewReplacer( + "version: 3.54.0", "version: 3.55.0", + "linux/amd64: 680859dbb4d881a9c72d4d9a8f510825450849af8567deacd7302c01124416fb # the runners'", "linux/amd64: "+a+" # the runners'", + `darwin/arm64: "`+strings.Repeat("1", 64)+`"`, `darwin/arm64: "`+d+`"`, + ).Replace(string(raw)) + if string(out) != want { + t.Errorf("the bump wrote:\n%s", out) + } + if _, err := v.SetPins("task", "3.55.0", map[string]string{"linux/amd64": a}); err == nil { + t.Error("a bump that drops a platform was written") + } + if _, err := v.SetPins("task", "3.55.0", map[string]string{"linux/amd64": a, "darwin/arm64": d, "linux/arm64": a}); err == nil { + t.Error("a bump that adds a platform was written") + } + if _, err := v.SetPins("none", "1", nil); err == nil { + t.Error("an entry the file does not have was bumped") + } +}