feat(gitlab): read opt-in files in batches through GraphQL

docs/project/e2e.md

@@ -171,7 +171,7 @@ Secrets never appear on a command line: curl reads its `PRIVATE-TOKEN` header fr
 
 ### What the tests cover
 
-- **`TestConformance`**: the platform contract suite `internal/platform/conformance` against `gitlab.NewReader` and `gitlab.NewWriter`. Every subtest gets a fresh subgroup `acme/conf-<rand>`, which inherits the roles of `acme`; nested namespaces are subgroups of it, forks come from a sibling subgroup that the reader can read (GitLab shows `forked_from_project` only to those who can read the source), and merge requests from a fork are the person's, from a fork in the person's namespace. GitLab cannot lower an inherited role on one project, so a read-only project lives in a separate top-level group: the reader and the writer are Reporters there when they are service accounts, and do not see it when they are group access token bots (whose bot can join its own group only). GitLab reserves some project paths (`files`, `create`, `edit`, ...: the routes below a project), so the fixture appends `-project` to those names of the suite. The members of a group get access to a new project in it from a Sidekiq job, seconds later on a busy instance: the fixtures wait until the accounts see each new project. `Renamed`: a renamed project's old path leads to it; GitLab finds users by their current username only, so the renamed-account half skips with a finding when an old login finds nobody.
+- **`TestConformance`**: the platform contract suite `internal/platform/conformance` against `gitlab.NewReader` and `gitlab.NewWriter`. Every subtest gets a fresh subgroup `acme/conf-<rand>`, which inherits the roles of `acme`; nested namespaces are subgroups of it, forks come from a sibling subgroup that the reader can read (GitLab shows `forked_from_project` only to those who can read the source), and merge requests from a fork are the person's, from a fork in the person's namespace. GitLab cannot lower an inherited role on one project, so a read-only project lives in a separate top-level group: the reader and the writer are Reporters there when they are service accounts, and do not see it when they are group access token bots (whose bot can join its own group only). GitLab reserves some project paths (`files`, `create`, `edit`, ...: the routes below a project), so the fixture appends `-project` to those names of the suite. The members of a group get access to a new project in it from a Sidekiq job, seconds later on a busy instance: the fixtures wait until the accounts see each new project. `Renamed`: a renamed project's old path leads to it; GitLab finds users by their current username only, so the renamed-account half skips with a finding when an old login finds nobody. `ReadFiles`: the GraphQL batch read of one path across projects (a regular file, an executable, binary content, a file over the limit, a symlink, a submodule, a directory, a missing file, an empty project) agrees with `ReadFile`, and a path through a symlinked directory is never a file.
 - **`TestAdminTokenRefused`**: the driver refuses root's token in `Self` and `Target` (it could act as anyone through Sudo); `Probe` works with it.
 - **`TestDistribute`**: `plan` and `distribute` through `cli.Main` against projects of `acme` and `acme/sub` selected by a random topic (with `subgroups: true`) and by a repo entry:
   - merge requests by the writer with the title, label, body, marker and a commit with touchmark's trailers, and no body line that GitLab could run as a quick action;

internal/distribute/optin.go

@@ -9,12 +9,13 @@ import (
 )
 
 // Opt-in files: read one by one (Reader.ReadFile), or in batches where the
-// provider's reader is a platform.BatchReader (GitHub's GraphQL aliases), so
+// provider's reader is a platform.BatchReader (GraphQL: GitHub's aliases,
+// GitLab's projects(ids:)), so
 // that a plan over thousands of targets reads their opt-in files in a few
 // requests.
 
 // optInChunk is how many targets one BatchReader call reads the opt-in
-// files of: the batch of GitHub's GraphQL query.
+// files of: the batch of GitHub's and GitLab's GraphQL queries.
 const optInChunk = 50
 
 // optInBatch reads the opt-in files of one provider's targets in chunks, in

internal/platform/conformance/conformance.go

@@ -192,6 +192,7 @@ func Run(t *testing.T, newFixture func(t *testing.T) Fixture) {
 		{"ReadFile/symlink", testReadFileSymlink},
 		{"ReadFile/submodule", testReadFileSubmodule},
 		{"ReadFile/empty-repo", testReadFileEmptyRepo},
+		{"ReadFiles", testReadFiles},
 		{"Remote", testRemote},
 		{"PRs", testPRs},
 		{"PRs/fork", testPRsFork},

internal/platform/conformance/reader.go

@@ -361,6 +361,78 @@ func testReadFileEmptyRepo(t *testing.T, fx Fixture) {
 	wantErr(t, "ReadFile in an empty repository", err, platform.ErrNotFound, platform.ClassNotFound)
 }
 
+// testReadFiles checks platform.BatchReader, for the drivers whose reader
+// is one: one path read across repositories where it is a regular file, an
+// executable, binary content, a file over the limit, a symlink, a
+// submodule, a directory, missing, and an empty repository comes back as
+// ReadFile reads it in each: the same file, or an error of the same kind
+// (ErrNotFound, ErrNotRegular, ErrTooLarge). A nested path through a
+// symlinked directory is never a file.
+func testReadFiles(t *testing.T, fx Fixture) {
+	ctx := t.Context()
+	br, ok := fx.Reader().(platform.BatchReader)
+	if !ok {
+		t.Skip("the reader reads files one by one")
+	}
+	const p, max = "cfg.yml", 1024
+	specs := []RepoSpec{
+		{Name: "batch-file", Files: []File{{Path: p, Content: []byte("version: 1\n")}}},
+		{Name: "batch-exec", Files: []File{{Path: p, Mode: "100755", Content: []byte("#!/bin/sh\necho batch\n")}}},
+		{Name: "batch-binary", Files: []File{{Path: p, Content: []byte{0, 1, 2, 0xff, 0xfe, 0}}}},
+		{Name: "batch-large", Files: []File{{Path: p, Content: bytes.Repeat([]byte("x"), 2*max)}}},
+		{Name: "batch-link", Files: []File{readme[0], {Path: p, Mode: "120000", Content: []byte("README.md")}}},
+		{Name: "batch-sub", Files: []File{readme[0], {Path: p, Mode: "160000", Content: []byte("0123456789abcdef0123456789abcdef01234567")}}},
+		{Name: "batch-dir", Files: []File{{Path: p + "/inner.md", Content: []byte("inner\n")}}},
+		{Name: "batch-missing", Files: readme},
+		{Name: "batch-empty"},
+	}
+	repos := make([]platform.Repo, len(specs))
+	for i, s := range specs {
+		repos[i] = fx.CreateRepo(t, s)
+	}
+	files, err := br.ReadFiles(ctx, repos, p, max)
+	var fe platform.FileErrors
+	if err != nil && !errors.As(err, &fe) {
+		t.Fatalf("ReadFiles: %v", err)
+	}
+	if len(files) != len(repos) || (fe != nil && len(fe) != len(repos)) {
+		t.Fatalf("ReadFiles: %d files, %d errors for %d repositories", len(files), len(fe), len(repos))
+	}
+	for i, r := range repos {
+		want, wantErr := fx.Reader().ReadFile(ctx, r, "", p, max)
+		var got error
+		if fe != nil {
+			got = fe[i]
+		}
+		if wantErr == nil {
+			if f := files[i]; got != nil || f.OID != want.OID || f.Mode != want.Mode || !bytes.Equal(f.Content, want.Content) {
+				t.Errorf("%s: ReadFiles = %s %s %q, %v; ReadFile = %s %s %q", specs[i].Name, f.Mode, f.OID, f.Content, got, want.Mode, want.OID, want.Content)
+			}
+			continue
+		}
+		if got == nil {
+			t.Errorf("%s: ReadFiles read %q; ReadFile failed: %v", specs[i].Name, files[i].Content, wantErr)
+			continue
+		}
+		for _, sentinel := range []error{platform.ErrNotFound, platform.ErrNotRegular, platform.ErrTooLarge} {
+			if errors.Is(got, sentinel) != errors.Is(wantErr, sentinel) {
+				t.Errorf("%s: ReadFiles: %v; ReadFile: %v", specs[i].Name, got, wantErr)
+			}
+		}
+	}
+
+	linked := fx.CreateRepo(t, RepoSpec{Name: "batch-dirlink", Files: []File{
+		{Path: "docs/" + p, Content: []byte("behind the link\n")},
+		{Path: "conf", Mode: "120000", Content: []byte("docs")},
+	}})
+	files, err = br.ReadFiles(ctx, []platform.Repo{linked}, "conf/"+p, max)
+	if err == nil {
+		t.Errorf("ReadFiles through a symlinked directory = %q", files[0].Content)
+	} else if !errors.Is(err, platform.ErrNotFound) && !errors.Is(err, platform.ErrNotRegular) {
+		t.Errorf("ReadFiles through a symlinked directory: %v; want ErrNotFound or ErrNotRegular", err)
+	}
+}
+
 func testRemote(t *testing.T, fx Fixture) {
 	repo := fx.CreateRepo(t, RepoSpec{Name: "remote", Files: readme})
 	rem, err := fx.Reader().Remote(t.Context(), repo)

internal/platform/gitlab/batch.go

@@ -0,0 +1,261 @@
+package gitlab
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/bedrock-python/touchmark/internal/platform"
+	"github.com/bedrock-python/touchmark/internal/throttle"
+)
+
+// batchSize is how many projects one GraphQL request reads a file of. Its
+// query complexity stays far below GitLab's limits (200 without a token,
+// 250 with one: https://docs.gitlab.com/api/graphql/#limits); gitlab.com
+// scored 45 for 50 projects on 2026-10-08.
+const batchSize = 50
+
+// batchFilesQuery reads one path at the default branch head of projects,
+// by id: the head commit, which tells that the default branch exists, and
+// the path's blob. Data only in variables.
+const batchFilesQuery = `query($ids: [ID!]!, $first: Int!, $paths: [String!]!) {
+  projects(ids: $ids, first: $first) {
+    nodes {
+      id
+      repository {
+        tree { lastCommit { sha } }
+        blobs(paths: $paths, first: 1) { nodes { path mode oid size storedExternally rawTextBlob } }
+      }
+    }
+  }
+}`
+
+// gqlProject is one node of batchFilesQuery.
+type gqlProject struct {
+	ID         string `json:"id"`
+	Repository *struct {
+		Tree *struct {
+			LastCommit *struct {
+				SHA string `json:"sha"`
+			} `json:"lastCommit"`
+		} `json:"tree"`
+		Blobs *struct {
+			Nodes []gqlBlob `json:"nodes"`
+		} `json:"blobs"`
+	} `json:"repository"`
+}
+
+// gqlBlob is a RepositoryBlob: mode as git writes it ("100644", "40000"),
+// size a BigInt, rawTextBlob null for binary content.
+type gqlBlob struct {
+	Path             string  `json:"path"`
+	Mode             string  `json:"mode"`
+	OID              string  `json:"oid"`
+	Size             bigInt  `json:"size"`
+	StoredExternally *bool   `json:"storedExternally"`
+	RawTextBlob      *string `json:"rawTextBlob"`
+}
+
+// bigInt is GraphQL's BigInt, which GitLab sends as a string or a number.
+type bigInt int64
+
+func (b *bigInt) UnmarshalJSON(data []byte) error {
+	s := strings.Trim(string(data), `"`)
+	if s == "null" || s == "" {
+		*b = 0
+		return nil
+	}
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return fmt.Errorf("BigInt %s: %w", data, err)
+	}
+	*b = bigInt(n)
+	return nil
+}
+
+// ReadFiles reads path at the default branch head of each project
+// (platform.BatchReader): one GraphQL request per 50 projects, by id
+// (projects(ids:) follows renames), each with its head commit and the
+// blob at path (repository.blobs). A blob's mode tells regular files from
+// symlinks, submodules and directories; a text blob comes in the answer
+// and is checked against its id. What the answer does not settle is read
+// with ReadFile, which judges it as it always does: a project the request
+// did not return, one without a head commit (an empty project), a nested
+// path it found no blob for (a symlink on the way), a blob stored
+// externally (LFS), binary or not hashing to its id. A path in the
+// repository's root that a resolved head lacks is ErrNotFound.
+//
+// A request that fails with a rate limit, a refused credential or the end
+// of ctx fails the call; another failure, or an answer with errors, reads
+// that request's projects with ReadFile, and an instance without GraphQL
+// (404) is read file by file from then on.
+func (d *reader) ReadFiles(ctx context.Context, repos []platform.Repo, path string, max int64) ([]platform.File, error) {
+	const op = "read files"
+	if err := checkTreePath(path); err != nil {
+		return nil, invalid(op, "%v", err)
+	}
+	if max < 0 {
+		return nil, invalid(op, "negative size limit %d", max)
+	}
+	files := make([]platform.File, len(repos))
+	errs := make([]error, len(repos))
+	failed := false
+	single := func(i int) error {
+		f, err := d.ReadFile(ctx, repos[i], "", path, max)
+		switch {
+		case err == nil:
+			files[i] = f
+		case stops(err):
+			return err
+		default:
+			errs[i], failed = err, true
+		}
+		return nil
+	}
+	var batch []int
+	for i, r := range repos {
+		if id, err := strconv.ParseInt(r.ID, 10, 64); err == nil && id > 0 && !d.c.noGraphQL.Load() {
+			batch = append(batch, i)
+			continue
+		}
+		if err := single(i); err != nil {
+			return nil, err
+		}
+	}
+	for start := 0; start < len(batch); start += batchSize {
+		chunk := batch[start:min(start+batchSize, len(batch))]
+		ids := make([]string, len(chunk))
+		for j, i := range chunk {
+			ids[j] = projectGID(repos[i].ID)
+		}
+		var data struct {
+			Projects struct {
+				Nodes []gqlProject `json:"nodes"`
+			} `json:"projects"`
+		}
+		var got map[string]*gqlProject
+		err := d.c.graphql(ctx, op, batchFilesQuery, map[string]any{"ids": ids, "first": len(chunk), "paths": []string{path}}, &data)
+		switch {
+		case err == nil:
+			got = make(map[string]*gqlProject, len(data.Projects.Nodes))
+			for k := range data.Projects.Nodes {
+				got[data.Projects.Nodes[k].ID] = &data.Projects.Nodes[k]
+			}
+		case stops(err):
+			return nil, err
+		case platform.ClassOf(err) == platform.ClassNotFound, platform.ClassOf(err) == platform.ClassInvalid:
+			// No GraphQL here, or one without these fields: file by file
+			// from now on.
+			d.c.noGraphQL.Store(true)
+		}
+		for _, i := range chunk {
+			f, settled, err := batchFile(op, repos[i], path, max, got[projectGID(repos[i].ID)])
+			switch {
+			case !settled:
+				if err := single(i); err != nil {
+					return nil, err
+				}
+			case err != nil:
+				errs[i], failed = err, true
+			default:
+				files[i] = f
+			}
+		}
+	}
+	if failed {
+		return files, platform.FileErrors(errs)
+	}
+	return files, nil
+}
+
+// projectGID is the global id of the project with numeric id.
+func projectGID(id string) string { return "gid://gitlab/Project/" + id }
+
+// batchFile turns a project's node of the batch into a File or ReadFile's
+// error; settled is false when ReadFile must judge it (see ReadFiles).
+func batchFile(op string, r platform.Repo, path string, max int64, p *gqlProject) (f platform.File, settled bool, err error) {
+	if p == nil || p.Repository == nil || p.Repository.Tree == nil || p.Repository.Tree.LastCommit == nil ||
+		!isHexOID(p.Repository.Tree.LastCommit.SHA) || p.Repository.Blobs == nil {
+		return platform.File{}, false, nil
+	}
+	var b *gqlBlob
+	for k := range p.Repository.Blobs.Nodes {
+		if p.Repository.Blobs.Nodes[k].Path == path {
+			b = &p.Repository.Blobs.Nodes[k]
+		}
+	}
+	if b == nil {
+		if strings.Contains(path, "/") {
+			return platform.File{}, false, nil
+		}
+		return platform.File{}, true, notFound(op, "%s: %s at %s", r.Path, path, p.Repository.Tree.LastCommit.SHA)
+	}
+	mode := normalMode(b.Mode)
+	switch {
+	case mode != modeFile && mode != modeExecutable:
+		return platform.File{}, true, fmt.Errorf("%s: %s: %s has mode %s: %w", op, r.Path, path, mode, platform.ErrNotRegular)
+	case int64(b.Size) > max:
+		return platform.File{}, true, fmt.Errorf("%s: %s: %s has %d bytes, more than %d: %w", op, r.Path, path, b.Size, max, platform.ErrTooLarge)
+	case !isHexOID(b.OID), b.StoredExternally != nil && *b.StoredExternally, b.RawTextBlob == nil:
+		return platform.File{}, false, nil
+	}
+	content := []byte(*b.RawTextBlob)
+	if int64(len(content)) != int64(b.Size) || blobID(content, len(b.OID)) != strings.ToLower(b.OID) {
+		// GraphQL returns text as UTF-8: what does not hash to the blob is
+		// read as it is.
+		return platform.File{}, false, nil
+	}
+	return platform.File{Path: path, Mode: mode, OID: strings.ToLower(b.OID), Content: content}, true, nil
+}
+
+// gqlRequest and gqlResponse are a GraphQL request and its answer.
+type (
+	gqlRequest struct {
+		Query     string         `json:"query"`
+		Variables map[string]any `json:"variables"`
+	}
+	gqlResponse struct {
+		Data   json.RawMessage `json:"data"`
+		Errors []struct {
+			Message string `json:"message"`
+		} `json:"errors"`
+	}
+)
+
+// graphql sends a GraphQL query of op, a read, to the instance's
+// /api/graphql with the token as a bearer token (anonymously without one)
+// and decodes its data into out. An answer with errors fails: a timeout
+// as ClassTransient, anything else ClassInvalid (a field the instance
+// lacks, a complexity over its limit).
+func (c *client) graphql(ctx context.Context, op, query string, vars map[string]any, out any) error {
+	u, ok := strings.CutSuffix(c.api, "/v4")
+	if !ok {
+		return &platform.Error{Op: op, Class: platform.ClassNotFound, Err: fmt.Errorf("no GraphQL endpoint next to %s", c.api)}
+	}
+	var resp gqlResponse
+	_, err := c.http.JSON(throttle.AsRead(ctx), http.MethodPost, u+"/graphql", c.gqlAuth, gqlRequest{Query: query, Variables: vars}, &resp)
+	if err != nil {
+		return c.apiError(op, err)
+	}
+	if len(resp.Errors) > 0 {
+		msg := resp.Errors[0].Message
+		class := platform.ClassInvalid
+		if strings.Contains(strings.ToLower(msg), "timeout") || strings.Contains(strings.ToLower(msg), "timed out") {
+			class = platform.ClassTransient
+		}
+		return &platform.Error{Op: op, Class: class, Err: errors.New(c.mask("GraphQL: " + oneLine(msg)))}
+	}
+	if len(resp.Data) == 0 || string(resp.Data) == "null" {
+		return shapeError(op, "GraphQL answered without data")
+	}
+	if err := json.Unmarshal(resp.Data, out); err != nil {
+		return shapeError(op, "GraphQL data does not decode: %v", err)
+	}
+	return nil
+}
+
+var _ platform.BatchReader = (*reader)(nil)