From d2df40951bae3f65f72f191033a69443e3c0e3ff Mon Sep 17 00:00:00 2001 From: AlexeyShalaev <75322386+AlexeyShalaev@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:06:36 +0300 Subject: [PATCH] feat(gitlab): read opt-in files in batches through GraphQL GitLab's reader is now a platform.BatchReader: one GraphQL request per 50 projects, by id (projects(ids:) follows renames), with each project's head commit and the blob at the path (repository.blobs). A blob's mode tells regular files from symlinks, submodules and directories, a size over the limit is ErrTooLarge, and a text blob comes in the answer, checked against its id. A root path that a resolved head lacks is ErrNotFound. Everything else goes to ReadFile, which judges it as before: a project the answer lacks, an empty project, a nested path, a blob stored in LFS, binary, or not hashing to its id. GraphQL takes the token as a bearer token, and a query counts as a read. A rate limit, a refused credential or the end of ctx fails the call; an answer with errors reads its projects file by file, and an instance that turns GraphQL down is read file by file from then on. gitlab.com scored 45 of its 200 for 50 projects (2026-10-08). Tests: unit tests on the fake API, and a conformance case, ReadFiles, that holds every BatchReader to ReadFile's answers across a regular file, an executable, binary content, a file over the limit, a symlink, a submodule, a directory, a missing file and an empty repository; it passes on the GitHub driver against ghfake and runs on live GitLab. --- docs/project/e2e.md | 2 +- internal/distribute/optin.go | 5 +- internal/platform/conformance/conformance.go | 1 + internal/platform/conformance/reader.go | 72 +++++ internal/platform/gitlab/batch.go | 261 +++++++++++++++++++ internal/platform/gitlab/batch_test.go | 224 ++++++++++++++++ internal/platform/gitlab/client.go | 10 + 7 files changed, 572 insertions(+), 3 deletions(-) create mode 100644 internal/platform/gitlab/batch.go create mode 100644 internal/platform/gitlab/batch_test.go diff --git a/docs/project/e2e.md b/docs/project/e2e.md index 0d0ebba..2c40b73 100644 --- a/docs/project/e2e.md +++ b/docs/project/e2e.md @@ -171,7 +171,7 @@ Secrets never appear on a command line: curl reads its `PRIVATE-TOKEN` header fr ### What the tests cover -- **`TestConformance`**: the platform contract suite `internal/platform/conformance` against `gitlab.NewReader` and `gitlab.NewWriter`. Every subtest gets a fresh subgroup `acme/conf-`, which inherits the roles of `acme`; nested namespaces are subgroups of it, forks come from a sibling subgroup that the reader can read (GitLab shows `forked_from_project` only to those who can read the source), and merge requests from a fork are the person's, from a fork in the person's namespace. GitLab cannot lower an inherited role on one project, so a read-only project lives in a separate top-level group: the reader and the writer are Reporters there when they are service accounts, and do not see it when they are group access token bots (whose bot can join its own group only). GitLab reserves some project paths (`files`, `create`, `edit`, ...: the routes below a project), so the fixture appends `-project` to those names of the suite. The members of a group get access to a new project in it from a Sidekiq job, seconds later on a busy instance: the fixtures wait until the accounts see each new project. `Renamed`: a renamed project's old path leads to it; GitLab finds users by their current username only, so the renamed-account half skips with a finding when an old login finds nobody. +- **`TestConformance`**: the platform contract suite `internal/platform/conformance` against `gitlab.NewReader` and `gitlab.NewWriter`. Every subtest gets a fresh subgroup `acme/conf-`, which inherits the roles of `acme`; nested namespaces are subgroups of it, forks come from a sibling subgroup that the reader can read (GitLab shows `forked_from_project` only to those who can read the source), and merge requests from a fork are the person's, from a fork in the person's namespace. GitLab cannot lower an inherited role on one project, so a read-only project lives in a separate top-level group: the reader and the writer are Reporters there when they are service accounts, and do not see it when they are group access token bots (whose bot can join its own group only). GitLab reserves some project paths (`files`, `create`, `edit`, ...: the routes below a project), so the fixture appends `-project` to those names of the suite. The members of a group get access to a new project in it from a Sidekiq job, seconds later on a busy instance: the fixtures wait until the accounts see each new project. `Renamed`: a renamed project's old path leads to it; GitLab finds users by their current username only, so the renamed-account half skips with a finding when an old login finds nobody. `ReadFiles`: the GraphQL batch read of one path across projects (a regular file, an executable, binary content, a file over the limit, a symlink, a submodule, a directory, a missing file, an empty project) agrees with `ReadFile`, and a path through a symlinked directory is never a file. - **`TestAdminTokenRefused`**: the driver refuses root's token in `Self` and `Target` (it could act as anyone through Sudo); `Probe` works with it. - **`TestDistribute`**: `plan` and `distribute` through `cli.Main` against projects of `acme` and `acme/sub` selected by a random topic (with `subgroups: true`) and by a repo entry: - merge requests by the writer with the title, label, body, marker and a commit with touchmark's trailers, and no body line that GitLab could run as a quick action; diff --git a/internal/distribute/optin.go b/internal/distribute/optin.go index 708b333..295d466 100644 --- a/internal/distribute/optin.go +++ b/internal/distribute/optin.go @@ -9,12 +9,13 @@ import ( ) // Opt-in files: read one by one (Reader.ReadFile), or in batches where the -// provider's reader is a platform.BatchReader (GitHub's GraphQL aliases), so +// provider's reader is a platform.BatchReader (GraphQL: GitHub's aliases, +// GitLab's projects(ids:)), so // that a plan over thousands of targets reads their opt-in files in a few // requests. // optInChunk is how many targets one BatchReader call reads the opt-in -// files of: the batch of GitHub's GraphQL query. +// files of: the batch of GitHub's and GitLab's GraphQL queries. const optInChunk = 50 // optInBatch reads the opt-in files of one provider's targets in chunks, in diff --git a/internal/platform/conformance/conformance.go b/internal/platform/conformance/conformance.go index f84bc13..02ce41e 100644 --- a/internal/platform/conformance/conformance.go +++ b/internal/platform/conformance/conformance.go @@ -192,6 +192,7 @@ func Run(t *testing.T, newFixture func(t *testing.T) Fixture) { {"ReadFile/symlink", testReadFileSymlink}, {"ReadFile/submodule", testReadFileSubmodule}, {"ReadFile/empty-repo", testReadFileEmptyRepo}, + {"ReadFiles", testReadFiles}, {"Remote", testRemote}, {"PRs", testPRs}, {"PRs/fork", testPRsFork}, diff --git a/internal/platform/conformance/reader.go b/internal/platform/conformance/reader.go index 2062146..85e2ae7 100644 --- a/internal/platform/conformance/reader.go +++ b/internal/platform/conformance/reader.go @@ -361,6 +361,78 @@ func testReadFileEmptyRepo(t *testing.T, fx Fixture) { wantErr(t, "ReadFile in an empty repository", err, platform.ErrNotFound, platform.ClassNotFound) } +// testReadFiles checks platform.BatchReader, for the drivers whose reader +// is one: one path read across repositories where it is a regular file, an +// executable, binary content, a file over the limit, a symlink, a +// submodule, a directory, missing, and an empty repository comes back as +// ReadFile reads it in each: the same file, or an error of the same kind +// (ErrNotFound, ErrNotRegular, ErrTooLarge). A nested path through a +// symlinked directory is never a file. +func testReadFiles(t *testing.T, fx Fixture) { + ctx := t.Context() + br, ok := fx.Reader().(platform.BatchReader) + if !ok { + t.Skip("the reader reads files one by one") + } + const p, max = "cfg.yml", 1024 + specs := []RepoSpec{ + {Name: "batch-file", Files: []File{{Path: p, Content: []byte("version: 1\n")}}}, + {Name: "batch-exec", Files: []File{{Path: p, Mode: "100755", Content: []byte("#!/bin/sh\necho batch\n")}}}, + {Name: "batch-binary", Files: []File{{Path: p, Content: []byte{0, 1, 2, 0xff, 0xfe, 0}}}}, + {Name: "batch-large", Files: []File{{Path: p, Content: bytes.Repeat([]byte("x"), 2*max)}}}, + {Name: "batch-link", Files: []File{readme[0], {Path: p, Mode: "120000", Content: []byte("README.md")}}}, + {Name: "batch-sub", Files: []File{readme[0], {Path: p, Mode: "160000", Content: []byte("0123456789abcdef0123456789abcdef01234567")}}}, + {Name: "batch-dir", Files: []File{{Path: p + "/inner.md", Content: []byte("inner\n")}}}, + {Name: "batch-missing", Files: readme}, + {Name: "batch-empty"}, + } + repos := make([]platform.Repo, len(specs)) + for i, s := range specs { + repos[i] = fx.CreateRepo(t, s) + } + files, err := br.ReadFiles(ctx, repos, p, max) + var fe platform.FileErrors + if err != nil && !errors.As(err, &fe) { + t.Fatalf("ReadFiles: %v", err) + } + if len(files) != len(repos) || (fe != nil && len(fe) != len(repos)) { + t.Fatalf("ReadFiles: %d files, %d errors for %d repositories", len(files), len(fe), len(repos)) + } + for i, r := range repos { + want, wantErr := fx.Reader().ReadFile(ctx, r, "", p, max) + var got error + if fe != nil { + got = fe[i] + } + if wantErr == nil { + if f := files[i]; got != nil || f.OID != want.OID || f.Mode != want.Mode || !bytes.Equal(f.Content, want.Content) { + t.Errorf("%s: ReadFiles = %s %s %q, %v; ReadFile = %s %s %q", specs[i].Name, f.Mode, f.OID, f.Content, got, want.Mode, want.OID, want.Content) + } + continue + } + if got == nil { + t.Errorf("%s: ReadFiles read %q; ReadFile failed: %v", specs[i].Name, files[i].Content, wantErr) + continue + } + for _, sentinel := range []error{platform.ErrNotFound, platform.ErrNotRegular, platform.ErrTooLarge} { + if errors.Is(got, sentinel) != errors.Is(wantErr, sentinel) { + t.Errorf("%s: ReadFiles: %v; ReadFile: %v", specs[i].Name, got, wantErr) + } + } + } + + linked := fx.CreateRepo(t, RepoSpec{Name: "batch-dirlink", Files: []File{ + {Path: "docs/" + p, Content: []byte("behind the link\n")}, + {Path: "conf", Mode: "120000", Content: []byte("docs")}, + }}) + files, err = br.ReadFiles(ctx, []platform.Repo{linked}, "conf/"+p, max) + if err == nil { + t.Errorf("ReadFiles through a symlinked directory = %q", files[0].Content) + } else if !errors.Is(err, platform.ErrNotFound) && !errors.Is(err, platform.ErrNotRegular) { + t.Errorf("ReadFiles through a symlinked directory: %v; want ErrNotFound or ErrNotRegular", err) + } +} + func testRemote(t *testing.T, fx Fixture) { repo := fx.CreateRepo(t, RepoSpec{Name: "remote", Files: readme}) rem, err := fx.Reader().Remote(t.Context(), repo) diff --git a/internal/platform/gitlab/batch.go b/internal/platform/gitlab/batch.go new file mode 100644 index 0000000..924500a --- /dev/null +++ b/internal/platform/gitlab/batch.go @@ -0,0 +1,261 @@ +package gitlab + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/bedrock-python/touchmark/internal/platform" + "github.com/bedrock-python/touchmark/internal/throttle" +) + +// batchSize is how many projects one GraphQL request reads a file of. Its +// query complexity stays far below GitLab's limits (200 without a token, +// 250 with one: https://docs.gitlab.com/api/graphql/#limits); gitlab.com +// scored 45 for 50 projects on 2026-10-08. +const batchSize = 50 + +// batchFilesQuery reads one path at the default branch head of projects, +// by id: the head commit, which tells that the default branch exists, and +// the path's blob. Data only in variables. +const batchFilesQuery = `query($ids: [ID!]!, $first: Int!, $paths: [String!]!) { + projects(ids: $ids, first: $first) { + nodes { + id + repository { + tree { lastCommit { sha } } + blobs(paths: $paths, first: 1) { nodes { path mode oid size storedExternally rawTextBlob } } + } + } + } +}` + +// gqlProject is one node of batchFilesQuery. +type gqlProject struct { + ID string `json:"id"` + Repository *struct { + Tree *struct { + LastCommit *struct { + SHA string `json:"sha"` + } `json:"lastCommit"` + } `json:"tree"` + Blobs *struct { + Nodes []gqlBlob `json:"nodes"` + } `json:"blobs"` + } `json:"repository"` +} + +// gqlBlob is a RepositoryBlob: mode as git writes it ("100644", "40000"), +// size a BigInt, rawTextBlob null for binary content. +type gqlBlob struct { + Path string `json:"path"` + Mode string `json:"mode"` + OID string `json:"oid"` + Size bigInt `json:"size"` + StoredExternally *bool `json:"storedExternally"` + RawTextBlob *string `json:"rawTextBlob"` +} + +// bigInt is GraphQL's BigInt, which GitLab sends as a string or a number. +type bigInt int64 + +func (b *bigInt) UnmarshalJSON(data []byte) error { + s := strings.Trim(string(data), `"`) + if s == "null" || s == "" { + *b = 0 + return nil + } + n, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return fmt.Errorf("BigInt %s: %w", data, err) + } + *b = bigInt(n) + return nil +} + +// ReadFiles reads path at the default branch head of each project +// (platform.BatchReader): one GraphQL request per 50 projects, by id +// (projects(ids:) follows renames), each with its head commit and the +// blob at path (repository.blobs). A blob's mode tells regular files from +// symlinks, submodules and directories; a text blob comes in the answer +// and is checked against its id. What the answer does not settle is read +// with ReadFile, which judges it as it always does: a project the request +// did not return, one without a head commit (an empty project), a nested +// path it found no blob for (a symlink on the way), a blob stored +// externally (LFS), binary or not hashing to its id. A path in the +// repository's root that a resolved head lacks is ErrNotFound. +// +// A request that fails with a rate limit, a refused credential or the end +// of ctx fails the call; another failure, or an answer with errors, reads +// that request's projects with ReadFile, and an instance without GraphQL +// (404) is read file by file from then on. +func (d *reader) ReadFiles(ctx context.Context, repos []platform.Repo, path string, max int64) ([]platform.File, error) { + const op = "read files" + if err := checkTreePath(path); err != nil { + return nil, invalid(op, "%v", err) + } + if max < 0 { + return nil, invalid(op, "negative size limit %d", max) + } + files := make([]platform.File, len(repos)) + errs := make([]error, len(repos)) + failed := false + single := func(i int) error { + f, err := d.ReadFile(ctx, repos[i], "", path, max) + switch { + case err == nil: + files[i] = f + case stops(err): + return err + default: + errs[i], failed = err, true + } + return nil + } + var batch []int + for i, r := range repos { + if id, err := strconv.ParseInt(r.ID, 10, 64); err == nil && id > 0 && !d.c.noGraphQL.Load() { + batch = append(batch, i) + continue + } + if err := single(i); err != nil { + return nil, err + } + } + for start := 0; start < len(batch); start += batchSize { + chunk := batch[start:min(start+batchSize, len(batch))] + ids := make([]string, len(chunk)) + for j, i := range chunk { + ids[j] = projectGID(repos[i].ID) + } + var data struct { + Projects struct { + Nodes []gqlProject `json:"nodes"` + } `json:"projects"` + } + var got map[string]*gqlProject + err := d.c.graphql(ctx, op, batchFilesQuery, map[string]any{"ids": ids, "first": len(chunk), "paths": []string{path}}, &data) + switch { + case err == nil: + got = make(map[string]*gqlProject, len(data.Projects.Nodes)) + for k := range data.Projects.Nodes { + got[data.Projects.Nodes[k].ID] = &data.Projects.Nodes[k] + } + case stops(err): + return nil, err + case platform.ClassOf(err) == platform.ClassNotFound, platform.ClassOf(err) == platform.ClassInvalid: + // No GraphQL here, or one without these fields: file by file + // from now on. + d.c.noGraphQL.Store(true) + } + for _, i := range chunk { + f, settled, err := batchFile(op, repos[i], path, max, got[projectGID(repos[i].ID)]) + switch { + case !settled: + if err := single(i); err != nil { + return nil, err + } + case err != nil: + errs[i], failed = err, true + default: + files[i] = f + } + } + } + if failed { + return files, platform.FileErrors(errs) + } + return files, nil +} + +// projectGID is the global id of the project with numeric id. +func projectGID(id string) string { return "gid://gitlab/Project/" + id } + +// batchFile turns a project's node of the batch into a File or ReadFile's +// error; settled is false when ReadFile must judge it (see ReadFiles). +func batchFile(op string, r platform.Repo, path string, max int64, p *gqlProject) (f platform.File, settled bool, err error) { + if p == nil || p.Repository == nil || p.Repository.Tree == nil || p.Repository.Tree.LastCommit == nil || + !isHexOID(p.Repository.Tree.LastCommit.SHA) || p.Repository.Blobs == nil { + return platform.File{}, false, nil + } + var b *gqlBlob + for k := range p.Repository.Blobs.Nodes { + if p.Repository.Blobs.Nodes[k].Path == path { + b = &p.Repository.Blobs.Nodes[k] + } + } + if b == nil { + if strings.Contains(path, "/") { + return platform.File{}, false, nil + } + return platform.File{}, true, notFound(op, "%s: %s at %s", r.Path, path, p.Repository.Tree.LastCommit.SHA) + } + mode := normalMode(b.Mode) + switch { + case mode != modeFile && mode != modeExecutable: + return platform.File{}, true, fmt.Errorf("%s: %s: %s has mode %s: %w", op, r.Path, path, mode, platform.ErrNotRegular) + case int64(b.Size) > max: + return platform.File{}, true, fmt.Errorf("%s: %s: %s has %d bytes, more than %d: %w", op, r.Path, path, b.Size, max, platform.ErrTooLarge) + case !isHexOID(b.OID), b.StoredExternally != nil && *b.StoredExternally, b.RawTextBlob == nil: + return platform.File{}, false, nil + } + content := []byte(*b.RawTextBlob) + if int64(len(content)) != int64(b.Size) || blobID(content, len(b.OID)) != strings.ToLower(b.OID) { + // GraphQL returns text as UTF-8: what does not hash to the blob is + // read as it is. + return platform.File{}, false, nil + } + return platform.File{Path: path, Mode: mode, OID: strings.ToLower(b.OID), Content: content}, true, nil +} + +// gqlRequest and gqlResponse are a GraphQL request and its answer. +type ( + gqlRequest struct { + Query string `json:"query"` + Variables map[string]any `json:"variables"` + } + gqlResponse struct { + Data json.RawMessage `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` + } +) + +// graphql sends a GraphQL query of op, a read, to the instance's +// /api/graphql with the token as a bearer token (anonymously without one) +// and decodes its data into out. An answer with errors fails: a timeout +// as ClassTransient, anything else ClassInvalid (a field the instance +// lacks, a complexity over its limit). +func (c *client) graphql(ctx context.Context, op, query string, vars map[string]any, out any) error { + u, ok := strings.CutSuffix(c.api, "/v4") + if !ok { + return &platform.Error{Op: op, Class: platform.ClassNotFound, Err: fmt.Errorf("no GraphQL endpoint next to %s", c.api)} + } + var resp gqlResponse + _, err := c.http.JSON(throttle.AsRead(ctx), http.MethodPost, u+"/graphql", c.gqlAuth, gqlRequest{Query: query, Variables: vars}, &resp) + if err != nil { + return c.apiError(op, err) + } + if len(resp.Errors) > 0 { + msg := resp.Errors[0].Message + class := platform.ClassInvalid + if strings.Contains(strings.ToLower(msg), "timeout") || strings.Contains(strings.ToLower(msg), "timed out") { + class = platform.ClassTransient + } + return &platform.Error{Op: op, Class: class, Err: errors.New(c.mask("GraphQL: " + oneLine(msg)))} + } + if len(resp.Data) == 0 || string(resp.Data) == "null" { + return shapeError(op, "GraphQL answered without data") + } + if err := json.Unmarshal(resp.Data, out); err != nil { + return shapeError(op, "GraphQL data does not decode: %v", err) + } + return nil +} + +var _ platform.BatchReader = (*reader)(nil) diff --git a/internal/platform/gitlab/batch_test.go b/internal/platform/gitlab/batch_test.go new file mode 100644 index 0000000..311fb6e --- /dev/null +++ b/internal/platform/gitlab/batch_test.go @@ -0,0 +1,224 @@ +package gitlab + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "testing" + + "github.com/bedrock-python/touchmark/internal/platform" +) + +const optIn = ".engineering-assets.yml" + +// gqlNode is a project node of batchFilesQuery's answer, as GitLab sends it +// (size a string: BigInt). +func gqlNode(id int, sha string, blobs ...map[string]any) map[string]any { + var tree any + if sha != "" { + tree = map[string]any{"lastCommit": map[string]any{"sha": sha}} + } + if blobs == nil { + blobs = []map[string]any{} + } + return map[string]any{"id": projectGID(strconv.Itoa(id)), "repository": map[string]any{ + "tree": tree, "blobs": map[string]any{"nodes": blobs}, + }} +} + +func gqlBlobOf(path, mode, content string) map[string]any { + return map[string]any{"path": path, "mode": mode, "oid": blobID([]byte(content), 40), + "size": strconv.Itoa(len(content)), "storedExternally": false, "rawTextBlob": content} +} + +// serveGraphQL answers POST /api/graphql with the nodes of the projects +// the request asks for, by global id. +func serveGraphQL(fx *fixture, nodes map[string]map[string]any) { + fx.handle(http.MethodPost, "/api/graphql", func(w http.ResponseWriter, r *http.Request) { + var req gqlRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, http.StatusBadRequest, msg("bad request")) + return + } + var out []any + ids, _ := req.Variables["ids"].([]any) + for _, id := range ids { + if n, ok := nodes[fmt.Sprint(id)]; ok { + out = append(out, n) + } + } + if out == nil { + out = []any{} + } + writeJSON(w, http.StatusOK, map[string]any{"data": map[string]any{"projects": map[string]any{"nodes": out}}}) + }) +} + +// unreadable declares the files API of project id as GitLab answers a +// project it cannot find: ReadFile's ClassUnknown. +func unreadable(fx *fixture, id int) { + fx.json(http.MethodGet, fmt.Sprintf("/projects/%d/repository/files/%s", id, escape(optIn)), http.StatusNotFound, msg("404 Project Not Found")) +} + +func TestReadFiles(t *testing.T) { + fx := newFixture(t) + const sha = "c0ffee0000000000000000000000000000000001" + text := "version: 1\npacks: [base]\n" + nodes := map[string]map[string]any{} + add := func(id int, n map[string]any) { nodes[projectGID(strconv.Itoa(id))] = n } + add(1, gqlNode(1, sha, gqlBlobOf(optIn, "100644", text))) + add(2, gqlNode(2, sha)) + add(3, gqlNode(3, sha, gqlBlobOf(optIn, "40000", "tree"))) + add(4, gqlNode(4, sha, gqlBlobOf(optIn, "120000", "target"))) + add(5, gqlNode(5, sha, gqlBlobOf(optIn, "100755", strings.Repeat("x", 300)))) + binary := gqlBlobOf(optIn, "100644", "\x00\x01") + binary["rawTextBlob"] = nil + add(6, gqlNode(6, sha, binary)) + add(7, gqlNode(7, "")) + lfs := gqlBlobOf(optIn, "100644", "version https://git-lfs.github.com/spec/v1\n") + lfs["storedExternally"] = true + add(9, gqlNode(9, sha, lfs)) + serveGraphQL(fx, nodes) + for _, id := range []int{6, 7, 8, 9} { + unreadable(fx, id) + } + var repos []platform.Repo + for id := 1; id <= 9; id++ { + repos = append(repos, platform.Repo{Host: fx.provider().Host, ID: strconv.Itoa(id), Path: fmt.Sprintf("acme/r%d", id)}) + } + repos = append(repos, platform.Repo{Host: fx.provider().Host, ID: "acme/byname", Path: "acme/byname"}) + fx.json(http.MethodGet, "/projects/acme%2Fbyname/repository/files/"+escape(optIn), http.StatusNotFound, msg("404 Project Not Found")) + + files, err := fx.reader.ReadFiles(t.Context(), repos, optIn, 256) + var fe platform.FileErrors + if !errors.As(err, &fe) || len(fe) != len(repos) { + t.Fatalf("ReadFiles: %v", err) + } + if f := files[0]; fe[0] != nil || string(f.Content) != text || f.Mode != "100644" || f.OID != blobID([]byte(text), 40) { + t.Errorf("a regular file: %+v, %v", f, fe[0]) + } + if !errors.Is(fe[1], platform.ErrNotFound) { + t.Errorf("a missing file: %v", fe[1]) + } + for _, i := range []int{2, 3} { + if !errors.Is(fe[i], platform.ErrNotRegular) { + t.Errorf("r%d (a directory, a symlink): %v", i+1, fe[i]) + } + } + if !errors.Is(fe[4], platform.ErrTooLarge) { + t.Errorf("a file over the limit: %v", fe[4]) + } + // Binary content, an empty project, a project the answer lacks, LFS and + // a project named by path are ReadFile's to judge: never ErrNotFound + // from the batch. + for _, i := range []int{5, 6, 7, 8, 9} { + if fe[i] == nil || errors.Is(fe[i], platform.ErrNotFound) || platform.ClassOf(fe[i]) == platform.ClassNotFound { + t.Errorf("%s: %v", repos[i].Path, fe[i]) + } + } + posts := fx.requests(http.MethodPost, "/api/graphql") + if len(posts) != 1 { + t.Fatalf("%d GraphQL requests, want 1", len(posts)) + } + if posts[0].Auth != "Bearer "+fx.token || posts[0].Token != "" { + t.Errorf("GraphQL credentials: Authorization %q, PRIVATE-TOKEN %q", posts[0].Auth, posts[0].Token) + } + var req gqlRequest + if err := json.Unmarshal([]byte(posts[0].Body), &req); err != nil || req.Query != batchFilesQuery || + fmt.Sprint(req.Variables["paths"]) != "["+optIn+"]" || fmt.Sprint(req.Variables["first"]) != "9" { + t.Errorf("the request: %v %+v", err, req) + } + for _, i := range []int{0, 1, 2, 3, 4} { + if n := len(fx.requests(http.MethodGet, fmt.Sprintf("/projects/%s/repository/files/%s", repos[i].ID, escape(optIn)))); n != 0 { + t.Errorf("%s: %d file reads, want none", repos[i].Path, n) + } + } + if w := fx.writes(); len(w) != 1 { + t.Errorf("writes %+v: only the GraphQL POST", w) + } +} + +// TestReadFilesChunks: 120 projects take three requests, in order; a nested +// path the answer lacks goes to ReadFile. +func TestReadFilesChunks(t *testing.T) { + fx := newFixture(t) + nodes := map[string]map[string]any{} + var repos []platform.Repo + for id := 1; id <= 120; id++ { + nodes[projectGID(strconv.Itoa(id))] = gqlNode(id, "c0ffee0000000000000000000000000000000001") + repos = append(repos, platform.Repo{Host: fx.provider().Host, ID: strconv.Itoa(id), Path: fmt.Sprintf("acme/r%d", id)}) + } + serveGraphQL(fx, nodes) + _, err := fx.reader.ReadFiles(t.Context(), repos, optIn, 256) + var fe platform.FileErrors + if !errors.As(err, &fe) { + t.Fatalf("ReadFiles: %v", err) + } + for i, e := range fe { + if !errors.Is(e, platform.ErrNotFound) { + t.Errorf("r%d: %v", i+1, e) + } + } + posts := fx.requests(http.MethodPost, "/api/graphql") + if len(posts) != 3 { + t.Fatalf("%d requests, want 3", len(posts)) + } + for k, want := range []string{"50", "50", "20"} { + var req gqlRequest + _ = json.Unmarshal([]byte(posts[k].Body), &req) + if got := fmt.Sprint(req.Variables["first"]); got != want { + t.Errorf("request %d: first %s, want %s", k, got, want) + } + } + + fx.reset() + unreadable2 := func(id int, p string) { + fx.json(http.MethodGet, fmt.Sprintf("/projects/%d/repository/files/%s", id, escape(p)), http.StatusNotFound, msg("404 Project Not Found")) + } + unreadable2(1, "config/"+optIn) + _, err = fx.reader.ReadFiles(t.Context(), repos[:1], "config/"+optIn, 256) + if !errors.As(err, &fe) || errors.Is(fe[0], platform.ErrNotFound) { + t.Errorf("a nested path the answer lacks: %v", err) + } +} + +// TestReadFilesFailures: a rate limit or a refused credential fails the +// call; an answer with errors reads its projects file by file, and from +// then on the client skips GraphQL. +func TestReadFilesFailures(t *testing.T) { + repo := func(fx *fixture) []platform.Repo { + return []platform.Repo{{Host: fx.provider().Host, ID: "1", Path: "acme/r1"}} + } + for _, tc := range []struct { + status int + class platform.Class + }{{http.StatusUnauthorized, platform.ClassAuth}, {http.StatusTooManyRequests, platform.ClassRateLimited}} { + fx := newFixture(t) + fx.json(http.MethodPost, "/api/graphql", tc.status, msg("no")) + if _, err := fx.reader.ReadFiles(t.Context(), repo(fx), optIn, 256); platform.ClassOf(err) != tc.class { + t.Errorf("HTTP %d: %v, want %v", tc.status, err, tc.class) + } + } + + fx := newFixture(t) + fx.json(http.MethodPost, "/api/graphql", http.StatusOK, map[string]any{ + "data": nil, "errors": []any{map[string]any{"message": "Field 'blobs' doesn't exist on type 'Repository'"}}, + }) + unreadable(fx, 1) + for run := range 2 { + _, err := fx.reader.ReadFiles(t.Context(), repo(fx), optIn, 256) + var fe platform.FileErrors + if !errors.As(err, &fe) || errors.Is(fe[0], platform.ErrNotFound) { + t.Errorf("run %d: %v", run, err) + } + } + if n := len(fx.requests(http.MethodPost, "/api/graphql")); n != 1 { + t.Errorf("%d GraphQL requests, want 1: the second call reads file by file", n) + } + if n := len(fx.requests(http.MethodGet, "/projects/1/repository/files/"+escape(optIn))); n != 2 { + t.Errorf("%d file reads, want 2", n) + } +} diff --git a/internal/platform/gitlab/client.go b/internal/platform/gitlab/client.go index ddc2bfa..de30f7d 100644 --- a/internal/platform/gitlab/client.go +++ b/internal/platform/gitlab/client.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "github.com/bedrock-python/touchmark/internal/auth" "github.com/bedrock-python/touchmark/internal/config" @@ -49,6 +50,10 @@ type client struct { web, host string token string // "" when anonymous auth *httpx.Auth + // gqlAuth is the token as a bearer token, for GraphQL; noGraphQL is set + // once the instance turned a GraphQL request down (ReadFiles). + gqlAuth *httpx.Auth + noGraphQL atomic.Bool masks *redact.Registry mu sync.Mutex @@ -98,6 +103,7 @@ func newClient(p config.ResolvedProvider, cred auth.Credential, hc *httpx.Client c.token = cred.Token c.masks.Add(cred.Token, basicUser) c.auth = &httpx.Auth{Hosts: []string{strings.ToLower(apiURL.Host)}, Name: tokenHeader, Header: c.apiHeader} + c.gqlAuth = &httpx.Auth{Hosts: []string{strings.ToLower(apiURL.Host)}, Name: "Authorization", Header: c.bearerHeader} case auth.App: return nil, fmt.Errorf("gitlab: provider %s: a GitHub App cannot sign in to GitLab; use a token of a service account or a group access token", p.ID) default: @@ -127,6 +133,10 @@ func baseURL(what, raw string) (string, *url.URL, error) { // apiHeader is the PRIVATE-TOKEN header of API requests. func (c *client) apiHeader(context.Context) (string, error) { return c.token, nil } +// bearerHeader is the Authorization header of GraphQL requests +// (https://docs.gitlab.com/api/graphql/#authentication). +func (c *client) bearerHeader(context.Context) (string, error) { return "Bearer " + c.token, nil } + // gitHeader is the Authorization header of git over HTTPS. func (c *client) gitHeader(ctx context.Context) (string, error) { if err := ctx.Err(); err != nil {