Repository navigation
Expand file tree
/
Copy pathbench_backing_test.go
More file actions
162 lines (150 loc) · 5.79 KB
/
Copy pathbench_backing_test.go
File metadata and controls
162 lines (150 loc) · 5.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
// Copyright (c) the go-embedded-ruby/ruby authors
//
// SPDX-License-Identifier: BSD-3-Clause
package ruby
import (
"flag"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"testing"
)
var updateBacking = flag.Bool("update-backing", false,
"rewrite bench/modules/BACKING.md from what go.mod currently requires")
// Every module under bench/modules has a documentation site of its own, and each
// of those sites opens by saying that rbgo *binds* a pure-Go library for it:
//
// go-ruby-set/set is the pure-Go library that rbgo binds for Ruby's set.
// rbgo's number reflects this pure-Go library doing the work.
//
// That sentence is a claim about this repository, and it stops being true the
// moment a module moves in-tree. It did: #330 reimplemented Set here for MRI 4.0
// conformance, #333's go mod tidy dropped the now-unused require the next day,
// and the page in the other org went on attributing rbgo's number to a library
// rbgo no longer used — for seven weeks, and with a figure that had moved by
// 3.7x underneath it.
//
// Nothing about that trade was wrong. What was missing was anything that would
// say it had happened. This test is that: it writes down, beside the benchmark
// scripts, which of them run on an external go-ruby library and which run on
// code in this repository, and fails when the file and go.mod disagree.
//
// Regenerate with: go test -run TestBenchmarkedModulesSayWhereTheirCodeLives -update-backing
func TestBenchmarkedModulesSayWhereTheirCodeLives(t *testing.T) {
scripts, err := filepath.Glob(filepath.Join("bench", "modules", "*.rb"))
if err != nil {
t.Fatalf("listing the benchmark scripts: %v", err)
}
if len(scripts) == 0 {
t.Fatal("no benchmark scripts found; this test would pass by measuring nothing")
}
gomod, err := os.ReadFile("go.mod")
if err != nil {
t.Fatalf("reading go.mod: %v", err)
}
required := requiredModules(string(gomod))
// A control: the file must name modules this repository really requires, or
// the parse is wrong and every row below would read "in-tree" for the wrong
// reason.
if len(required) < 50 {
t.Fatalf("go.mod parsed to %d requirements, which cannot be right", len(required))
}
names := make([]string, 0, len(scripts))
for _, s := range scripts {
names = append(names, strings.TrimSuffix(filepath.Base(s), ".rb"))
}
sort.Strings(names)
var b strings.Builder
b.WriteString("<!-- Generated by TestBenchmarkedModulesSayWhereTheirCodeLives. Do not edit by hand. -->\n")
b.WriteString("# What each benchmarked module runs on\n\n")
b.WriteString("`bench/modules/<m>.rb` exercises Ruby's `<m>`. This says whether rbgo serves\n")
b.WriteString("it from an external `go-ruby-<m>` library or from code in this repository —\n")
b.WriteString("which is what the per-library documentation sites claim, and what stops being\n")
b.WriteString("true when a module moves in-tree.\n\n")
b.WriteString("| Module | Runs on |\n| --- | --- |\n")
for _, m := range names {
path := fmt.Sprintf("github.com/go-ruby-%s/%s", m, m)
if required[path] {
fmt.Fprintf(&b, "| %s | `%s` |\n", m, path)
} else {
fmt.Fprintf(&b, "| %s | **in-tree** (this repository) |\n", m)
}
}
want := b.String()
out := filepath.Join("bench", "modules", "BACKING.md")
if *updateBacking {
if err := os.WriteFile(out, []byte(want), 0o644); err != nil {
t.Fatalf("writing %s: %v", out, err)
}
t.Logf("wrote %s", out)
return
}
got, err := os.ReadFile(out)
if err != nil {
t.Fatalf("reading %s (regenerate with -update-backing): %v", out, err)
}
// Compared without carriage returns: git hands this file to a Windows
// checkout with CRLF endings, and a guard about which module runs where must
// not fail over how the working tree was written. It did, on the first run of
// CI that saw it.
if unixEndings(string(got)) != unixEndings(want) {
t.Errorf("%s no longer matches go.mod.\n\nA module has moved between an external\n"+
"go-ruby library and this repository. Regenerate with:\n\n"+
" go test -run TestBenchmarkedModulesSayWhereTheirCodeLives -update-backing\n\n"+
"and then say so wherever that module's performance is published: the page\n"+
"in its own org opens by naming the library rbgo binds for it.\n\n%s",
out, firstDifference(unixEndings(string(got)), unixEndings(want)))
}
}
// unixEndings drops carriage returns so that the comparison is about the table
// and not about the checkout.
func unixEndings(s string) string { return strings.ReplaceAll(s, "\r", "") }
// requiredModules pulls the module paths out of go.mod's require blocks. It is
// deliberately a parse of the text rather than a call to `go list`: the test has
// to answer the same question a reader of go.mod would.
func requiredModules(gomod string) map[string]bool {
out := map[string]bool{}
inBlock := false
for _, line := range strings.Split(gomod, "\n") {
s := strings.TrimSpace(line)
if i := strings.Index(s, "//"); i >= 0 {
s = strings.TrimSpace(s[:i])
}
switch {
case s == "require (":
inBlock = true
continue
case inBlock && s == ")":
inBlock = false
continue
case strings.HasPrefix(s, "require "):
s = strings.TrimSpace(strings.TrimPrefix(s, "require "))
case !inBlock:
continue
}
if fields := strings.Fields(s); len(fields) >= 2 {
out[fields[0]] = true
}
}
return out
}
// firstDifference names the first line that differs, because a diff of a
// generated table is unreadable and the answer is always one row.
func firstDifference(got, want string) string {
g, w := strings.Split(got, "\n"), strings.Split(want, "\n")
for i := 0; i < len(g) || i < len(w); i++ {
gl, wl := "", ""
if i < len(g) {
gl = g[i]
}
if i < len(w) {
wl = w[i]
}
if gl != wl {
return fmt.Sprintf("first difference at line %d:\n file: %q\n go.mod: %q", i+1, gl, wl)
}
}
return "(the files differ only in trailing bytes)"
}