|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "archive/tar" |
| 5 | + "compress/gzip" |
| 6 | + "crypto/sha512" |
| 7 | + "encoding/hex" |
| 8 | + "errors" |
| 9 | + "fmt" |
| 10 | + "io" |
| 11 | + "log" |
| 12 | + "net" |
| 13 | + "os" |
| 14 | + "os/exec" |
| 15 | + "path/filepath" |
| 16 | + "runtime" |
| 17 | + "strings" |
| 18 | + "time" |
| 19 | +) |
| 20 | + |
| 21 | +const ( |
| 22 | + // clickhouseVersion is the ClickHouse release to install. It is the |
| 23 | + // release internal/goldeneye generates the dialect from, and the binary |
| 24 | + // is cached where goldeneye caches it, so the two share one download. |
| 25 | + clickhouseVersion = "25.8.2.29" |
| 26 | + |
| 27 | + // clickhousePassword is the password the default user is given: the |
| 28 | + // one docker-compose.yml and CI use. |
| 29 | + clickhousePassword = "mysecretpassword" |
| 30 | + |
| 31 | + clickhouseTCPPort = "9000" |
| 32 | + clickhouseHTTPPort = "8123" |
| 33 | +) |
| 34 | + |
| 35 | +// clickhouseAsset is one downloadable build of ClickHouse: a |
| 36 | +// clickhouse-common-static tarball holding the binary at usr/bin/clickhouse, |
| 37 | +// with the SHA-512 ClickHouse publishes next to it. |
| 38 | +type clickhouseAsset struct { |
| 39 | + URL string |
| 40 | + SHA512 string |
| 41 | +} |
| 42 | + |
| 43 | +var clickhouseAssets = map[string]clickhouseAsset{ |
| 44 | + "linux/amd64": { |
| 45 | + URL: "https://github.com/ClickHouse/ClickHouse/releases/download/v" + clickhouseVersion + "-lts/clickhouse-common-static-" + clickhouseVersion + "-amd64.tgz", |
| 46 | + SHA512: "6ff0aa1ffac6e564970174422ecde0d645cdb96812247a6e544d39cad6d78a514265f90a2bc7b4bad49903cea96eddd16a415a45b2aeaf9164461be76331bdee", |
| 47 | + }, |
| 48 | + "linux/arm64": { |
| 49 | + URL: "https://github.com/ClickHouse/ClickHouse/releases/download/v" + clickhouseVersion + "-lts/clickhouse-common-static-" + clickhouseVersion + "-arm64.tgz", |
| 50 | + SHA512: "68204ca4d4e472790f808ee376251fae82e58066a31f35a40d15d442ce5988d697f18a1208d28b8bb8e2dfad4b20b7fcb5107e2178472abcd97251b8de7f058e", |
| 51 | + }, |
| 52 | +} |
| 53 | + |
| 54 | +// clickhouseDir is where the release is cached, and the server's |
| 55 | +// configuration and data are kept under it. |
| 56 | +func clickhouseDir() (string, error) { |
| 57 | + cache, err := os.UserCacheDir() |
| 58 | + if err != nil { |
| 59 | + return "", err |
| 60 | + } |
| 61 | + return filepath.Join(cache, "sqlc-clickhouse", clickhouseVersion), nil |
| 62 | +} |
| 63 | + |
| 64 | +func clickhouseBinary(dir string) string { |
| 65 | + return filepath.Join(dir, "clickhouse") |
| 66 | +} |
| 67 | + |
| 68 | +// installClickHouse downloads the ClickHouse release for this platform and |
| 69 | +// unpacks its binary into the cache, checking the download against the |
| 70 | +// pinned SHA-512. It is a no-op when the binary is already there, and |
| 71 | +// skips platforms the release is not published for. |
| 72 | +func installClickHouse() error { |
| 73 | + log.Printf("--- Installing ClickHouse %s ---", clickhouseVersion) |
| 74 | + |
| 75 | + platform := runtime.GOOS + "/" + runtime.GOARCH |
| 76 | + asset, ok := clickhouseAssets[platform] |
| 77 | + if !ok { |
| 78 | + log.Printf("clickhouse is not published for %s, skipping", platform) |
| 79 | + return nil |
| 80 | + } |
| 81 | + |
| 82 | + dir, err := clickhouseDir() |
| 83 | + if err != nil { |
| 84 | + return err |
| 85 | + } |
| 86 | + binary := clickhouseBinary(dir) |
| 87 | + if _, err := os.Stat(binary); err == nil { |
| 88 | + log.Printf("clickhouse %s is already installed at %s", clickhouseVersion, binary) |
| 89 | + return nil |
| 90 | + } |
| 91 | + if err := os.MkdirAll(dir, 0o755); err != nil { |
| 92 | + return err |
| 93 | + } |
| 94 | + |
| 95 | + archive := filepath.Join(dir, "clickhouse.tgz") |
| 96 | + log.Printf("downloading %s", asset.URL) |
| 97 | + if err := downloadFile(archive, asset.URL); err != nil { |
| 98 | + return fmt.Errorf("downloading clickhouse: %w", err) |
| 99 | + } |
| 100 | + defer os.Remove(archive) |
| 101 | + |
| 102 | + sum, err := sha512File(archive) |
| 103 | + if err != nil { |
| 104 | + return err |
| 105 | + } |
| 106 | + if sum != asset.SHA512 { |
| 107 | + return fmt.Errorf("clickhouse download has SHA-512 %s, want %s", sum, asset.SHA512) |
| 108 | + } |
| 109 | + |
| 110 | + log.Printf("unpacking the binary into %s", binary) |
| 111 | + if err := extractClickHouse(archive, binary); err != nil { |
| 112 | + return fmt.Errorf("unpacking clickhouse: %w", err) |
| 113 | + } |
| 114 | + return nil |
| 115 | +} |
| 116 | + |
| 117 | +// extractClickHouse copies usr/bin/clickhouse out of the release tarball. |
| 118 | +func extractClickHouse(archive, binary string) error { |
| 119 | + f, err := os.Open(archive) |
| 120 | + if err != nil { |
| 121 | + return err |
| 122 | + } |
| 123 | + defer f.Close() |
| 124 | + gz, err := gzip.NewReader(f) |
| 125 | + if err != nil { |
| 126 | + return err |
| 127 | + } |
| 128 | + tr := tar.NewReader(gz) |
| 129 | + for { |
| 130 | + hdr, err := tr.Next() |
| 131 | + if errors.Is(err, io.EOF) { |
| 132 | + return errors.New("tarball does not contain usr/bin/clickhouse") |
| 133 | + } |
| 134 | + if err != nil { |
| 135 | + return err |
| 136 | + } |
| 137 | + if hdr.Typeflag != tar.TypeReg || !strings.HasSuffix(hdr.Name, "/usr/bin/clickhouse") { |
| 138 | + continue |
| 139 | + } |
| 140 | + tmp, err := os.CreateTemp(filepath.Dir(binary), "clickhouse-*.partial") |
| 141 | + if err != nil { |
| 142 | + return err |
| 143 | + } |
| 144 | + if _, err := io.Copy(tmp, tr); err != nil { |
| 145 | + tmp.Close() |
| 146 | + os.Remove(tmp.Name()) |
| 147 | + return err |
| 148 | + } |
| 149 | + if err := tmp.Close(); err != nil { |
| 150 | + os.Remove(tmp.Name()) |
| 151 | + return err |
| 152 | + } |
| 153 | + if err := os.Chmod(tmp.Name(), 0o755); err != nil { |
| 154 | + os.Remove(tmp.Name()) |
| 155 | + return err |
| 156 | + } |
| 157 | + return os.Rename(tmp.Name(), binary) |
| 158 | + } |
| 159 | +} |
| 160 | + |
| 161 | +// sha512File computes the SHA-512 hash of a file and returns the hex string. |
| 162 | +func sha512File(path string) (string, error) { |
| 163 | + f, err := os.Open(path) |
| 164 | + if err != nil { |
| 165 | + return "", err |
| 166 | + } |
| 167 | + defer f.Close() |
| 168 | + h := sha512.New() |
| 169 | + if _, err := io.Copy(h, f); err != nil { |
| 170 | + return "", err |
| 171 | + } |
| 172 | + return hex.EncodeToString(h.Sum(nil)), nil |
| 173 | +} |
| 174 | + |
| 175 | +// clickhouseConfig is the server's configuration: it listens on localhost |
| 176 | +// only, keeps its data next to the configuration, and takes its users from |
| 177 | +// users.xml beside it. |
| 178 | +const clickhouseConfig = `<clickhouse> |
| 179 | + <logger> |
| 180 | + <level>warning</level> |
| 181 | + <console>1</console> |
| 182 | + </logger> |
| 183 | + <path>./data/</path> |
| 184 | + <tcp_port>` + clickhouseTCPPort + `</tcp_port> |
| 185 | + <http_port>` + clickhouseHTTPPort + `</http_port> |
| 186 | + <listen_host>127.0.0.1</listen_host> |
| 187 | + <users_config>users.xml</users_config> |
| 188 | + <user_directories> |
| 189 | + <users_xml> |
| 190 | + <path>users.xml</path> |
| 191 | + </users_xml> |
| 192 | + </user_directories> |
| 193 | +</clickhouse> |
| 194 | +` |
| 195 | + |
| 196 | +// clickhouseUsers gives the default user a password. Without one the |
| 197 | +// server restricts the user to localhost, which is fine here, but the |
| 198 | +// tests, docker-compose.yml and CI all connect with the same password. |
| 199 | +const clickhouseUsers = `<clickhouse> |
| 200 | + <profiles> |
| 201 | + <default/> |
| 202 | + </profiles> |
| 203 | + <quotas> |
| 204 | + <default/> |
| 205 | + </quotas> |
| 206 | + <users> |
| 207 | + <default> |
| 208 | + <password>` + clickhousePassword + `</password> |
| 209 | + <networks> |
| 210 | + <ip>::/0</ip> |
| 211 | + </networks> |
| 212 | + <profile>default</profile> |
| 213 | + <quota>default</quota> |
| 214 | + </default> |
| 215 | + </users> |
| 216 | +</clickhouse> |
| 217 | +` |
| 218 | + |
| 219 | +// startClickHouse starts a ClickHouse server in the background, serving |
| 220 | +// on localhost, and waits until it answers a query. It is a no-op when a |
| 221 | +// server already does, and skips platforms the release is not installed |
| 222 | +// on. |
| 223 | +func startClickHouse() error { |
| 224 | + log.Println("--- Starting ClickHouse ---") |
| 225 | + |
| 226 | + dir, err := clickhouseDir() |
| 227 | + if err != nil { |
| 228 | + return err |
| 229 | + } |
| 230 | + binary := clickhouseBinary(dir) |
| 231 | + if _, err := os.Stat(binary); err != nil { |
| 232 | + if _, ok := clickhouseAssets[runtime.GOOS+"/"+runtime.GOARCH]; !ok { |
| 233 | + log.Printf("clickhouse is not published for %s/%s, skipping", runtime.GOOS, runtime.GOARCH) |
| 234 | + return nil |
| 235 | + } |
| 236 | + return fmt.Errorf("clickhouse is not installed: run `sqlc-test-setup install clickhouse` first") |
| 237 | + } |
| 238 | + |
| 239 | + if clickhouseReady(binary) { |
| 240 | + log.Println("clickhouse is already running and accepting connections") |
| 241 | + return nil |
| 242 | + } |
| 243 | + |
| 244 | + server := filepath.Join(dir, "server") |
| 245 | + if err := os.MkdirAll(filepath.Join(server, "data"), 0o755); err != nil { |
| 246 | + return err |
| 247 | + } |
| 248 | + if err := os.WriteFile(filepath.Join(server, "config.xml"), []byte(clickhouseConfig), 0o644); err != nil { |
| 249 | + return err |
| 250 | + } |
| 251 | + if err := os.WriteFile(filepath.Join(server, "users.xml"), []byte(clickhouseUsers), 0o644); err != nil { |
| 252 | + return err |
| 253 | + } |
| 254 | + logFile, err := os.Create(filepath.Join(server, "server.log")) |
| 255 | + if err != nil { |
| 256 | + return err |
| 257 | + } |
| 258 | + defer logFile.Close() |
| 259 | + |
| 260 | + cmd := exec.Command(binary, "server", "--config-file="+filepath.Join(server, "config.xml")) |
| 261 | + cmd.Dir = server |
| 262 | + cmd.Stdout = logFile |
| 263 | + cmd.Stderr = logFile |
| 264 | + cmd.SysProcAttr = detachedProcess() |
| 265 | + log.Printf("starting %s server in %s", binary, server) |
| 266 | + if err := cmd.Start(); err != nil { |
| 267 | + return fmt.Errorf("starting clickhouse: %w", err) |
| 268 | + } |
| 269 | + if err := cmd.Process.Release(); err != nil { |
| 270 | + return err |
| 271 | + } |
| 272 | + |
| 273 | + log.Println("waiting for clickhouse to accept connections") |
| 274 | + deadline := time.Now().Add(2 * time.Minute) |
| 275 | + for time.Now().Before(deadline) { |
| 276 | + if clickhouseReady(binary) { |
| 277 | + log.Println("clickhouse is accepting connections") |
| 278 | + return nil |
| 279 | + } |
| 280 | + time.Sleep(time.Second) |
| 281 | + } |
| 282 | + return fmt.Errorf("clickhouse did not start in time (see %s)", logFile.Name()) |
| 283 | +} |
| 284 | + |
| 285 | +// clickhouseReady reports whether a server answers on the TCP port with |
| 286 | +// the expected password. |
| 287 | +func clickhouseReady(binary string) bool { |
| 288 | + conn, err := net.DialTimeout("tcp", "127.0.0.1:"+clickhouseTCPPort, time.Second) |
| 289 | + if err != nil { |
| 290 | + return false |
| 291 | + } |
| 292 | + conn.Close() |
| 293 | + return exec.Command(binary, "client", "--host", "127.0.0.1", "--port", clickhouseTCPPort, |
| 294 | + "--password", clickhousePassword, "-q", "SELECT 1").Run() == nil |
| 295 | +} |
0 commit comments