diff --git a/infino/README.md b/infino/README.md index 4ae3e30a7c..608772349f 100644 --- a/infino/README.md +++ b/infino/README.md @@ -25,10 +25,12 @@ queries, so `tuned=no`. ./install # installs rustup if absent, then cargo build --release ``` -The release build is `opt-level = 3`, `lto = "fat"`, `codegen-units = 1` -(`bench/Cargo.toml`), a portable build with no `-C target-cpu`, matching the -`datafusion` variant's recipe. Pinned to a published crate version -(`infino = "0.1.10"`) so the result is reproducible from crates.io alone. +The release build is portable (no `-C target-cpu`), pinned to a published crate +version (`infino = "0.5.3"`) so the result is reproducible from crates.io alone. +LTO is sized to the machine by `install`: fat LTO + `codegen-units = 1` at +>= 12 GiB RAM (its single-pass link peaks ~7.6 GB), thin LTO below so the small +VMs still build. Reads use strong consistency, so a query issued right after +load sees every ingested row. ## Run (automated ClickBench flow) @@ -45,7 +47,7 @@ seq 0 10 | xargs -P11 -I{} wget -q --continue \ INFINO_SRC="hits_*.parquet" INFINO_MAX_ROWS=10000000 ./load # ingest once ./start # launch the server -printf '%s\n' "SELECT COUNT(*) FROM hits" | ./query # row count + seconds +printf '%s\n' "SELECT COUNT(*) FROM hits" | ./query # result table + seconds ./stop # shut the server down ``` @@ -54,9 +56,11 @@ printf '%s\n' "SELECT COUNT(*) FROM hits" | ./query # row count + seco `serve` opens the table once and answers one query per connection on `INFINO_SOCK` (default `./infino.sock`). Timing wraps `query_sql` only, so the socket round-trip is never counted. `./query` sends stdin to the server and -prints the row count to stdout and the elapsed seconds to stderr (the -ClickBench query-script contract). `./check` pings the server so the driver can -detect it coming up and going down. +prints the query result as a text table to stdout (a bounded preview: up to +1000 rows and `CLICKBENCH_OUTPUT_LIMIT` bytes, so a huge result cannot blow up) +and the elapsed seconds to stderr (the ClickBench query-script contract). The +result on stdout is also what the online playground shows the user. `./check` +pings the server so the driver can detect it coming up and going down. ## Environment diff --git a/infino/bench/Cargo.lock b/infino/bench/Cargo.lock index ac1e228b50..48581ddf70 100644 --- a/infino/bench/Cargo.lock +++ b/infino/bench/Cargo.lock @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -60,9 +60,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -93,9 +93,9 @@ dependencies = [ [[package]] name = "ar_archive_writer" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" dependencies = [ "object", ] @@ -347,9 +347,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" dependencies = [ "compression-codecs", "compression-core", @@ -359,9 +359,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.91" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", @@ -444,9 +444,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" dependencies = [ "arrayref", "arrayvec", @@ -530,9 +530,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -1484,13 +1484,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -1548,9 +1548,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "fixedbitset" @@ -1854,9 +1854,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -1899,9 +1899,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -2105,9 +2105,9 @@ dependencies = [ [[package]] name = "infino" -version = "0.1.10" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8cd41e026c0a83816a02fe3f0982be2dd372b9b024dcd1017f857dd9a62c200" +checksum = "ea5d9748bffe8ce05ac249d73fc70105a6cc5a6ccb7552a5d17e6c2b55f1d329" dependencies = [ "apache-avro", "arc-swap", @@ -2147,7 +2147,7 @@ dependencies = [ "serde_json", "static_assertions", "tempfile", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", "unicode-segmentation", @@ -2171,9 +2171,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" [[package]] name = "itertools" @@ -2202,9 +2202,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -2517,9 +2517,9 @@ dependencies = [ [[package]] name = "object" -version = "0.37.3" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ "memchr", ] @@ -2555,7 +2555,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", "url", @@ -2789,9 +2789,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" dependencies = [ "ar_archive_writer", "cc", @@ -2827,7 +2827,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -2849,7 +2849,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.19", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -3048,9 +3048,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -3171,9 +3171,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "once_cell", "ring", @@ -3473,9 +3473,9 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" dependencies = [ "cc", "cfg-if", @@ -3581,11 +3581,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.20", ] [[package]] @@ -3601,9 +3601,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", @@ -3672,13 +3672,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -3942,9 +3942,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -3955,9 +3955,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" dependencies = [ "js-sys", "wasm-bindgen", @@ -3965,9 +3965,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3975,9 +3975,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", @@ -3988,9 +3988,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] @@ -4010,9 +4010,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -4030,9 +4030,9 @@ dependencies = [ [[package]] name = "wide" -version = "1.5.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfdfe6a32973f2d1b268b8895845a8a96cac2f0191e72c27cc929036060dbf89" +checksum = "de2aaf408e58689c2096682331b1f42bb2d9f2ed6b11560407d023cd0a6c634e" dependencies = [ "bytemuck", "safe_arch", @@ -4246,18 +4246,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", @@ -4326,9 +4326,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" [[package]] name = "zmij" diff --git a/infino/bench/Cargo.toml b/infino/bench/Cargo.toml index 66e4d5354f..f8ba202b7e 100644 --- a/infino/bench/Cargo.toml +++ b/infino/bench/Cargo.toml @@ -8,8 +8,8 @@ name = "clickbench-infino" path = "src/main.rs" [dependencies] -infino = "0.1.10" -arrow = "58" +infino = "0.5.3" +arrow = { version = "58", features = ["prettyprint"] } arrow-array = "58" arrow-schema = "58" parquet = "58" diff --git a/infino/bench/src/main.rs b/infino/bench/src/main.rs index df55cd2d9c..b804f3c2e4 100644 --- a/infino/bench/src/main.rs +++ b/infino/bench/src/main.rs @@ -36,7 +36,7 @@ use arrow_schema::{DataType, Field, Schema, SchemaRef}; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use infino::{ - connect, connect_with, CompactionSettings, ConnectOptions, IndexSpec, OptimizeOptions, + connect_with, CompactionSettings, ConnectOptions, Consistency, IndexSpec, OptimizeOptions, }; type R = Result>; @@ -52,35 +52,30 @@ fn sock_path() -> String { } fn open() -> R { - let mut opts = ConnectOptions::new(); - let mut custom = false; + // Strong read consistency. infino 0.5 changed the default from strong to + // BoundedStaleness(1s). ClickBench needs exact results, so pin strong everywhere. + let mut opts = ConnectOptions::new().with_read_consistency(Consistency::Strong); + for (k, v) in env::vars() { if let Some(key) = k.strip_prefix("INFINO_STORAGE_") { opts = opts.with_storage_option(key.to_lowercase(), v); - custom = true; } } if let Ok(dir) = env::var("INFINO_CACHE_DIR") { opts = opts.with_cache_dir(dir); - custom = true; } - // Raise the disk-cache budget above the 10 GiB default so a large corpus - // (e.g. 100M rows, tens of GB of superfiles) fits on a big disk instead of + // Raise the disk-cache budget above the default so a large corpus (e.g. + // 100M rows, tens of GB of superfiles) fits on a big disk instead of // thrashing / falling back to range-only reads. Bytes. if let Some(b) = env::var("INFINO_CACHE_BUDGET") .ok() .and_then(|s| s.parse::().ok()) { opts = opts.with_cache_budget_bytes(b); - custom = true; } - Ok(if custom { - connect_with(uri(), opts)? - } else { - connect(uri())? - }) + Ok(connect_with(uri(), opts)?) } /// Target arrow type for a source parquet field. infino queries its own table @@ -227,14 +222,17 @@ fn serve() -> R<()> { Ok(()) } -/// Handle one client: read the SQL, run it, write back one line -/// `OK ` on success -/// `ERR ` on failure +/// Handle one client. The response is a header line followed by the result: +/// `OK \n` on success +/// `ERR ` on failure /// The client sends the SQL then half-closes its write side; that EOF is what /// lets `read_to_string` return. The read timeout bounds a client that never -/// does, so one stuck caller cannot wedge the single-threaded server. Timing -/// wraps `query_sql` only — the socket read above and the write below are -/// excluded, so the reported seconds are pure query latency. +/// does, so one stuck caller cannot wedge the single-threaded server. +/// +/// Timing wraps `query_sql` only: the result is formatted AFTER `elapsed()`, so +/// the reported seconds stay pure query latency (unchanged from before). We +/// return the actual rows, not just a count, because the playground shows this +/// output to the user; a bounded preview keeps a huge result from blowing up. fn handle_conn(db: &infino::Connection, mut stream: UnixStream) -> R<()> { stream.set_read_timeout(Some(Duration::from_secs(300)))?; let mut sql = String::new(); @@ -246,43 +244,113 @@ fn handle_conn(db: &infino::Connection, mut stream: UnixStream) -> R<()> { let start = Instant::now(); let resp = match db.query_sql(sql) { Ok(batches) => { + let secs = start.elapsed().as_secs_f64(); let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - format!("OK {rows} {:.6}\n", start.elapsed().as_secs_f64()) + + format!("OK {rows} {secs:.6}\n{}", format_preview(&batches)) } Err(e) => format!("ERR {e}\n"), }; + stream.write_all(resp.as_bytes())?; + Ok(()) } -/// Send `sql` to the running server and return its single-line response. The -/// `shutdown(Write)` signals end-of-request (EOF) so the server's -/// `read_to_string` returns; without it both sides would block forever. +/// Byte cap on the result the server sends back, matching the playground's +/// `CLICKBENCH_OUTPUT_LIMIT` (its in-VM agent truncates at the same bound). +fn output_limit() -> usize { + env::var("CLICKBENCH_OUTPUT_LIMIT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(262_144) +} + +/// Pretty-print a bounded preview of the result as a text table (the boxed +/// style datafusion-cli / duckdb use). Caps both the number of rows rendered +/// and the byte size, so formatting never touches more than a preview even if +/// the query returns millions of rows. +fn format_preview(batches: &[RecordBatch]) -> String { + const MAX_PREVIEW_ROWS: usize = 1000; + let mut preview = Vec::new(); + let mut taken = 0; + + for b in batches { + if taken >= MAX_PREVIEW_ROWS { + break; + } + let n = (MAX_PREVIEW_ROWS - taken).min(b.num_rows()); + preview.push(b.slice(0, n)); + taken += n; + } + + let mut s = match arrow::util::pretty::pretty_format_batches(&preview) { + Ok(t) => t.to_string(), + Err(e) => format!(""), + }; + + let cap = output_limit(); + + if s.len() > cap { + let mut c = cap; + while c > 0 && !s.is_char_boundary(c) { + c -= 1; + } + s.truncate(c); + s.push_str("\n... (truncated)"); + } + s +} + +/// Send `sql` to the running server and return its full response: a header line +/// (`OK ` or `ERR `) followed, on success, by the +/// result table. The `shutdown(Write)` signals end-of-request (EOF) so the +/// server's `read_to_string` returns; without it both sides would block forever. fn ask(sql: &str) -> R { let mut stream = UnixStream::connect(sock_path())?; stream.write_all(sql.as_bytes())?; stream.shutdown(Shutdown::Write)?; + let mut resp = String::new(); stream.read_to_string(&mut resp)?; - Ok(resp.trim().to_string()) + + // Do not trim: the response body is the result table, whose whitespace and + // newlines are meaningful. Only the header line is parsed by the caller. + Ok(resp) } /// Client: read one SQL statement from stdin, send it to the server, print the -/// row count to stdout and the elapsed seconds to stderr (the ClickBench -/// query-script contract: last stderr line is fractional seconds). +/// result table to stdout and the elapsed seconds to stderr (the ClickBench +/// query-script contract: stdout is the query result, the last stderr line is +/// fractional seconds). The result on stdout is also what the playground shows +/// the user. fn query() -> R<()> { let mut sql = String::new(); std::io::stdin().read_to_string(&mut sql)?; let resp = ask(&sql)?; - if let Some(rest) = resp.strip_prefix("OK ") { - let mut it = rest.split_whitespace(); - let rows: usize = it.next().unwrap_or("0").parse().unwrap_or(0); - let secs = it.next().unwrap_or("0"); - println!("{rows} rows"); + let (header, body) = resp.split_once('\n').unwrap_or((resp.as_str(), "")); + + if let Some(rest) = header.strip_prefix("OK ") { + // header: "OK ". `rows` is the full count; the body + // table is a preview (up to MAX_PREVIEW_ROWS), so the footer shows the + // real total, which also signals when the table was row-capped. + let mut fields = rest.split_whitespace(); + let rows = fields.next().unwrap_or("0"); + let secs = fields.next().unwrap_or("0"); + print!("{body}"); + if !body.is_empty() && !body.ends_with('\n') { + println!(); + } + println!("({rows} rows)"); eprintln!("{secs}"); Ok(()) } else { - Err(resp.strip_prefix("ERR ").unwrap_or(&resp).to_string().into()) + Err(header + .strip_prefix("ERR ") + .unwrap_or(header) + .trim() + .to_string() + .into()) } } @@ -306,9 +374,7 @@ fn main() { "serve" => serve(), "query" => query(), "check" => check(), - other => { - Err(format!("unknown subcommand {other:?} (want load|serve|query|check)").into()) - } + other => Err(format!("unknown subcommand {other:?} (want load|serve|query|check)").into()), }; if let Err(e) = result { eprintln!("{e}"); diff --git a/infino/benchmark.sh b/infino/benchmark.sh index e7712624f0..0ec46d6f11 100755 --- a/infino/benchmark.sh +++ b/infino/benchmark.sh @@ -15,19 +15,8 @@ export BENCH_DURABLE=yes # oversubscribe it. Skip by default. export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" -# Disk cache lives beside the data so warm tries read cached column chunks. -export INFINO_CACHE_DIR="${INFINO_CACHE_DIR:-./cache}" - -# Superfile segment size: many mid-size segments let the scan parallelise -# across all cores. 256 MB fits a 16-core box on up. -export INFINO_TARGET_SF_MB="${INFINO_TARGET_SF_MB:-256}" - -# Disk-cache budget sized to the machine so the whole dataset stays resident -# (the 10 GiB default range-reads a >10 GiB dataset). ~75% of RAM: portable -# across machines, overridable. Memory sizing, not per-query tuning. -if [ -z "${INFINO_CACHE_BUDGET:-}" ] && [ -r /proc/meminfo ]; then - ram_kb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo) - export INFINO_CACHE_BUDGET=$(( ram_kb * 1024 * 3 / 4 )) -fi +# infino tuning (cache dir + budget, superfile segment size). Shared with the +# raw load/start scripts so the playground gets the same config. +. "$(dirname "$0")/config.sh" exec ../lib/benchmark-common.sh diff --git a/infino/config.sh b/infino/config.sh new file mode 100644 index 0000000000..2d16bd35e1 --- /dev/null +++ b/infino/config.sh @@ -0,0 +1,20 @@ +# Shared infino configuration, sourced by load, start, and benchmark.sh, so the +# raw scripts (which the playground runs directly, without benchmark.sh) get the +# same tuning as the benchmark. Everything is a :- fallback, so an explicit env +# from the caller still wins. + +# Disk cache lives beside the data so warm queries read cached column chunks. +export INFINO_CACHE_DIR="${INFINO_CACHE_DIR:-./cache}" + +# Superfile segment size: many mid-size segments let the scan parallelise across +# cores. 256 MB fits a 16-core box on up. Without this the default ~1 GiB +# segments leave queries single-file-bound (seconds instead of sub-second). +export INFINO_TARGET_SF_MB="${INFINO_TARGET_SF_MB:-256}" + +# Disk-cache budget: 24 GiB, comfortably above the ~11 GB (100M-row) dataset so +# every superfile stays cached (the DiskCacheStore holds them mmap-backed) and +# warm queries are fast. A fixed size, not a RAM fraction: the cache lives on +# the 200 GB sysdisk, and actual use is capped by the ~11 GB dataset, which fits +# in RAM on every ClickBench machine (>= 16 GB). The default (10 GiB) is below +# the dataset, so it range-reads and queries turn slow. +export INFINO_CACHE_BUDGET="${INFINO_CACHE_BUDGET:-25769803776}" # 24 GiB diff --git a/infino/load b/infino/load index 0a37cb571f..4f7ec5dc4e 100755 --- a/infino/load +++ b/infino/load @@ -1,6 +1,10 @@ #!/bin/bash set -e +# Tuning (256 MB segments, RAM-sized cache) so the raw playground load matches +# the benchmark; benchmark.sh sources this too. +. "$(dirname "$0")/config.sh" + # Idempotent: blow away any prior local catalog (remote backends are dropped # in-binary via drop_table). rm -rf data diff --git a/infino/query b/infino/query index 7c1e067300..89799d2a68 100755 --- a/infino/query +++ b/infino/query @@ -1,6 +1,7 @@ #!/bin/bash # Client: reads a SQL query from stdin and sends it to the running infino -# server (started by ./start). Stdout: row count. Stderr: query runtime in -# fractional seconds on the last line (the ClickBench query-script contract). +# server (started by ./start). Stdout: the query result as a text table (also +# what the playground shows the user). Stderr: query runtime in fractional +# seconds on the last line (the ClickBench query-script contract). set -e exec ./bench/target/release/clickbench-infino query diff --git a/infino/start b/infino/start index 6a04b49b5a..50e2a869af 100755 --- a/infino/start +++ b/infino/start @@ -4,6 +4,9 @@ # driver's ./check loop finds a live server. set -e +# Same tuning as the load (cache dir + budget); benchmark.sh sources this too. +. "$(dirname "$0")/config.sh" + BIN=./bench/target/release/clickbench-infino rm -f infino.sock