diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..846b5fe --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,23 @@ +# One shared build directory for all 45 crates in this repo, placed OUTSIDE +# the repo root — as a sibling of your clone, not inside it. +# +# Two reasons it lives outside rather than in ./target: +# +# 1. mdbook's source directory is the repo root (`src = "."` in book.toml), +# and mdbook copies every non-markdown file under src into the rendered +# book. It has no exclude mechanism and does not honour .gitignore, so a +# target/ dir anywhere under the root gets duplicated into book/ on every +# build. That is how book/ once reached 7.2 GB. +# 2. The 44 topic crates are deliberately independent packages, not workspace +# members, so each one would otherwise build its own copy of rand, +# criterion and friends. Sharing one directory means sharing those builds: +# it is the difference between 6.6 GB and about 1.5 GB, and it makes +# ./verify.sh's cold run several times faster. +# +# Nothing here changes how any individual crate builds — `cargo run`, +# `cargo test` and `cargo bench` all work exactly as the guides describe from +# inside any topics/*/experiments directory. If you would rather keep artifacts +# inside your clone, delete this file; just run `cargo clean` before +# `mdbook build` if you do. +[build] +target-dir = "../.dlp-target" diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 0cf32c3..aa677a8 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -1,17 +1,25 @@ name: book +# The book is built on pull requests as well as on master, because the two ways +# it breaks — a mermaid diagram that no longer parses, and a SUMMARY.md entry +# pointing at a moved file — are both invisible in markdown and obvious in the +# rendered output. Catching those after the merge means catching them after they +# have already deployed to Pages. +# +# Only a push to master deploys. Pull requests build and stop. + on: push: branches: [master] + pull_request: workflow_dispatch: +# Least privilege by default; the deploy job elevates for itself. permissions: contents: read - pages: write - id-token: write concurrency: - group: pages + group: book-${{ github.ref }} cancel-in-progress: true jobs: @@ -42,13 +50,52 @@ jobs: - name: Bundle PDF into site run: cp book/pdf/output.pdf book/html/database-learning-path.pdf + # Every chapter in SUMMARY.md must have produced a page, and every mermaid + # block must have reached the renderer as a mermaid block rather than as a + # plain code fence. Both are silent failures in markdown and obvious in the + # rendered output, which is the whole reason to check here. + - name: Check the rendered book + run: | + fail=0 + checked=0 + while read -r page; do + # mdbook renders README.md as the directory's index.html + case "$page" in + */README.md) html="book/html/${page%README.md}index.html" ;; + README.md) html="book/html/index.html" ;; + *) html="book/html/${page%.md}.html" ;; + esac + checked=$((checked + 1)) + if [ ! -f "$html" ]; then + echo "::error::SUMMARY.md lists $page but $html was not rendered" + fail=1 + fi + done < <(grep -oE '\]\(([^)]+\.md)\)' SUMMARY.md | sed 's/](//; s/)//' | sort -u) + echo "checked $checked chapters from SUMMARY.md" + + # print.html concatenates every page, so it is excluded or every + # diagram would be counted twice. + src=$(grep -rho '^```mermaid' --include='*.md' . --exclude-dir=book --exclude-dir=drafts | wc -l | tr -d ' ') + out=$(grep -rho 'class="mermaid"' book/html --include='*.html' --exclude='print.html' | wc -l | tr -d ' ') + echo "mermaid blocks: $src in source, $out rendered" + if [ "$out" -lt "$src" ]; then + echo "::error::$((src - out)) mermaid block(s) did not render as mermaid" + fail=1 + fi + exit $fail + - uses: actions/upload-pages-artifact@v3 + if: github.event_name == 'push' with: path: book/html deploy: needs: build + if: github.event_name == 'push' runs-on: ubuntu-latest + permissions: + pages: write + id-token: write environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..ab060da --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,92 @@ +name: verify + +# The repo's central claim is that every performance figure in the guides comes +# from code in the repo that you can run. This workflow is what keeps that +# claim honest: it runs the same ./verify.sh a reader would, so a lane that +# stops building or stops running fails the build instead of quietly rotting. +# +# Note on `cargo test`: it is NOT the gate here. Each topic ships two exercise +# lanes as `todo!()` stubs whose tests are the specification, so a red test +# suite is the intended state of a fresh clone. The bin lanes in verify.sh are +# the part that must always run. + +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: verify-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + # a cold full run is ~20 min of compiling plus ~15 min of benchmarks + timeout-minutes: 75 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/.ci-target + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + # Each job sets CARGO_TARGET_DIR to an absolute in-workspace path, which + # takes precedence over .cargo/config.toml's ../.dlp-target. Necessary + # because actions/cache rejects any path containing ".." — it logs + # "Invalid pattern" and then silently caches nothing, so every run + # recompiles all 45 crates from scratch. The reason the checked-in config + # points outside the clone is mdbook, which never runs in this workflow. + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + ${{ github.workspace }}/.ci-target + key: verify-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: verify-${{ runner.os }}- + + - name: Every measured lane must build and run + run: ./verify.sh --summary + + warnings: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/.ci-target + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + ${{ github.workspace }}/.ci-target + key: warn-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: warn-${{ runner.os }}- + + # rustc warnings only — deliberately NOT clippy. A reader's first + # experience of a topic crate should not be a wall of noise on top of the + # intentional todo!()s, and stub bodies that legitimately ignore their + # arguments carry a local #[allow] with a reason. Style lints across 44 + # independent teaching crates are a different argument, not this gate. + - name: No rustc warnings in the experiment crates + env: + RUSTFLAGS: -D warnings + run: | + fail=0 + for d in topics/*/experiments capstone; do + echo "::group::$d" + (cd "$d" && cargo build --all-targets --quiet) || fail=1 + echo "::endgroup::" + done + exit $fail diff --git a/.gitignore b/.gitignore index f52323f..acbd3c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ target/ -Cargo.lock .DS_Store /book/ /mermaid.min.js diff --git a/CLAUDE.md b/CLAUDE.md index 75c5065..8c44b8f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,12 +15,16 @@ A self-paced database-internals learning path, rendered as an mdBook (`book.toml ## Content rules - **Never assert a number you did not verify.** Download and read the actual paper; cite the section or table it came from. If a figure cannot be checked against the source, it does not go in. -- **Every performance claim must come from code in this repo.** Write the experiment, run it, and record the measurement. `./verify.sh` runs every measured benchmark lane — a new topic's lane belongs in its `BENCHES` list. -- **Report the negative result.** If a published technique does not reproduce on the local generator, that is the finding: say so, explain which premise is absent, and add an exercise that constructs the case where it holds. (Topic 42's multi-hit booster is the worked example.) -- **Generators are seeded**, so every figure reproduces exactly apart from timings. +- **Every performance claim must come from code in this repo.** Write the experiment, run it, and record the measurement. `./verify.sh` runs every measured lane — a new topic's lane belongs in its `BENCHES` list, and the headline belongs in `FINDINGS.md`. Two topics (4, 10) deliberately have no lane because their benches measure only the reader's code; say so in the README rather than inventing a number. +- **A benchmark that prints an implausible number is a bug in the benchmark.** Topic 12 once reported 19,047,619 GB/s from a hoisted timing loop. `black_box` the inputs, and sanity-check any figure against the hardware's actual limits before recording it. +- **Report the negative result.** If a published technique does not reproduce on the local generator, that is the finding: say so, explain which premise is absent, and add an exercise that constructs the case where it holds. (Topic 42's multi-hit booster is the worked example; topic 3's non-step-function height ladder is another.) +- **Generators are seeded**, so every figure reproduces exactly apart from timings. Lockfiles are committed for the same reason. +- **Exercise lanes must degrade, not crash.** A bench binary on a fresh clone prints its provided lanes and a `[stub — ...]` note for the rest, and exits 0. Never let a `todo!()` panic hide a measurement above it. ## Topic package shape -Each `topics/NN-name/` contains: `README.md` (study guide, opening with *the problem, measured* — the provided benchmark lane's real output), four to seven `reading-*.md` guides in the concept-first format (framing lead → "the problem in one sentence" → numbered `### Step N` sections → how to read the source → questions → done-when → references), `notes.md` (predictions vs measurements, paper numbers, cross-topic threads, open questions), and `experiments/` — a Rust crate with **lane 1 implemented and two lanes stubbed**, where the stub tests are the specification and the reference numbers live in `notes.md`. +Each `topics/NN-name/` contains: `README.md` (study guide, opening with *the problem, measured* — the provided benchmark lane's real output), four to seven `reading-*.md` guides in the concept-first format (framing lead → "the problem in one sentence" → numbered `### Step N` sections → how to read the source → questions → `## Done when` checklist → references), `notes.md` (a `## Baseline (provided lane, , measured )` section recording the real output, *then* the reader's prediction worksheet — leave those cells empty, they are the exercise), and `experiments/` — a Rust crate with **lane 1 implemented and two lanes stubbed**, where the stub tests are the specification and the reference numbers live in `notes.md`. -When adding a topic: PLAN.md section, the package above, a capstone `M`NN milestone row in PROGRESS.md, SUMMARY.md entries whose link titles match the on-disk H1s exactly, a SESSION-LOG.md entry carrying every measured number, and one commit. +When adding a topic: PLAN.md section, the package above, a `FINDINGS.md` row, a `verify.sh` lane, a capstone `M`NN milestone row in PROGRESS.md, SUMMARY.md entries whose link titles match the on-disk H1s exactly, a SESSION-LOG.md entry (with a `## date — topic NN — title` heading) carrying every measured number, and one commit. + +Reference clones live in `~/repos`; the commit each was read at is recorded once in the pin table at the end of `resources/codebases.md` — regenerate with `python3 tools/pin-table.py`, never hand-edit it. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 089546d..675f938 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,12 +44,22 @@ Every topic follows the same shape, because the shape is what makes it checkable then `### Step N` sections that build each concept using only terms defined in earlier steps, then how to read the source material with the concepts in hand, questions to answer, a "done when" checklist, and references. -- **`notes.md`** — a predictions-vs-measurements table (write the prediction *before* - running the benchmark), the paper numbers worth keeping, worked cross-topic threads, +- **`notes.md`** — a `## Baseline (provided lane, , measured )` section + recording the provided lane's real output with the analysis, then a + predictions-vs-measurements worksheet. **The worksheet's cells are meant to be + empty**: they are the reader's exercise, filled in before running the benchmark, not + a gap to be backfilled. Then the paper numbers worth keeping, cross-topic threads, and open questions. - **`experiments/`** — a Rust crate with **lane 1 implemented** and **two lanes stubbed**. The stub tests are the specification; the reference numbers live in - `notes.md`. + `notes.md`. A bench binary must print its provided lanes and a `[stub — ...]` note + for the unimplemented ones, then exit 0 — a `todo!()` panic must never take down a + measurement above it. + +Two topics (4 and 10) deliberately have no provided lane, because their benchmarks +measure only the reader's own implementation. Their READMEs open by saying so and by +giving the arithmetic or the external oracle to predict against instead. That is a +legitimate shape for a topic; inventing a number to fill the slot is not. ## Conventions @@ -65,8 +75,11 @@ Every topic follows the same shape, because the shape is what makes it checkable and add an exercise to construct the case where it holds. (See topic 42's multi-hit booster for the worked example.) - **Code reading is done against pinned clones** under `~/repos/` rather than vendored - here, with the commit recorded in the guide so the `file:line` anchors mean - something. + here. The commit each clone was read at is recorded **once**, in the pin table at + the end of [resources/codebases.md](resources/codebases.md), so the thousands of + `file:line` anchors in the guides mean something. Regenerate it with + `python3 tools/pin-table.py` after cloning or updating a reference repo — putting + a SHA in each guide instead would mean thousands of them drifting separately. - **Generators are seeded.** Anyone must be able to reproduce a figure exactly. - **Notes capture *why* a design wins** and what it trades away — not summaries. @@ -83,11 +96,25 @@ every push to `master` and deploys to GitHub Pages. Before committing content, b locally and check that mermaid diagrams render and internal links resolve — a broken link is invisible in markdown and obvious in the book. +A second workflow ([verify.yml](.github/workflows/verify.yml)) runs +`./verify.sh --summary` and a `-D warnings` build of all 45 crates on every push and +pull request. It is the gate that keeps the repo's central claim true, so a lane that +stops running is a red build. Note that `cargo test` is deliberately **not** a gate: +the stub tests are the specification and are supposed to fail on a fresh clone. + +One mdbook wrinkle worth knowing: `src = "."`, so mdbook copies every non-markdown +file under the repo root into `book/`. It honours neither `.gitignore` nor any exclude +list, which is why cargo artifacts are pushed outside the clone by +[.cargo/config.toml](.cargo/config.toml). If you remove that file, run `cargo clean` +before `mdbook build` or you will copy gigabytes of build output into the book. + ## Running the experiments ```bash ./verify.sh # every measured lane, with output ./verify.sh --summary # just the pass/fail table +./verify.sh --list # every lane and what it measures, run nothing +./verify.sh --criterion # also the slow criterion lanes (topic 0) ./verify.sh 40 41 # only these topics cd topics/40-security-attack-graphs/experiments diff --git a/FINDINGS.md b/FINDINGS.md new file mode 100644 index 0000000..473bcde --- /dev/null +++ b/FINDINGS.md @@ -0,0 +1,139 @@ +# Findings + +One measured headline per topic, and the command that re-derives it. This is the +whole argument for this format over a reading list in one table: a link +collection cannot be wrong in a way you can detect, and every row below can. + +Every figure here comes from a benchmark in this repo, measured on an **Apple M3 +Pro (5P + 6E, 36 GB)** on 2026-07-28. Generators are seeded, so counts, +ratios and distributions reproduce exactly; timings will differ on your +hardware. Run everything with `./verify.sh`, one topic with `./verify.sh 12`, or +`./verify.sh --list` to see every lane. + +Two topics have no row: **4 (LSM deep dive)** and **10 (query planning)** are the +two whose benchmarks measure only *your* implementation, so there is nothing to +report on a fresh clone. Their READMEs say so and explain what to predict +instead. + +| # | Topic | Measured finding | Lane | +|---|---|---|---| +| 0 | [Performance Toolbox](topics/00-performance-toolbox/README.md) | The DRAM latency ladder verified at ~1 / 5 / 100 ns — and `cache_ladder` measured its own 8 MB working set until the pointer chase was fixed to carry state. **21%** of a HashMap lookup is SipHash. | `./verify.sh --criterion 00` | +| 1 | [Storage Engine Landscape](topics/01-storage-engine-landscape/README.md) | Same 108 MB of records: fjall (LSM) writes **48 MB**, redb (CoW B-tree) writes **6.8 GB** — space amp **0.45× vs 63.28×**, a 140× spread. | `./verify.sh 01` | +| 2 | [In-Memory Structures](topics/02-in-memory-structures/README.md) | hashbrown insert: p50 **42 ns**, max **58.4 ms**. A 1.4-millionfold spread inside one operation, invisible to any throughput number. | `./verify.sh 02` | +| 3 | [B-Tree Internals](topics/03-btree-internals/README.md) | The "height is the metric" story fails: lookups climb **862 → 1101 ns** from 1e6 to 4e6 keys while height stays at 3. Height sets pages touched; cache residency sets what a touch costs. | `./verify.sh 03` | +| 5 | [Durability & WAL](topics/05-durability-wal/README.md) | `write()` **857k/s**, `fsync` **44k/s**, `F_FULLFSYNC` **337/s** — a 2540× spread, and only the last is durable on this drive. | `./verify.sh 05` | +| 6 | [Buffer Pool](topics/06-buffer-pool/README.md) | mmap page reads: p50 **42 ns**, max **182 µs**. A 4300× spread, entirely minor page faults the database cannot see or schedule. | `./verify.sh 06` | +| 7 | [Networking & Protocols](topics/07-networking-protocols/README.md) | Identical zero-work requests: **44k ops/s at P=1**, **12.3M at P=256** — a **279×** swing that is pure syscalls and round trips. | `./verify.sh 07` | +| 8 | [Transactions & MVCC](topics/08-transactions-mvcc/README.md) | A global mutex delivers ~**600k txn/s** on read-heavy, write-heavy and hot-key workloads alike. Flat, because it already serialized everything. | `./verify.sh 08` | +| 9 | [Concurrency](topics/09-concurrency/README.md) | A global mutex gets **2.9× slower** from 1 to 16 threads (8.65 → 2.96 Mops/s). Padding "independent" counters to 128 B is worth **17.8×**; 64 B only half-fixes it on M-series. | `./verify.sh 09` | +| 11 | [Execution Models](topics/11-execution-models/README.md) | Volcano tops out at **103 M rows/s**, and gets *slower* as selectivity rises (74.7 M at 95%) — surviving the filter is what costs, not the filter. | `./verify.sh 11` | +| 12 | [Columnar Analytics](topics/12-columnar-analytics/README.md) | The scan floor is **24–57 GB/s** on a 150 GB/s machine. This lane previously printed **19,047,619 GB/s** — a hoisted loop, caught by its own implausibility. | `./verify.sh 12` | +| 13 | [Graph Engines](topics/13-graph-engines/README.md) | The same two-hop query is **101× slower** from supernodes than from random nodes (4.9 µs → 495 µs) — and reaches *fewer* distinct nodes. | `./verify.sh 13` | +| 14 | [Vector Search](topics/14-vector-search/README.md) | Brute force: **117 QPS** at recall 1.000. That single point is what every ANN index is betting against. | `./verify.sh 14` | +| 15 | [Replication & Consensus](topics/15-replication-consensus/README.md) | Follower fsync policy alone spans **59×** (341 → 20,174 entries/s). Batching fixes the median and leaves the p99 at 2980 µs. | `./verify.sh 15` | +| 16 | [Testing & Correctness](topics/16-testing-correctness/README.md) | Seeded crash testing catches planted bugs at **48.8% to 99.6%** per seed — same harness, four wildly different odds of ever finding out. | `./verify.sh 16` | +| 17 | [SIMD](topics/17-simd/README.md) | Eight accumulators and no intrinsics: **8.88 → 26.32 GB/s**. Branchy filtering collapses to **0.95 GB/s** at 50% selectivity while branchless stays flat at ~10. | `./verify.sh 17` | +| 18 | [GPU Acceleration](topics/18-gpu/README.md) | **No crossover up to 2^24 elements.** At 16 M, upload alone costs 7197 µs against a 2723 µs CPU total — the transfer tax, measured. | `./verify.sh 18` | +| 19 | [JIT & Compilation](topics/19-jit/README.md) | Interpretation cost compounds with expression size: 7 → 511 nodes costs the interpreter **94×** but the vectorized evaluator only **47×**, so the gap widens from 6× to 12×. | `./verify.sh 19` | +| 20 | [GraphBLAS](topics/20-graphblas/README.md) | SpMV bandwidth decays **20.7 → 12.3 GB/s** as the graph grows. Hypersparse indexing is **50× smaller** (80.4 MB → 1.59 MB) and sweeps rows **175× faster**. | `./verify.sh 20` | +| 21 | [Formal Methods](topics/21-formal/README.md) | The hand-ordered rewriter answers `(a*2)/2` with `(a << 1) / 2` and stops. One locally-excellent rewrite destroys the cancellation — the phase-ordering trap, in four lines. | `./verify.sh 21` | +| 22 | [Standard Benchmarks](topics/22-benchmarks/README.md) | TPC-H Q1 and Q6 measured at **5.2–5.7** and **9.0–14.4 GB/s** effective; YCSB-E's p999 is **12.9 µs** against read-only's 4.0 µs. | `./verify.sh 22` | +| 23 | [Full-Text Search](topics/23-fulltext/README.md) | Exhaustive BM25 spans **0.009 ms to 10.378 ms** across four two-term queries — 272,310 postings against 159. Term rarity, not query complexity. | `./verify.sh 23` | +| 24 | [Graph Algorithms](topics/24-graph-algorithms/README.md) | Same node and edge count, RMAT vs uniform: **15.6 M triangles vs 5428**, and 447 ms vs 195 ms. Degree skew is the workload. | `./verify.sh 24` | +| 25 | [Graph ML](topics/25-graph-ml/README.md) | The message-passing kernel *is* an SpMM: **4.31 ms at 16.82 GFLOP/s**, against 5.65 ms for the dense transform beside it. | `./verify.sh 25` | +| 26 | [Probabilistic Structures](topics/26-probabilistic/README.md) | A point miss costs **246 ns** (binary search) or **299 ns** (BTreeMap); a 224 MB HashSet does it in **28 ns**. That gap is what a filter is bidding for. | `./verify.sh 26` | +| 27 | [Streaming & IVM](topics/27-streaming/README.md) | 100 edge changes to a 500k-edge graph costs **1111 ms** to re-derive a wedge join — the batch is 0.02% of the graph, the work is 100% of it. | `./verify.sh 27` | +| 28 | [Cloud-Native Storage](topics/28-cloud-native/README.md) | Local NVMe p50 **0.10 ms**; raw S3 p50 **14.17 ms**, p99 **112.99 ms**. A 140× median gap and a far worse tail. | `./verify.sh 28` | +| 29 | [Distributed Transactions](topics/29-distributed-txn/README.md) | The workload's own conflict rate goes **0.3% → 99.6%** as Zipf θ moves 0.5 → 1.3. Contention is a property of the data, before any protocol. | `./verify.sh 29` | +| 30 | [Time-Series](topics/30-timeseries/README.md) | delta+varint gives **11.00 B/sample for all four shapes** — a constant series compresses exactly as well as random noise, because only the timestamp is being compressed. | `./verify.sh 30` | +| 31 | [CRDTs](topics/31-crdts/README.md) | Last-write-wins on 10 keys with per-write sync loses **94.98%** of writes — 37,991 of 40,000 acknowledged writes that no replica remembers. | `./verify.sh 31` | +| 32 | [HTAP](topics/32-htap/README.md) | One copy, one coarse lock: adding full scans takes writes from **10.5 M per 2 s to 94**, and p99 from 334 ns to **2.7 s**. Every scan is a write outage. | `./verify.sh 32` | +| 33 | [Temporal Graphs](topics/33-temporal-graphs/README.md) | Static reachability reports 25,031 reachable pairs where time-respecting paths number **137** — **99.5% false positives** on the sparse contact graph. | `./verify.sh 33` | +| 34 | [Debugging & Diagnosis](topics/34-debugging/README.md) | A closed-loop benchmark reports **p99 = 1.0 µs** where an open-loop one reports **90 ms** on identical work — coordinated omission, a 90,000× lie. | `./verify.sh 34` | +| 35 | [Overload Control](topics/35-overload/README.md) | A 10-second outage ends at t=40 s. At 140 QPS (of 300 capacity) goodput stays at **zero until t=161 s**; at 280 QPS it **never recovers** — the outage outlives its own trigger. | `./verify.sh 35` | +| 36 | [Sharding & Rebalancing](topics/36-sharding/README.md) | Growing 16 shards to 17 moves **94.1% of all keys** (ideal: 5.9%), and it gets *worse* the larger you are. | `./verify.sh 36` | +| 37 | [Distributed Query](topics/37-distributed-query/README.md) | With 1-in-100 slow leaves, **63.4% of 100-leaf** fan-outs hit at least one. Waiting for 95% of leaves instead of all: p99 **10.0 → 9.9 ms**, p50 5.6 → 9.6. | `./verify.sh 37` | +| 38 | [GraphRAG & Agent Memory](topics/38-graphrag-agent-memory/README.md) | Independent passage ranking finds the answer at rank **1.00** at one hop and **9.21** — chance — at two. Vector RAG's multi-hop collapse. | `./verify.sh 38` | +| 39 | [Fraud & Identity Graphs](topics/39-fraud-identity-graphs/README.md) | Two row-based rankers fail in *opposite* regimes: degree ranking scores **0.00** precision without camouflage, obscurity ranking **0.00** with it. | `./verify.sh 39` | +| 40 | [Security & Attack Graphs](topics/40-security-attack-graphs/README.md) | A directory reporting **8 privileged accounts, forever** has **1969 of 2000 users** holding a path to Domain Admin — and your exposure number depends on how long the collector ran. | `./verify.sh 40` | +| 41 | [On-Chain Analytics](topics/41-onchain-analytics/README.md) | The industry-default haircut rule marks **98% of addresses** tainted from one theft; 658 of them are under 0.1% tainted. An 1816 court case does better. | `./verify.sh 41` | +| 42 | [Recommendations & Social](topics/42-recommendations-social/README.md) | Recommending bestsellers to everyone gets **35.3% hit-rate@50** with **92.2% overlap** between users' lists. Popularity is not a weak baseline. | `./verify.sh 42` | +| 43 | [Ops Dependency Graphs](topics/43-ops-dependency-graphs/README.md) | One gray failure: **34 of 55 services alert** and the broken one is not among them — it ranks 35th by failure count, 41st by error rate, at exactly the baseline. | `./verify.sh 43` | + +## How to read this table + +Some rows are pleasant confirmations of theory. The interesting ones are not: + +- **Row 3 and row 12 contradict their own topic's tidy story.** The B-tree ladder + is not a step function, and the columnar scan lane once printed a number + 20,000× faster than the memory bus. Both are in the guides as the finding, + because a curriculum that only reports confirmations is not measuring anything. +- **Rows 8, 22, 30 and 42 are baselines that refuse to be weak.** A global mutex + is flat rather than bad; delta+varint compresses noise as well as constants; + bestseller lists get a third of users right. If your improvement cannot beat + these, that is worth knowing before you build it. +- **Rows 34 through 43 are all measurement failures rather than system + failures** — the closed loop, the alert storm, the per-node statistic, the + compliance report. The system is fine; the number you were looking at was not. + +If a row does not reproduce on your machine beyond timing differences, that is +the most useful bug report this repo can receive — +[open an issue](https://github.com/AviAvni/database-learning-path/issues) with +the output. + +## Which topics lean on which + +The order is a suggestion, not a prerequisite chain — but the guides do thread, +and some threads carry real weight. Solid arrows are dependencies worth +respecting; the clusters are just groupings. + +```mermaid +flowchart TD + T0["0 · measurement discipline"] + T0 --> T1["1 · storage engines
RUM triangle"] + T1 --> T3["3 · B-trees"] + T1 --> T4["4 · LSM"] + T0 --> T2["2 · in-memory structures"] + T3 --> T6["6 · buffer pool"] + T4 --> T5["5 · WAL & fsync"] + T5 --> T15["15 · replication"] + T6 --> T3 + T2 --> T9["9 · concurrency"] + T9 --> T8["8 · MVCC"] + T10["10 · planning"] --> T11["11 · execution"] + T11 --> T12["12 · columnar"] + T12 --> T17["17 · SIMD"] + T11 --> T19["19 · JIT"] + T17 --> T18["18 · GPU"] + T2 --> T13["13 · graph engines"] + T13 --> T20["20 · GraphBLAS"] + T20 --> T24["24 · graph algorithms"] + T24 --> T25["25 · graph ML"] + T13 --> T14["14 · vector search"] + T12 --> T23["23 · full-text"] + T15 --> T29["29 · distributed txn"] + T15 --> T31["31 · CRDTs"] + T5 --> T16["16 · testing"] + T16 --> T21["21 · formal methods"] + T0 --> T22["22 · benchmarks"] + T0 --> T34["34 · diagnosis"] + T34 --> T35["35 · overload"] + T35 --> T36["36 · sharding"] + T36 --> T37["37 · distributed query"] + T24 --> T38["38-43 · six graph use cases"] + T14 --> T38 +``` + +Three threads are worth following deliberately, because the same wall shows up +in each of them under a different name: + +1. **The fsync wall.** Topic 5 measures 337 durable commits/s. Topic 15's + per-entry follower fsync lands at 341 entries/s. Topic 28 finds the same + physics at 14 ms per S3 GET. One constant, three architectures built to dodge it. +2. **Skew.** Topic 13's 101× supernode gap, topic 24's 15.6 M-vs-5428 triangle + counts, topic 36's hot shard at 3.8× ideal, topic 20's load-balancing menu. + Uniform data is the exception in every one of them. +3. **The measurement itself lying.** Topic 0 states the failure modes, topic 12 + contains one (a hoisted loop printing 19 million GB/s), topic 34 quantifies + the closed-loop version at 90,000×, and topics 39–43 are each a different + statistic pointing confidently at the wrong thing. diff --git a/PROGRESS.md b/PROGRESS.md index d2b7de3..2379b55 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,53 +1,65 @@ # Progress -Status: `todo` → `in progress` → `done`. Add a one-line takeaway when done. +Two different things get tracked here, and conflating them is misleading: -| # | Topic | Status | Takeaway | -|---|-------|--------|----------| -| 0 | The Performance Toolbox | done | Benchmarks lie by default: my own cache_ladder measured its own cache footprint until the walker carried state; flamegraph showed 21% of HashMap lookup time is SipHash; DRAM ladder verified at ~1/5/100 ns. | -| 1 | Storage Engine Landscape: B-Tree vs LSM | in progress | | -| 2 | In-Memory Structures: Hash Tables, Skip Lists, Tries | todo | | -| 3 | B-Tree Internals & Paged Storage | todo | | -| 4 | LSM-Tree Deep Dive | todo | | -| 5 | Durability: WAL, fsync, Crash Recovery | todo | | -| 6 | Buffer Pool & Memory Management | todo | | -| 7 | Networking, Protocols & Event Loops | todo | | -| 8 | Transactions & MVCC | todo | | -| 9 | Concurrency: Latches, Lock-Free & Epochs | todo | | -| 10 | Query Engines I: Parsing, Planning, Optimization | todo | | -| 11 | Query Engines II: Execution Models | todo | | -| 12 | Columnar Storage & Analytics | todo | | -| 13 | Graph Engines | todo | | -| 14 | Vector Search | todo | | -| 15 | Replication, Consensus & Distribution | todo | | -| 16 | Testing & Correctness Engineering | todo | | -| 17 | SIMD & Hardware-Conscious Data Processing | todo | | -| 18 | GPU Acceleration for Databases | todo | | -| 19 | JIT & Query Compilation | todo | | -| 20 | Sparse Linear Algebra & GraphBLAS Internals | todo | | -| 21 | Formal Methods & Verification | todo | | -| 22 | Standard Benchmarks: TPC-H, TPC-C, YCSB, LDBC | todo | | -| 23 | Full-Text Search & Inverted Indexes | todo | | -| 24 | Advanced Graph Algorithms & Analytics | todo | | -| 25 | Graph Neural Networks & Graph ML | todo | | -| 26 | Indexing & Probabilistic Data Structures | todo | | -| 27 | Streaming & Incremental View Maintenance | todo | | -| 28 | Cloud-Native & Disaggregated Storage | todo | | -| 29 | Distributed Transactions | todo | | -| 30 | Time-Series Engines | todo | | -| 31 | CRDTs & Multi-Master Replication | todo | | -| 32 | HTAP Architectures | todo | | -| 33 | Temporal Graphs | todo | | -| 34 | Debugging & Production Diagnosis | todo | | -| 35 | Overload Control & Resource Governance | todo | | -| 36 | Sharding, Partitioning & Rebalancing | todo | | -| 37 | Distributed Query Execution | todo | | -| 38 | GraphRAG & Agent Memory (graph use case 1/6) | todo | | -| 39 | Fraud Rings & Identity Graphs (graph use case 2/6) | todo | | -| 40 | Security & Attack Graphs (graph use case 3/6) | todo | | -| 41 | On-Chain & Crypto Analytics (graph use case 4/6) | todo | | -| 42 | Recommendations & Social Graphs (graph use case 5/6) | todo | | -| 43 | Network & IT-Ops Dependency Graphs (graph use case 6/6) | todo | | +- **Package** — does `topics/NN-name/` exist and hold up? That means a study + guide, four to seven reading guides, `notes.md`, and an experiments crate + whose provided lane runs and whose numbers are recorded. All 44 are built; + `./verify.sh` re-derives every one of their measured lanes. +- **Studied** — have *I* actually worked through the material and the two + exercise lanes? That is a much smaller number, and it is the honest one. + `todo` here does not mean the topic is missing; it means the exercises are + still exercises. + +Status values: `todo` → `in progress` → `done`. A takeaway lands when Studied +is done. + +| # | Topic | Package | Studied | Takeaway | +|---|-------|---------|---------|----------| +| 0 | The Performance Toolbox | done | done | Benchmarks lie by default: my own cache_ladder measured its own cache footprint until the walker carried state; flamegraph showed 21% of HashMap lookup time is SipHash; DRAM ladder verified at ~1/5/100 ns. | +| 1 | Storage Engine Landscape: B-Tree vs LSM | done | in progress | | +| 2 | In-Memory Structures: Hash Tables, Skip Lists, Tries | done | todo | | +| 3 | B-Tree Internals & Paged Storage | done | todo | | +| 4 | LSM-Tree Deep Dive | done | todo | | +| 5 | Durability: WAL, fsync, Crash Recovery | done | todo | | +| 6 | Buffer Pool & Memory Management | done | todo | | +| 7 | Networking, Protocols & Event Loops | done | todo | | +| 8 | Transactions & MVCC | done | todo | | +| 9 | Concurrency: Latches, Lock-Free & Epochs | done | todo | | +| 10 | Query Engines I: Parsing, Planning, Optimization | done | todo | | +| 11 | Query Engines II: Execution Models | done | todo | | +| 12 | Columnar Storage & Analytics | done | todo | | +| 13 | Graph Engines | done | todo | | +| 14 | Vector Search | done | todo | | +| 15 | Replication, Consensus & Distribution | done | todo | | +| 16 | Testing & Correctness Engineering | done | todo | | +| 17 | SIMD & Hardware-Conscious Data Processing | done | todo | | +| 18 | GPU Acceleration for Databases | done | todo | | +| 19 | JIT & Query Compilation | done | todo | | +| 20 | Sparse Linear Algebra & GraphBLAS Internals | done | todo | | +| 21 | Formal Methods & Verification | done | todo | | +| 22 | Standard Benchmarks: TPC-H, TPC-C, YCSB, LDBC | done | todo | | +| 23 | Full-Text Search & Inverted Indexes | done | todo | | +| 24 | Advanced Graph Algorithms & Analytics | done | todo | | +| 25 | Graph Neural Networks & Graph ML | done | todo | | +| 26 | Indexing & Probabilistic Data Structures | done | todo | | +| 27 | Streaming & Incremental View Maintenance | done | todo | | +| 28 | Cloud-Native & Disaggregated Storage | done | todo | | +| 29 | Distributed Transactions | done | todo | | +| 30 | Time-Series Engines | done | todo | | +| 31 | CRDTs & Multi-Master Replication | done | todo | | +| 32 | HTAP Architectures | done | todo | | +| 33 | Temporal Graphs | done | todo | | +| 34 | Debugging & Production Diagnosis | done | todo | | +| 35 | Overload Control & Resource Governance | done | todo | | +| 36 | Sharding, Partitioning & Rebalancing | done | todo | | +| 37 | Distributed Query Execution | done | todo | | +| 38 | GraphRAG & Agent Memory (graph use case 1/6) | done | todo | | +| 39 | Fraud Rings & Identity Graphs (graph use case 2/6) | done | todo | | +| 40 | Security & Attack Graphs (graph use case 3/6) | done | todo | | +| 41 | On-Chain & Crypto Analytics (graph use case 4/6) | done | todo | | +| 42 | Recommendations & Social Graphs (graph use case 5/6) | done | todo | | +| 43 | Network & IT-Ops Dependency Graphs (graph use case 6/6) | done | todo | | ## Capstone milestones (falkordb-rs-next-gen from scratch) diff --git a/README.md b/README.md index a76722c..29b6d20 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ list. The difference is that the interesting claims here are *demonstrated*, wit a seeded benchmark you can run in seconds, and the surprising results are the point of the topic rather than a footnote. -Four examples, each the opening measurement of its topic: +Six examples, each the opening measurement of its topic: | finding | topic | |---|---| @@ -28,13 +28,17 @@ Four examples, each the opening measurement of its topic: | Growing a hash-sharded cluster from 16 to 17 nodes moves **94.1% of all keys**, and gets *worse* the larger you are | [36 — Sharding & Rebalancing](topics/36-sharding/README.md) | | A directory whose console reports **8 privileged accounts, forever** has **1969 of 2000 users** holding a path to Domain Admin | [40 — Security & Attack Graphs](topics/40-security-attack-graphs/README.md) | | The industry-default Bitcoin taint rule marks **93% of all addresses** as tainted from one theft; an 1816 English court case brings it to **1.35%** | [41 — On-Chain & Crypto Analytics](topics/41-onchain-analytics/README.md) | +| A global mutex gets **2.9× slower** going from 1 thread to 16 — not "stops scaling", actually negative | [9 — Concurrency](topics/09-concurrency/README.md) | +| Identical zero-work requests run at **44k/s pipelined 1-deep and 12.3M/s at 256** — a 279× swing that is pure syscalls | [7 — Networking & Protocols](topics/07-networking-protocols/README.md) | -Each of those runs today: +**[FINDINGS.md](FINDINGS.md) has all 42 of them in one table**, with the command +that re-derives each. Every one runs today: ```bash git clone https://github.com/AviAvni/database-learning-path cd database-learning-path ./verify.sh 34 36 40 41 # or ./verify.sh for all of them +./verify.sh --list # every lane and what it measures ``` ## Start here @@ -74,7 +78,18 @@ downloaded and read; every figure quoted is attributed to a specific section or and every performance claim in the repo comes from code in this repo that you can run. `./verify.sh` exists so you never have to trust any of it — it builds and runs each measured benchmark and prints the output, and the generators are seeded, so the -figures reproduce exactly apart from timings. +figures reproduce exactly apart from timings. CI runs the same script on every push, +so a lane that stops building or stops running fails the build rather than quietly +rotting. + +The format has already caught one of its own claims. Topic 12's scan lane used to +report **19,047,619 GB/s** — roughly 20,000× the machine's memory bandwidth — +because the timing loop let the optimizer hoist the work out from under it. It +was found by reading the printed output and noticing the number was impossible, +which is the whole point: a reading list cannot contain a detectably wrong +number, and a printed measurement can. It is written up in the topic rather than +quietly deleted, and it is topic 0's first failure mode occurring in this repo's +own code. That is the real argument for this format over a reading list: a link collection cannot be wrong in a way you can detect, and this can. @@ -86,19 +101,25 @@ that is the most useful contribution possible here. ## Layout ``` +FINDINGS.md every measured result, one row per topic + the command PLAN.md the full 44-topic curriculum — the source of truth -PROGRESS.md status tracker + capstone milestones +PROGRESS.md status: which packages exist vs which I have studied SESSION-LOG.md detailed build log, one entry per topic -verify.sh re-run every measured benchmark +verify.sh re-run every measured benchmark (--list, --summary) +tools/ pin-table.py — regenerates the read-against commit table topics/NN-name/ - README.md study guide: concepts, measured results, exercises + README.md study guide: opens with the problem, measured reading-*.md one guide per paper or codebase, concept-first - notes.md predictions vs measurements, cross-topic threads + notes.md measured baseline, prediction worksheet, cross-topic threads experiments/ Rust crate: one implemented lane, two as exercises -capstone/ the graph engine, one milestone per topic -resources/ flat indexes: papers, codebases, tools +capstone/ the graph engine, one milestone per topic (M0 built so far) +resources/ flat indexes: papers, codebases + the anchor pin table +drafts/ unpublished prose spun out of the topics; not part of the book ``` +Build artifacts land in `../.dlp-target`, outside the clone — see +[.cargo/config.toml](.cargo/config.toml) for why. + ## Background Written by [Avi Avni](https://github.com/AviAvni), a core developer of diff --git a/SESSION-LOG.md b/SESSION-LOG.md index a70f04d..8b06707 100644 --- a/SESSION-LOG.md +++ b/SESSION-LOG.md @@ -9,53 +9,229 @@ Every performance figure quoted below is reproducible with `./verify.sh` (see [README.md](README.md)); timings depend on hardware, everything else is seeded. -- 2026-07-27 — **topic 43 Network & IT-Ops Dependency Graphs added** (sixth and last of the graph use-case deep dives; the 38-43 expansion is complete): study guide (**the alert storm and the gray failure measured** — bench lane 1: synthetic microservice topology, 4 frontends / three tiers of 10-16-20 / 5 shared infra leaves, 152 configured edges of which **113 are reachable** (an unreachable-configuration finding in itself), 40,000 requests, with a planted **gray failure** on the most-depended-on infra leaf — `infra-0` is SLOW on 55% of calls rather than failing, and its callers time out, so the errors are manufactured one hop ABOVE the cause; result: **34 of 55 services alert above a 5% error rate and the broken service is not one of them**, its own error rate is **0.0040 = exactly the baseline**, it ranks **35 of 55 by failure count and 41 of 55 by error rate** (error-rate ranking puts the three front ends at the top, i.e. it points at the services furthest from the fault), and **all five infra leaves sit at 0.0040-0.0041, statistically indistinguishable** — no sorting of any per-node column can separate them, which is the entire argument for the topic; lane 2 reference — localization across five topologies: per-node baselines average **rank 36.4 (failure count) and 44.0 (error rate) with 0/5 top-3**, while a **correlation-weighted random walk and Sherlock's Ferret at k=1 both average rank 1.0 with 5/5 top-1**, at 22.8 ms and 21.4 ms respectively; the instructive ablation is that a **backward-only walk ranks the cause 3rd instead of 1st** because it drains into the leaves with no way to climb out, so the forward and self edges are what let the correlation weights bite; and the detail that makes the Ferret implementation work is **clamping the fitted severity to [0,1]** — a severity is a probability, so a candidate simply not on enough requests would need one above 1 to explain the observed rates; lane 3 reference — Dapper sampling as **two different questions with two different answers**: edge recall stays at **1.000 all the way down to 39 traces (1/1024)** while rare-path recall collapses **1.000 → 0.249 → 0.062 → 0.016 → 0.004 → 0.001**, and the mean latency stays within **5.8%** while the **p99 error reaches 25.6%** — one sample, three verdicts depending on whether the question is aggregate, rare-event or tail, with the honest caveat stated in the output that edge recall saturates this early only because this topology has little path diversity), 4 reading guides (Dapper 2010 read in full — the ubiquity/continuous-monitoring requirements that force negligible overhead, trace trees with clock skew handled by causality rather than NTP, the three instrumentation points (thread-local context, the common control-flow library, the single RPC framework) that make transparency possible in <1000 lines of C++ and <800 of Java, **out-of-band collection for two reasons** (in-band trace data would dwarf sub-10 KB RPC responses and bias analyses; in-band assumes perfectly-nested RPCs, which middleware violates) at a cost of a bimodal p98 collection latency, the overhead budget (**204 ns root span / 176 ns non-root / 9 ns unsampled annotation / 40 ns sampled / <0.3% of a core / 426 bytes per span / <0.01% of network traffic**) and the 9-vs-40 ns split that made 70%-of-spans annotation coverage possible, **Table 2's sampling cost (+16.3% latency at 1/1, +2.12% at 1/16, −0.20% at 1/1024 inside experimental error)**, the "if a notable execution pattern surfaces once it will surface thousands of times" argument together with its own caveat about low-volume services, adaptive sampling by rate-per-unit-time with the probability recorded alongside the trace, and the critical detail that collection-time sampling hashes the **trace id** so whole traces are kept or dropped — a shredded trace has no causal structure left; Sherlock SIGCOMM'07 read in full — the **(P_up, P_troubled, P_down)** three-state model with *troubled* defined as "servers or links continue to function but users perceive poor performance" (differential observability, a decade early), the three node types (root-cause / observation / meta) and **three meta-nodes** with their truth tables — noisy-max (with probability `1−d` the child escapes its parent's state), selector (a noisy-max node would give a client a **25% chance of being up with both load-balanced servers down**), failover — the **always-troubled / always-down pseudo-causes at 0.001** ("1 in 1000 failures are caused by a component not in our model") and router-path edges at 0.9999 as explicitly-priced model error, the **O(3ⁿ) → O(n)** propagation reduction for noisy-max nodes, and **Ferret**: 3^r assignment vectors cut to at most **(2r)^k** by Observation 3.1 ("it is very likely that at any point in time only a few root-cause nodes are troubled or down", error "vanishingly small for k = 4 onwards") plus **two orders of magnitude** from Observation 3.2, scored by fitting two Gaussians (200 ms vs 2 s) to historical response times with a null-hypothesis significance test, over a dependency graph *discovered* from packet co-occurrence within a **10 ms dependency interval** discounted at (10ms)/I; Pivot Tracing SOSP'15 read in full as **the database paper hiding in an operations topic** — the two failures of ordinary monitoring ("one size does not fit all", with the Apache issue-tracker evidence and HBase's "all users pay the 10% overhead"; and crossing boundaries), the query language and the **happened-before join `Q1 ⋈ Q2` over Lamport's →**, the five advice primitives OBSERVE/UNPACK/FILTER/PACK/EMIT woven at runtime with no jumps or recursion and guaranteed termination, **baggage** as a per-request tuple container propagated across thread/process/machine boundaries so joins evaluate **in situ** rather than centrally (Magpie's strategy is Figure 6a), and **Table 3's rewrite rules pushing projection, selection and aggregation down to the source tracepoints for a 600 → 6 tuples/s reduction** — predicate pushdown and join placement, in a tracing system; Huang et al. HotOS'17 *Gray Failure* — the observer/app/ground-truth model and the four-cell table whose fourth cell is **differential observability**, why every redundancy mechanism is inert under it (they are all keyed on the observer's view), three structural reasons detection is hard, and the escalation argument that makes a gray failure the trigger of a topic-35 metastable failure), experiments crate `opsgraph-experiments` (`services.rs` PROVIDED — the topology generator, the gray-failure workload with slow-dependency-plus-caller-timeout propagation, traces carrying paths/edges/latency, both per-node baselines, the symptom correlation, and `participation` = P(service on path | entry frontend), which is the deliberately weak observable Ferret must work from; `rca.rs` stub — `random_walk_rca` with three edge types and `sherlock_single_fault`; `sampling.rs` stub — whole-trace `sample`, `edge_recall`, `rare_path_recall`; **4 provided tests pass, 9 fix the contract for the stubs**, including that a gray failure must not trip its own alert, that all infra error rates stay within 0.01 of each other, that the walk beats both baselines and a backward-only walk is strictly worse, that the ranking is stable across five seeds, and that sampling keeps whole traces), and capstone M43 (trace ingest as an incrementally-maintained dependency graph with sketched edge weights, both localization procedures over the topic-18 CSR, and a happened-before join operator in the query engine with Pivot Tracing's pushdown rewrites — deliverable numbers include **top-1 accuracy under sampling**, the question the whole topic converges on and which none of the four papers answers). Cross-topic threads worked: 38/42 (personalized PageRank a third time, same justification), 34 (this is topic 34 at cluster scale), 37 (fan-out arithmetic explains the storm; hedging works where failure detection does not), 35 (gray failure as trigger, retry storm as sustaining loop), 10 (Pivot Tracing's Table 3 is an optimizer), 26 (edge weights must be sketches — the p99 row is why), 40 (the same graph question with the arrows reversed), 27 (trace ingest is a stream, the dependency graph a materialized view over it), 21 (Sherlock's model is tuned, not verified). -- 2026-07-27 — **topic 42 Recommendations & Social Graphs added** (fifth of the six graph use-case deep dives): study guide (**the popularity trap measured** — bench lane 1: synthetic bipartite interaction graph, 3000 users x 6000 items, 30 communities, Zipf(1.1) popularity tail, 60,000 training edges and 6,000 held-out engagements; the bestseller list gets **hit-rate@50 = 0.340** with a personalization score of only **0.155** (and that only because each user's own items are filtered out — everybody is handed the same list), while Pixie's unmodified Algorithm 1 reaches 0.403 but **45% of every returned list is the bestseller list again**, because an unbiased walk's stationary distribution goes as degree — Pixie's own complaint from §3.1, "low degree nodes with fewer edges contribute less signal ... smaller boards are more likely to produce highly relevant recommendations"; lane 2 reference — the Pixie ablation over 300 users x 8 query pins x 30,000 steps: going from one query pin to eight with sub-linear step allocation takes hit rate **0.403 → 0.823** (the biggest single win, and the least clever idea), **early stopping runs in 35% of the steps at 2.2× the speed keeping 0.793 top-50 overlap with hit rate unchanged** — almost exactly the paper's "84% overlap at a third of the runtime" — and **the multi-hit booster shows NO gain at all**, 0.823 unboosted vs 0.803 boosted at one interest per user and 0.563 vs 0.547 at three, which is the more instructive result: the arithmetic is right (the unit test pins (√2+√2)²=8 against a single-source 4) but the generator does not contain Equation 3's premise, since it draws its held-out item from the same distribution as the training items, so a published trick's *domain assumption* has to be measured on your own data before you ship it (exercise 4 builds a graph where the premise holds); lane 3 reference — link prediction on a collaboration graph grown with preferential attachment + triadic closure where a random guess is right **0.314%** of the time (right inside Liben-Nowell's 0.147–0.475% band): preferential attachment **1.9×**, common neighbours **20.7×**, Jaccard **25.6×**, Adamic/Adar **22.3×** — the degree-only measure barely beats chance, exactly as in the paper), 4 reading guides (Pixie WWW'18 read in full — the 30–50%-engagement argument for real-time over batch, Algorithm 1 in twenty lines, the four innovations (user-feature biasing with `PersonalizedNeighbor` as a *subrange* operator, weighted query sets with Equation 1's sub-linear allocation `s_q = |E(q)|·(C − log|E(q)|)` where C must be the graph-wide maximum or the top query pin gets zero steps, Equation 3's multi-hit boost, and per-walk early stopping on n_p pins reaching n_v visits), the language-biasing table (En→Slovak target-language content **2.13% → 42.55%**, En→Japanese 16.35% → 80.33%), hit rate 6.3/23.1/52.2% at top-10/100/1000 against content-based 2.1/4.6/10.5%, A/B lifts of +48% on homefeed, the **pruning result that F1 peaks 58% above the unpruned graph at 20% of the edges**, and the implementation section's `edgeVec` object pool + open-addressed visit counter sized to N + **HugePages cutting page-table entries 512×**; GraphJet VLDB'16 read in full — four generations (Cassovary → Hadoop RealGraph → MagicRecs → GraphJet) and what killed each, the single-server bet with its "ten billion edges is a mere 80 GB" arithmetic and the challenge to distributed graph research, MagicRecs' reformulation of temporal edge detection as an **intersection of adjacency lists**, the five-method API and the two deliberate omissions (no deletes because interactions are point events, no timestamps as a space/quality trade), **temporally-partitioned index segments** with only the newest writable and whole-segment discard as coarse pruning, id mapping by double hashing where the hash IS the internal id (hence the power-of-two table chain) with edge type bit-packed to leave 2²⁹ ids, **edge pools whose slice sizes double** (`P_r` holds `n/2^{r−1}` slices of `2^r` edges; degree 25 → `P1(1),P2(2),P3(0),P4(0)`) justified as an allocator that assumes preferential attachment, single-writer/multi-reader with memory barriers instead of locks, background relayout of sealed segments for contiguous iteration, the **alias method** for O(1) degree-weighted cross-segment sampling, full vs subgraph SALSA (the subgraph fits in cache and needs only a left-to-right index, ~half the memory, at the cost of second-order paths), the deployment numbers (**1M edge insertions/s**, 500 rec/s per server at **p50 19 / p90 27 / p99 33 ms**, O(10⁹) edges in **<30 GB**, >99.99% over 30 days), and **§7.3's rejection of Redis `LPUSH` as an adjacency-list store for two named reasons — no memory-allocation optimization and no pruning mechanism — which is a two-item feature list for a Redis-module graph engine**; TAO ATC'13 read in full — the three failures of lookaside caching (inefficient edge lists, distributed control logic, expensive read-after-write), the two data shapes and four association queries, **creation-time locality** ("most of the data is old, but many of the queries are for the newest subset") forcing newest-first association lists and prefix caching, which in turn forces **refill rather than invalidate** (invalidating truncates a cached prefix and discards edges), sharding associations by `id1` so every query is one server — which is *why* there is no multi-hop traversal — the leader/follower hierarchy sized by **read misses being 25× as frequent as writes**, hot-spot handling by shard cloning and access-rate-triggered client-side caching, slab allocation with per-type arenas and **association counts packed into 14 bytes** for 20% more cache entries, and the production envelope (**96.4% read hit rate**, `assoc_get` 1.0 ms p50 hit vs 143 ms p99 miss, writes 12.1 ms in-region vs 74.4 ms from 58 ms away, **4.9 × 10⁻⁶ failed queries over 90 days**) plus the two workload tails that must be in any honest benchmark — **1% of `assoc_count` results ≥512K** and **64% of non-empty ranges returning exactly one edge**; Liben-Nowell & Kleinberg read in full — the training/test interval setup with κ=3 Core filtering, why **factor-improvement-over-random** is the only interpretable metric when raw accuracy is 0.147–0.475%, the measure catalogue (common neighbours, Jaccard, **Adamic/Adar's `1/log|Γ(z)|` hub discount**, preferential attachment, Katz, hitting/commute time normalized by the stationary distribution *because otherwise popular nodes dominate — the popularity trap from a third direction*, rooted PageRank, SimRank), Figure 3's table in which **preferential attachment scores 4.7–15.2× against common neighbours' 18.0–47.2× and Adamic/Adar's 16.8–54.8×**, "there is no single clear winner among the techniques", and the three meta-approaches — low-rank approximation, unseen bigrams, and a **clustering step that deletes low-confidence edges and recomputes, which is Pixie's graph pruning arrived at fifteen years earlier**), experiments crate `social-experiments` (`graphs.rs` PROVIDED — the bipartite interaction graph with communities, a Zipf tail, configurable interests per user and held-out engagements, plus the Liben-Nowell collaboration graph with preferential attachment + triadic closure and a train/test split, the baselines `popularity_topk` and `basic_random_walk` (= Pixie Algorithm 1), and the metrics `hit_rate` / `personalization` / `popularity_overlap` / `evaluate` with factor-over-random; `pixie.rs` stub — `allocate_steps`, `walk_per_query`, `multi_hit_boost`, `pixie_walk` with early stopping; `linkpred.rs` stub — the four measures; **2 provided tests pass, 8 fix the contract for the stubs**, including the boost arithmetic being exact and leaving single-source scores unchanged, every query pin getting ≥1 step with a step ratio strictly below the degree ratio, early stopping keeping ≥70% top-100 overlap in strictly fewer steps, two users from different communities sharing <50% of their top-50, Adamic/Adar's discount being arithmetically exact on a hand-built hub-vs-specialist case, and preferential attachment losing to both common neighbours and Adamic/Adar), and capstone M42 (a temporally-bounded bipartite interaction store with GraphJet's index segments and doubling edge pools over M31's storage, a Pixie-shaped walk procedure with sub-linear allocation and early stopping, and a TAO-shaped `assoc_range`/`assoc_time_range`/`assoc_count` API with time-ordered lists and cached counts — plus the benchmark GraphJet §7.3 explicitly invites, a Redis adjacency-list baseline measured on both counts it names). Cross-topic threads worked: 38 (one random-walk primitive, three seedings), 23/39 (Adamic-Adar = IDF = FRAUDAR column weights), 9 (single-writer deletes the whole latch hierarchy), 12 (sealed-segment relayout as LSM compaction; TAO's 14-byte count as a columnar instinct), 26 (alias method, bit-packing), 6 (TAO's cache is buffer management), 36 (shard-by-id1 and cloning vs migration), 40 (TAO vs Zanzibar on hot spots), 25 (low-rank approximation is where embeddings come from). -- 2026-07-27 — **topic 41 On-Chain & Crypto Analytics added** (fourth of the six graph use-case deep dives): study guide (**haircut taint diffusion measured** — bench lane 1: synthetic UTXO chain with planted ground truth, 400 entities / 20,400 transactions / 40,400 outputs / **30,342 addresses (76 addresses per entity — the pseudonymity illusion)**, one stolen coinbase worth 0.25% of all the money; haircut tainting ends up flagging **3657 of 3734 UTXOs (97.9%) and 3553 of 3627 addresses (98.0%)**, of which 658 are <0.1% tainted, 2997 are 0.1–5%, and **exactly two are above 5%** — the total is conserved to the satoshi, haircut does not invent money, it just stops being information; the real-chain version from Anderson et al.: the 2012 Linode theft of 46,653 BTC taints **16,855,619 addresses (93% of all of them) under haircut vs 245,120 (1.35%) under FIFO**, Flexcoin 2014 taints 10,421,112 (57%) vs 15,265; lane 2 reference — the three policies on the same theft: **poison flags 394.67× the stolen amount** (it re-counts each descendant output's full value, so the total explodes with fan-out), **haircut 1.00× spread over 97.9% of the UTXO set**, **FIFO 1.00× concentrated in 0.9% (32 UTXOs, one holding 22.5% of the flagged value)** — same conservation law, 114× narrower answer, at **3.1M transactions/s** because the whole algorithm is a queue splice; lane 3 reference — the clustering collapse curve: Heuristic 1 (co-spend) holds **precision exactly 1.000 at every change-reuse rate** because it keys on a property of the protocol, while Heuristic 2 (one-time change address) buys recall 0.041 → 0.397 and then goes **precision 1.000 → 0.661 → 0.502 → 0.089 → 0.009** as one change address in {∞, 100, 50, 20, 10} is reused, with the largest cluster growing **93 (1%) → 366 (3%) → 476 (4%) → 1894 (16%) → 7991 (71% of all addresses)** — union-find makes every false merge transitive and permanent, which is why a safe heuristic at recall 0.04 beats an effective one at precision 0.09), 4 reading guides (Meiklejohn et al. IMC'13 read in full — the 2013 parse (231,207 blocks / 16,086,073 txs / 12,056,684 keys), Heuristic 1's safety argument ("these entities would need to reveal their private keys to each other") taking 12M keys to **5,579,176 clusters**, Definition 4.3's four conditions with condition 4 as the heuristic's conscience (decline when two outputs are both fresh) and condition 3 explained by 23% of transactions using self-change, the **false-positive ladder 13% → 1% (excluding the Satoshi Dice payout pattern) → 0.28% (wait a day) → 0.17% / 7,382 addresses (wait a week)** = precision bought with latency, the **1.6M-key super-cluster** containing Mt. Gox + Instawallet + BitPay + Silk Road and its two named causes, and the leverage argument (2,197 clusters named covering 1.8M addresses = **1,600× manual tagging**) plus Satoshi Dice at ~60% of all activity; Anderson/Shumailov/Ahmed/Rietmann *Bitcoin Redux* WEIS'18 read in full with the RustyTaintChain @ 4e12fd0 code read — `nemo dat quod non habet` and why bitcoin being a commodity rather than money keeps theft victims' claims alive, poison/haircut/FIFO Figures 1–3, **Clayton's Case (1816)** as the precedent, the losslessness argument ("the transaction processes it in a lossless way... we can trace a bitcoin's heritage backwards as well as tracing taint forwards"), `TaintPart{name: u16, value: u64}:52` / `extract_taint:142` (the three branches — queue dry, run fits, run straddles the cut and must be split) / `combine_taints:174` (collisions between crime sources — why `name` is a u16 not a bool) / `reduce_taint:250` (run-length coalescing or the queue fragments forever), the mixer inversion ("one black coin and nine white coins into a laundry isn't ten white coins, but ten black ones — people designing money laundering mechanisms have been using quite the wrong metrics of quality"), and §5's self-undermining finding that most victims' coins never touched the chain at all because exchanges settle off-chain; BlockSci USENIX Sec'20 read in full — the design chain **append-only ⟹ static snapshots ⟹ ACID unnecessary ⟹ in-memory analytical database** and the "infinite COST" conjecture, Figure 2's transaction record (32-bit ids, 60-bit value + 4-bit address type in one word) with inputs/outputs stored **inline at a deliberate 19% space cost bought for sequential locality** (Table 4: 50.09 GB current vs 40.50 normalized vs 69.26 at 64-bit ids, on 489M txs / 1.198B inputs / 1.302B outputs), the snapshot illusion (disk table grows, each instance pins a block height, past state reconstructible because append-only), memory mapping giving zero-synchronisation parallelism because there is exactly one writer (load ~4 min, **full parallel pass 0.9 s on 16 vCPUs**, parse 5.5 h), the parser's bloom filter + multi-use address cache exploiting **88% of inputs spending outputs <4000 blocks old** and **8.6% of addresses used more than once accounting for 51% of occurrences**, union-find address linking in "a few minutes" yielding **474M clusters / 380M singletons / 809 over 20k / one supercluster >17M addresses**, the fluent DSL as a miniature query planner (7–11× over the helper method, 3–5× off hand C++), and **Table 3 benchmarked against Neo4j, Memgraph and RedisGraph — FalkorDB's own ancestor** (calculate fee: BlockSci 0.57 s vs Neo4j 303.69 vs RedisGraph did-not-finish vs Memgraph 187.02; but Neo4j-with-index *beats* single-threaded BlockSci on `Tx locktime > 0`, 0.05 vs 0.31 s) read row-by-row as a spec for what a graph engine must add to win scan-shaped queries back; Weber et al. KDD'19 read in full — the Elliptic data set (203,769 nodes / 234,355 edges / 166 features / 2% illicit / 21% licit / 49 time steps with **no edges between time steps**), the 94-local vs 72-aggregated feature split making the comparison "learned vs hand-built one-hop aggregation", and the uncomfortable result **Random Forest illicit-F1 0.788 (0.796 with GCN embeddings concatenated) beating GCN 0.628**, Skip-GCN 0.705, EvolveGCN 0.720 — plus the **dark market shutdown at time step 43 that breaks every method even when retrained after every step with fresh ground truth**, and why micro-F1 >0.92 for every method is meaningless at 2% base rate), experiments crate `chain-experiments` (`chain.rs` PROVIDED — synthetic UTXO chain *with ground truth the real blockchain does not come with*: `address_entity` per address, one stolen coinbase, planted co-spending / change addresses / recipient address reuse, no fees so taint conservation is exactly testable; `taint.rs` — `haircut` provided, `poison` / `extract_taint` / `fifo` stubbed; `clustering.rs` — `UnionFind` and the O(addresses) pair-precision/recall scorer provided, `multi_input_clusters` / `change_output` (Definition 4.3) / `full_clusters` stubbed; **3 provided tests pass, 10 fix the contract for the stubs**, including FIFO conserving the stolen amount exactly, haircut touching >5× as many UTXOs, poison flagging >10×, every policy staying inside the descendant set with FIFO ⊆ poison, `extract_taint` splitting a straddling run rather than rounding, co-spend precision being exactly 1.000, Definition 4.3 declining every two-fresh-output transaction, and the 5%-reuse collapse below precision 0.2), and capstone M41 (incremental FIFO taint queues in the property layer with run-length coalescing, a maintained union-find cluster index re-pointing M39's machinery, and a BlockSci-shaped columnar transaction store so the two layouts can be compared on Table 3's queries). Cross-topic threads worked: 39 (clustering IS entity resolution — same union-find, hand-written conditions vs learned weights), 40 (both score a graph the adversary reads; prefer protocol properties and lossless measures), 12 (BlockSci's inline layout as the columnar argument), 32 (Table 3 read row-by-row is an HTAP brief), 1 (the taint policies as a RUM triangle), 33 (Elliptic's missing cross-time edges delete time-respecting paths by construction), 25 (build the aggregates and measure before reaching for a GNN), 36 (the infinite-COST conjecture rests on graph data resisting partitioning). Repos cloned for the code reads: `~/repos/RustyTaintChain` @ 4e12fd0, `~/repos/BlockSci` @ 14ccc93. -- 2026-07-27 — **topic 40 Security & Attack Graphs added** (third of the six graph use-case deep dives): study guide (**the list-vs-graph gap measured** — bench lane 1: synthetic AD-shaped directory, 2000 users / 400 groups / 1000 computers, five edge kinds (`MemberOf`, `AdminTo`, `HasSession`, `GenericAll`) with a planted over-privileged group, planted service-account groups and planted policy violations; the console answer is **5 direct tier-zero members, 8 with the one nested group expanded, and it never moves**, while attack-path reachability with 1% of users (20) in the over-privileged group goes **39 (1.9%) at zero sessions → 1969 (98.5%) at 100 sessions → 2000 (100%) at 500** — the cascade is that each newly exposed user's own sessions drag in everyone who is local admin on those machines, so **exposure is a function of collection time, not of how much privilege exists**; separately, with the gateway shut, **one Domain Admin token left on an ordinary workstation takes exposure from 8 users to 2000**; mean shortest attack path 6.03 hops, worst 8; lane 2 reference — **choke points are dominators**: in the reverse graph rooted at tier zero, node d dominates u iff every attack path from u crosses d, so d's dominator subtree IS its blast radius, pricing every single-node remediation in **0.8 ms vs 543 ms** for 3400 individual reachability re-runs (Cooper–Harvey–Kennedy iterative dominators, exact agreement with the delete-and-recompute oracle on every node in both regimes) — and the finding that matters more than the speedup, **tiering is what makes a graph have choke points**: identical 2000-user exposure, but the tiered directory has a group with a **1992-user (99.6%) blast radius** and greedy cuts 2000 → 8 → 5, while the flat one has **no single node whose removal frees a single user** and cutting the whole gateway set one node at a time reads 2000 → 2000 → 2000 → 2000 → 2000 → 8, so remediation is a set problem and the all-zeros dominator pass IS the report; lane 3 reference — Zanzibar Check by pointer chasing costs **19 → 559 tuple reads and 0.46 → 11.28 µs** as group nesting goes 2 → 32 while the Leopard-style flattened closure stays **4 → 12 probes at ~0.01 µs**, flat in depth, for a **1.7× entry tax** (6672 tuples → 11393 entries, closure quadratic in chain depth) and galloping intersection beats a linear merge by **>1000×** on a 1-vs-500,000 pair), 4 reading guides (BloodHound code read against ~/repos/bloodhound @ 1968388 — 104 `StringKind` node/edge kinds at `graphschema/ad/ad.go:28`, the four purposeful partitions `Relationships`/`ACLRelationships`/`PathfindingRelationships` (63 traversable kinds, the attacker's alphabet) / `PostProcessedRelationships` (**31 kinds that are derived, not collected** — `AdminTo`, `CanRDP`, `DCSync`, ADCS `ESC1..ESC13` — a materialized view refreshed by the four-stage pipeline at `analysis.go:346`), principal sets as **roaring bitmaps** (`cardinality.Duplex[uint64]`, `post.go:244`) and parallel BFS with `CheckedAdd` on a thread-safe bitmap as the visited set (`membership.go:81`), `tiering.go:37` `IsTierZero`, `agt.go` selector expansion diffed against previous state; Ammann/Wijesekera/Kaushik CCS'02 read in full — monotonicity ("the attacker never needs to backtrack"), the Sheyner numbers it replaced (**5 hosts, 8 exploits → 5,948 nodes / 68,364 edges / 2 hours / 229-bit state space** vs **at most 229 nodes** monotone), no negation in preconditions + `preConds ∩ postConds = ∅` ⟹ `markAttributes` is O(|A|²·|E|) converging in ≤|A| layers, `findMinimal`/`findAll`/`findShort` with Results 1–3, minimum attacks NP-complete but minimal easy, the 3-host example (60 attributes, 30 instantiated exploits, only 8 attributes ever change value), and **§2.3's three-sentence cut-set paragraph** which lane 2 makes precise; plus MulVAL CCS'06 read in full — logical attack graphs as tabled-Datalog derivation graphs (derivation nodes = AND, fact nodes = OR, primitive vs derived facts), XSB tabling for cycles and memoization, Theorems 1–3 (O(N²) derivation steps / O(N²) graph size / O(δN²) = O(N² log N) build), "useless edges" as a why-provenance test, **1000 fully-connected hosts on a Pentium 4** where Sheyner's tool blew up at 10 (Fig 14) and Sheyner's own 10-host/5-vuln case producing a **10-million-edge** graph in 15 min; Zanzibar ATC'19 read in full with SpiceDB @ 8422483 anchors — the relation-tuple grammar and why the user slot holds a userset, the three rewrite leaf kinds (`_this` / `computed_userset` / `tuple_to_userset`), Check as ∃-tuple ∨ ∃-userset-with-recursive-Check with concurrent leaf evaluation and subtree cancellation, **Leopard** (`GROUP2GROUP` ancestor→descendants, `MEMBER2GROUP` user→direct parents, membership = `O(min(|A|,|B|))` skip-list-seek intersection = topic 23's galloping intersect doing authorization; **1.56M QPS median, <150 µs median / <1 ms p99**, offline snapshot pipeline + Watch-fed incremental layer at ~500 updates/s, one tuple change → tens of thousands of index events), **zookies and the new enemy problem** (Example A neglecting ACL update order, Example B applying an old ACL to new content; the `≥` semantics is what lets Safe requests outnumber Recent by two orders of magnitude), hot spots §3.2.5 (consistent-hashed distributed cache forming "cache trees", **timestamp quantization to 1 or 10 s** so cache keys collide, lock table against stampedes, hot-object prefetch) and the surprise that a **10% check cache hit rate prevents 500K internal RPC/s** of hot-spotting, scale (>2 trillion tuples / ~100 TB / >10M QPS / Check Safe p50-p95-p99 = **3.0 / 9.46 / 15.0 ms** / >99.999% for 3 years) and the SpiceDB map of which parts are inherent vs Google-shaped (`graph/check.go:99→165→304→539→567`, `membershipset.go` set algebra **with caveats** so a result can be "maybe", `lookupsubjects.go:430` reverse arrow traversal, `dispatch/keys/computed.go:58` a `uint64` over a *canonicalized* expression, `singleflight.go:47` = Zanzibar's lock table verbatim, `defaultConcurrencyLimit = 50`); SLEUTH USENIX Sec'17 read in full — provenance graphs, the dependency-explosion problem, a main-memory dependence graph at **<10 bytes/event** vs ~250 B/edge for Neo4j-class stores and ~3 KB for STINGER/NetworkX (32-bit ids, events stored inside subjects, variable-length encoding down to 4-byte subject-event and 16-bit object-event records, delta timestamps, **6-byte bidirectional edges**, 38M events in **329 MB**, <100 ns decode), the tag design (t-tags benign-authentic/benign/unknown × c-tags secret/sensitive/private/public, and the **split of code vs data t-tags worth 1305× against 4.68×** for a single tag), four objective-based detection policies, **backward analysis as Dijkstra with tag-derived edge costs** (unknown→benign = 0, benign→benign = high, unknown→unknown = 1, stopping as soon as an entry point joins the shortest-path tree) and forward analysis pruning 100–500×, Table 11's end-to-end **38.5M events → 130** (297,100×) with a 54,517× average, and Table 7's 174 correct / 0 incorrect / 2 missed across eight DARPA campaigns where >99.9% of events were benign), experiments crate `attack-experiments` (lane 1 provided in `ad_graph.rs` with `AdConfig::tiered()` as the clean-directory preset; `chokepoint.rs` stub — `immediate_dominators` + `blast_radius`, with `exposure`, `rank_chokepoints` and the `blast_radius_naive` delete-and-recompute oracle provided; `authz.rs` stub — `check_pointer` with cycle protection and optional memoization, `LeopardIndex::build`, `intersect_galloping`, with the store generator and linear-merge straw man provided; **3 provided tests pass, 9 fix the contract for the stubs**, including exact dominator-vs-oracle agreement on every node in both directory regimes, index-equals-pointer-chasing on every user × group pair, cycle termination, and galloping-beats-merge by >1000×), and capstone M40 (edge-kind-filtered variable-length reachability as a Cypher procedure over M31's storage with the traversable-kind mask as a roaring set, a one-pass dominator choke-point procedure over the topic-18 CSR, and a Zanzibar-shaped `check(subject, resource#relation)` with a maintained closure index on the property layer). Cross-topic threads worked: 26/23 (roaring principal sets, galloping intersect), 27 (derived edges and the Leopard closure as materialized views; MulVAL's graph as a Datalog derivation), 1 (lane 3 is a RUM triangle), 37 (SpiceDB's bounded scatter-gather, and Zanzibar hedging to Spanner/Leopard but *never* between its own servers), 18 (CSR traversals), 12 (SLEUTH's encoding as the columnar argument), 33 (provenance as a contact sequence), 39 (both topics score a graph against an adversary who reads the score). -- 2026-07-26 — **topic 39 Fraud Rings & Identity Graphs added** (second of the six graph use-case deep dives; per user, 39-43 proceed without per-topic review): study guide (**camouflage kills row scores, measured** — bench lane 1: 5000×5000 Zipf(0.7)×Zipf(0.8) background, 50k edges, planted 25×100 block at density 1.0; precision@|fraud users| at camo/fraud-edge {0, 0.5, 1, 2}: degree-rank **0.00/0.28/0.60/0.76** (misses economical fraud, lights up only once camouflage inflates the row), obscurity-rank **0.52/0.00/0.00/0.00** (mirror image — dies the moment camo buys popular columns) — both are functions of the fraudster's own row and he tunes camo ≈ 0.5 to slip between them; FRAUDAR column-weighted peeling reference holds **F = 1.00 in every regime** while unweighted g degrades **1.00/0.95/0.69/0.65** (camo glues the block to the power-users × hit-products core); peel of a 100k×50k-node / **1,019,984-edge** graph in **~0.2 s** at F = 1.00; Fellegi–Sunter lane: 15,000 records (5000 entities × 3 dups, 5 fields pools [200 500 3650 200 2000] typo [.10 .07 .03 .12 .05]) — naive **112,492,500 pairs → blocked 271,012 (415×)** via two passes (last name OR dob), sampled **u = [0.0052 0.0021 0.0003 0.0051 0.0006] ≈ 1/pool**, EM **m = [0.80 0.86 0.94 0.78 0.90]** vs analytic (1−t)² [0.81 0.87 0.94 0.77 0.90], p = 0.184, link at 12 bits: **precision 0.989 / recall 0.992 in 48 ms**), 4 reading guides (FRAUDAR KDD'16 read in full — axioms, g(S)=f(S)/|S|, column weights 1/log(d+5), greedy peel O(|E| log |V|), Theorem 2 ½-approximation, Theorem 3 camouflage-resistance (camo lands on honest columns, block columns never change), F above 0.95 for 200×200 injected blocks under all four camo attacks, Twitter 41.7M users/1.47B edges → 4031×4313 block at 68% density with 57% hand-labeled fraud vs 12-25% controls; Winkler 2006 survey pp. 1-22 — FS decision rule R=P(γ|M)/P(γ|U) with T_λ/T_μ + clerical band proved optimal, per-field log2(m/u) weights, exact matching misses over 25% of census matches → Jaro-Winkler comparators, EM (Winkler 1988), multi-pass blocking 10¹⁷ → 10¹² pairs keeping 99.5% of matches, BigMatch 100M×4B at ~100k pairs/s with 10 passes in one data pass, 1990 census clerical 3000×3mo → 200×6wk; FlowScope AAAI'20 read in full — laundering = dense multi-step flow on k-partite X→W→Y, f_i=min(in,out), g=(1/|S|)Σ[(1+λ)f_i−λq_i] λ=4 so parking/camouflage LOWER the score, CBank 6.13M accounts/43.98M transfers with a labeled real ring (4 sources/12 mules/2 destinations ≈452M yuan): FAUC 0.761/0.843 vs FRAUDAR 0.529/0.704, F1 ≥ 0.9 down to 76M vs 180M injected — covered as guide + exercise 5, no stub; splink code read @ 04189f5 with 14 verified anchors — linker.py:66 façade, training.py:163 estimate_u_using_random_sampling / :231 one-EM-session-per-blocking-rule, expectation_maximisation.py:225 (E :18 / M :193), comparison_level.py:148 with match weight log2(m/u) :426 + _tf_adjustment_sql :667, graded levels comparison_level_library.py:406/:458/:493, predict.py:203 prior+weights → 1/(1+2^(−mw)), blocking.py:747 passes as SQL self-joins, clustering.py:43 → connected_components.py:121, dialects.py:24 one model on DuckDB/Spark/SQLite/PostgreSQL :270/:402/:532/:674), experiments crate (review_graph.rs PROVIDED — Zipf background + planted block + Zipf(1.5) popularity-biased camouflage + both naive rankers, 3 tests green incl. obscurity 0.75 → under 0.3 at camo 2; fraudar.rs + er.rs stubs with 6 contract tests: log-weighted F ≥ 0.9 with and without camo / unweighted F below 0.7 at camo 2 (measured 0.643) / g(returned) ≥ g(planted)/2; u within 0.005+expect of 1/pool / per-pass EM p,m within 0.05 of labeled empirical with the blocked field NaN / match-weight gap over 20 bits / blocking ≥ 20× / precision ≥ 0.95 recall ≥ 0.9; **the design discovery: a fixed-u EM over the unioned blocked candidates degenerates to fitted p → 1.0** (every candidate agrees on a blocking key by construction, class U cannot explain it) — the fix IS splink's API shape, one session per pass excluding its own blocking field, m averaged; margins that keep the contracts honest: block density 1.0 required (0.9 → log F 0.702), camo 4 breaks even log weights (0.619), 20×80 wide-short block caps per-column camo at 20 edges ≈ 3.06 weighted degree below block g 4.97, threshold 12 bits clears the coincidence patterns dob+city ≈10.3 / dob+first ≈10.6 / last+phone ≈10.95 where 8 bits chains precision down to 0.85; reference verified 9/9 then reverted, 0 warnings, bench prints lane 1 + `[stub …]` banners via catch_unwind), PLAN §39, capstone M39 (dense-block peel as a procedure over M31 storage reading weighted degrees off the topic-18 CSR + write-time identity resolution with blocking-key indexes, FS weights in the property layer, incremental union-find; targets: ~5M edges/s peel on 10M-edge synthetic, per-insert resolution latency at 1M records with two blocking indexes, precision/recall vs lane 3's 0.989/0.992). Same verified-facts-then-agents-write workflow; splink newly cloned, FRAUDAR/FlowScope/Winkler PDFs to /tmp, all read. -- 2026-07-26 — **topic 38 GraphRAG & Agent Memory added** (first of the six approved graph use-case deep dives, FalkorDB's core market; pilot — review before 39-43): study guide (**the path-finding collapse measured** — bench lane 1: mean rank of the true answer among 17 candidates, chance = 9.0 — mention-count ranking (vector RAG's shape) **1.00 at 1 hop → 9.21 at 2 hops → 8.71 at 3**; BFS distance **9.51/8.95/9.15** at all hops — coverage without association; PPR reference restores **1.00/1.00/1.00** since restart mass from both seeds SUMS at the meet node; one PPR query, 100k nodes / ~400k directed edges, 30 power iterations = **56.6 ms**; bi-temporal store reference: 10k entities × 10 job changes → **100,000 edges kept, 10,000 current, as-of scan 0.09 ms** — nothing deleted, any moment answerable), 4 reading guides (HippoRAG NeurIPS'24 read in full — hippocampal index analogy, 2-step OpenIE, synonymy τ=0.8, PPR damping 0.5, node specificity |Pᵢ|⁻¹; R@2/R@5 MuSiQue 40.9/51.9, 2Wiki 70.7/89.1, HotpotQA 60.5/77.7; 10-30× cheaper 6-13× faster than IRCoT; AR@5 2Wiki 37.1→75.7; Südhof path-finding case; Microsoft GraphRAG 2404.16130v2 read in full — 600-token chunks/gleanings, exact-match dedup with duplicate-count edge weights, hierarchical Leiden (graspologic), degree-ordered bottom-up community summaries, shuffled map-reduce with 0-100 helpfulness; Podcast 8,564 nodes/20,691 edges + News 15,754/19,520, indexing 281 min gpt-4-turbo; comprehensiveness win 72-83%, C0 = 26,657 tokens ≈ 2.6% of TS, 9-43× fewer; Claimify 34.18 vs 25.23 claims/answer; Zep 2501.13956 read in full — episode/entity/community tiers, §2.1 bi-temporal four timestamps, LLM edge invalidation keeps expired edges, dynamic label propagation, φ→ρ→χ retrieval; DMR 94.8 vs MemGPT 93.4, LongMemEval 60.2→71.2% with latency 28.9→2.58 s and context 115k→1.6k tokens, temporal +38.4%, regression single-session-assistant −17.7%; GraphRAG-SDK code read @ f42ab3d — fixed 9-step IngestionPipeline pipeline.py:35 with mandatory lexical graph + concurrent mentions∥index :175, 2-step GLiNER-then-LLM extraction graph_extraction.py:89, 4-strategy resolution ladder up to embedding+LLM llm_verified_resolution.py:192, vector+fulltext indices INSIDE FalkorDB vector_store.py:35, survivor-pattern dedup :228, rule-based router router.py:19, 9-step MultiPathRetrieval multi_path.py:48 with four parallel chunk paths and cosine rerank top_k=15), experiments crate (kg.rs PROVIDED — synthetic path-finding instances, 3 tests green; ppr.rs + temporal.rs stubs with 6 contract tests: PPR is a distribution + chain decay + meet-node rank 1; invalidate-without-delete t_invalid=Some(200)/t_expired=Some(205) + event-time reconstruction + late-fact known-vs-true split; bench lanes 2-3 print `[stub …]` until solved; reference verified 9/9 then reverted, 0 warnings), PLAN §38, capstone M38 (PPR as graph procedure over topic-18 CSR, bi-temporal versioning; targets: PPR recall@5 ≈ 1.0 where direct mention is chance, under 100 ms on 100k nodes, as-of within 2× current-only). -- 2026-07-26 — **topic 37 Distributed Query Execution added** (new topic, added to PLAN.md this session; second of the two approved scaling topics, completing the pair with 36): study guide (**the fan-out tail measured** — bench lane 1 run: analytic table P(any slow)=1−(1−p)ⁿ — at p=1/100: **1.0% for n=1 → 63.4% at n=100 → 99.3% at 500 → 100% at 1000**; at p=1/10,000 still **18.1% at n=2000** — fan-out exponentiates rarity into certainty, the component's p99 becomes the service's median at n≈70 since 0.99⁷⁰≈0.5; simulated 100-leaf scatter-gather at 1-in-100 slowness, 20k queries: **one-leaf p50/p95/p99 = 5.6/9.6/10.0 ms, wait-for-all-100 = 1000/1000/1000 ms, wait-for-95% = 9.6/9.9/9.9 ms** — the paper's Table 1 shape reproduced, good-enough results delete the tail; exchange-as-iterator ASCII, hedge timeline ASCII, DataFusion-vs-DistSQL production shapes), 4 reading guides (Volcano exchange TR CS/E 89-007/SIGMOD'90 verified against the PDF read in full — anonymous inputs so parallelism is one more iterator, packets through shared-memory queues, master/slave propagation-tree forking + primed processes, end-of-stream counted per producer 3×4=12, §4.4 broadcast-by-pinning + merging exchange must keep producers' records separate + exchange-in-the-middle makes flow control obsolete + fork-vs-reuse is a run-time switch, §4.5 two-level buffer locking never-hold-pool-lock-during-I/O + restart removes hold-and-wait = deadlock-free + ~100-instruction spin-locks + read-ahead/write-behind daemon, §4.6 vs GAMMA shared-memory/top-down/bushy vs shared-nothing/bottom-up/left-deep, §5 Sequent Symmetry 12×80386 numbers: **20.28 s single-process vs 28.00 s no-fork = 25.73 µs/record/exchange, forked 4-process pipeline 16.21 s beats single-process, packet sweep 171 s at 1 rec/packet → 94 at 2 → 15.0 at 50 → 13.7 at 83** = batching is a 12× swing, vectorization's argument made with processes; Tail at Scale CACM'13 verified against the PDF read in full — variability sources incl. SSD-GC-×100-reads, 63%/18% arithmetic, Table 1 real service 1/5/10 ms leaf → 40/87/140 ms at 100% vs 12/32/70 ms at 95% with slowest-5%-of-requests = half the p99, **hedged requests at p95 delay ≈5% extra load, BigTable 1000 keys/100 servers hedge-after-10 ms: p99.9 1,800→74 ms at +2% requests**, tied requests with cross-server cancellation + ≤1 ms stagger Table 2: idle p99.9 98→61 ms (−38%), with-terasort 159→108 ms (−32%), tied+terasort≈idle-unhedged at <1% disk overhead, probe-first loses 3 ways (staleness/estimation/herding), micro-partitions ~20/machine = 5% shed steps, latency-induced probation via shadow requests, canary requests on every Google fan-out, mutations easy: Paxos quorums inherently tail-tolerant; DataFusion RepartitionExec code-read with 20 verified anchors — RepartitionExec repartition/mod.rs:1150 + preserve_order :1160 = the merging exchange with per-(input,output) spill channels :398-538, BatchPartitioner :560 with **pinned seed-0 REPARTITION_RANDOM_STATE :592** so same-key-same-partition always (joins depend on it), partition_iter :825 routes whole batches round-robin but rows by hash via create_hashes :854 + strength-reduced % :675, distributor_channels.rs channels() :55 + Gate :62 + send :131 = N unbounded buffers with one global gate that parks senders only when ALL are non-empty (prevents distribution deadlocks in join plans), Partitioning enum partitioning.rs:117; **real finding: EnforceDistribution retired into EnsureRequirements** ensure_requirements/mod.rs:159, enforce_distribution.rs:18/:76 are helpers + the retirement note; cockroach DistSQL code-read with 18 verified anchors — checkSupportForPlanNode distsql_check.go:214, mustWrapNode :312 for no-processor-equivalent nodes, **PartitionSpans distsql_physical_planner.go:971 = the topic-36 bridge: range ownership becomes the parallel plan**, createPhysPlan :3604, OutputRouterSpec data.proto:149 with PASS_THROUGH/MIRROR/BY_HASH/BY_RANGE :152-:160 = Volcano's routing policies as a protobuf enum, Flow flowinfra/flow.go:72/Setup :272/Run :566, Outbox colrpc/outbox.go:50/:218/:323 + Inbox inbox.go:57/:212/:333 = exchange's two halves over gRPC with Inbox.Next an ordinary iterator — anonymous inputs surviving a network hop, hashRouter rowflow/routers.go:538 + vectorized HashRouter colflow/routers.go:443), experiments crate compiles: fanout.rs PROVIDED (two-mode leaf 1-10 ms fast/1000 ms stall, closed form, scatter_gather max + 95%-frac variant — 3 tests pass: 63.4%/18.1% exact arithmetic, simulation within ±2% at 20k trials, leaf-tail-becomes-service-median) — exchange.rs (Exchange::partition round-robin-cursor/splitmix64-hash to k outputs: deterministic-and-complete + balance-within-one-row + merge_sorted k-way keeps the multiset sorted) and hedge.rs (request_with_hedge fire-second-copy-only-past-delay: 10 ms hedge cuts p99.9 ≥10× + extra load <10% + zero-delay-doubles-requests) are `todo!()` stubs — 6 tests fail as todo panics; **reference solution verified 9/9 then reverted**: round-robin 229.6 M rows/s, hash 543.0 M rows/s balance 1.002, 8×500k merge 80.6 M rows/s, hedge@10 ms **p99.9 1000→18.3 ms at +0.5% requests** (hedge@0 = +100%, the degenerate case) — the paper's 1,800→74 ms shape; distq_bench lane 1 RUN (tables above), lanes 2-3 armed behind catch_unwind; notes.md predictions vs measurements + all verified anchors + PDF facts; M37 log: scatter-gather over M36's slots + in-engine exchange (hash for join build, round-robin for scans, per-producer end-of-stream counting) + hedged reads on slot replicas with p95 delay through topic-35's admission layer, targets = near-linear scale-up 1→2→4→8, p99.9 with one stalled shard within 2× no-stall when hedging, hedge overhead ≤5%. Same verified-facts-then-agents-write workflow; no new clones (datafusion/cockroach under ~/repos), Volcano + tail-at-scale PDFs to /tmp, both read in full. -- 2026-07-26 — **topic 36 Sharding, Partitioning & Rebalancing added** (new topic, added to PLAN.md this session; first of the two approved scaling topics — 37 distributed query execution is next): study guide (**mod-N's failure measured** — bench lane 1 run on 1M hashed keys: growing N→N+1 moves **80.0% at 4→5, 83.4% at 5→6, 88.9% at 8→9, 94.1% at 16→17** vs the ring's ideal 1/(N+1) = 20.0/16.7/11.1/5.9% — the closed form is exact, k mod N == k mod N+1 iff k mod N(N+1) < N (CRT), so movement = N/(N+1) and *worsens as you grow*; and the skew hashing provably can't fix: Zipf traffic on 16 hash shards, 10k keys/500k samples, hottest shard carries **9.5% at s=0.8 (1.5× the 6.25% ideal), 14.7% at s=1.0 (2.4×), 23.8% at s=1.2 (3.8×)** because a hash maps one key to one shard — only range splitting *between* keys or hot-key replication answers it; Dynamo strategy 1→2→3 table (fixed-partitions+movable-ownership beats boundaries-follow-node-identity: strategy-1 bootstrap "almost a day", strategy-3 metadata 3 orders smaller, partition-as-file), ring ASCII, redis MOVED-vs-ASK mermaid, cockroach split/merge/rebalance trigger table, edge-cut vs vertex-cut ASCII), 4 reading guides (Dynamo SOSP'07 verified against the PDF — MD5 128-bit ring, vnodes, preference list skipping same-physical-node, R+W>N with production (3,2,2), sloppy quorum + hinted handoff, per-range Merkle anti-entropy, §6.2's three partitioning strategies with Fig 8's efficiency numbers, imbalance 20%-low-load vs 10%-high, 99.94% one-version reads; PowerGraph OSDI'12 verified — α≈2 natural graphs, Twitter in-degree α=1.7 and 1%-of-vertices≈half-the-edges, Thm 5.1 random edge-cut = 1−1/p (87.5% at p=8), Thm 5.2 replication from the degree distribution with gains growing as α falls, Thm 5.3 vertex-cut ≤ ghosts of any edge-cut, greedy Cases 1-4, coordinated vs oblivious; redis cluster code-read with 14 verified anchors — CLUSTER_SLOTS=16384 cluster.h:23, keyHashSlot CRC16 & 0x3FFF + hash-tag carve-out :59, getNodeByQuery cluster.c:1191 → clusterRedirectClient :1443, CLUSTER_REDIR_ASK :1397 vs MOVED :1432 (MOVED updates the client slot map, ASK is one-shot + needs ASKING :1680), migrating_slots_to/importing_slots_from cluster_legacy.h:343-344, SETSLOT state machine cluster_legacy.c:6072-6075; cockroach rebalancing code-read with 12 verified anchors — RangeMaxBytes 512 MB zone.go:257, load splits at 2500 QPS replica_split_load.go:34 / 500 ms CPU :52, split_queue.go:145/:194, merge_queue.go:138, the Decider's windowed per-key sketch split/decider.go:155/:222/:329 with PopularKeyCount/NoSplitKeyCount as honest failure counters, AllocatorAction allocator.go:125, StoreRebalancer store_rebalancer.go:114/:218 lease-transfers-first — range splits are *semantic*, between keys, the answer to the Zipf row), experiments crate compiles: placement.rs PROVIDED (splitmix64, modn_movement, Zipf harmonic-CDF sampler, hot_shard_share — exact-80% + hashed-keys + Zipf-hot-shard tests) + graphs.rs PROVIDED (planted-partition + preferential-attachment generators, edge_cut, random baseline — random-cut≈0.875=Thm-5.1 test) — 4 provided tests pass, zero warnings — hashring.rs (consistent-hash ring with vnodes: ≈1/(N+1)-movement-all-to-the-new-node + remove-moves-only-its-keys + more-vnodes-tighter-balance contracts) and partitioner.rs (one-pass LDG greedy: score = placed-neighbors × (1−|P|/C), balanced-within-slack + beats-random-by-40% + deterministic contracts) are `todo!()` stubs — 6 tests fail as todo panics; shard_bench lane 1 RUN (tables above), lane 2 (ring 4→5 movement ≈20% vs mod-N 80%, removal moves only the removed node's share, balance vs vnodes 1/8/64/512) and lane 3 (edge-cut at k=8 random-vs-greedy on community + power-law graphs) armed behind catch_unwind; notes.md predictions vs measurements + all verified anchors + PDF facts; M36 log: slot = hash(vertex_key) & 0x3FFF with hash tags, edges live with source vertex, MOVED/ASK-style redirects + per-slot migration state machine with dual routing, migration runs at topic-35's lowest admission priority, targets = movement ≈1/(N+1), edge-cut + replication beat random on power-law, p99 during live migration within 2× steady state. Same verified-facts-then-agents-write workflow; no new clones (redis/cockroach under ~/repos), Dynamo + PowerGraph PDFs to /tmp. -- 2026-07-26 — **topic 35 Overload Control & Resource Governance added** (new topic, added to PLAN.md this session; grew out of "more topics for maintaining a database in production"): study guide (**metastable failure measured** — bench lane 1 run on a deterministic virtual-clock queueing sim reproducing HotOS'21 Fig 2: 300 QPS server, clients time out at 1 s and retry once, single 10 s outage at t=30 s — at **280 QPS offered load the outage queues ~2,800 requests, every timeout fires a retry, offered load locks at 560 QPS and goodput is still 0 at t=199 s** (160 s after the trigger ended, provably forever: queue grows 260 req/s); at **140 QPS the identical trigger + identical 280 QPS storm heals at t=161 s** because 280 is below the 300 QPS capacity — the dividing line is **hidden capacity = capacity/(1+retries) = 150 QPS**, and recovery takes ~2 min for a 10 s outage because drain rate = headroom = 20 QPS; metastable stable/vulnerable/metastable lifecycle ASCII, work-amplification table (retry ×2, look-aside cache ×10 at 90% hit rate, failover herds, slow error paths), detection ladder response-time-recursive-vs-CPU-busy≠overloaded-vs-**queuing-time-local** + CoDel min-sojourn, DAGOR cursor mermaid, cockroach slots-vs-tokens table, redis edge-surfaces table), 4 reading guides (metastable HotOS'21 verified against sigops PDF — trigger vs sustaining loop, root cause = the loop not the trigger, Fig 2's 280/560/300 arithmetic, stable below 150 / recovery needs retries below 20 QPS, 3000-QPS-app-on-300-QPS-db cache example, Facebook link-imbalance 2-years-undiagnosed one-line MRU-pool fix, "emergent behavior… one cannot write a unit or integration test", Kraken live-traffic testing, trigger intensity 151-vs-299, reproduction needs Tene-honest load gen → topic 34; DAGOR SoCC'18 verified against arXiv:1806.04075 — subsequent overload Def 1 with 0.5×0.5=25% random-shedding math, **avg queuing time 20 ms over 1 s/2000-req windows** explicitly not response time (recursive false positives: DAGOR_r sheds at 630 QPS where DAGOR_q reaches the 750 QPS saturation) and not CPU, business priority hash table Login-highest/Pay-above-IM-100×-complaints copied down the call tree, 128 hourly-rotated user sublevels fixing τ/τ−1 oscillation (session priority rejected: users re-rolled by logout/login), Algorithm 1 α=5% multiplicative-down β=1% additive-up on admit counts via priority-histogram prefix sums, collaborative shedding piggybacks the cursor upstream so rejects cost the overloaded server nothing, ~50% higher success than CoDel/SEDA on M², fairness uniform M¹–M⁴; redis code-read with 16 verified anchors — EVPOOL_SIZE 16 evict.c:36, evictionPoolPopulate :134 sampling maxmemory-samples=5 config.c:3223, getMaxmemoryState :384, performEvictions :532 before each command, OOM gate is_denyoom_command server.c:4391 → performEvictions()==EVICT_FAIL :4485 → rejectCommand oomerr :4498 reject-before-work, -BUSY server.c:2130 after busy_reply_threshold script.c:150, output-buffer limits checkClientOutputBufferLimits networking.c:5151 / async close :5215 as slow-consumer backpressure, CLIENT PAUSE pauseActions server.c:4850 — single thread can't shed by priority so every surface converts an unbounded queue (memory/replies/time) into a bounded fast error; cockroach admission code-read with 13 verified anchors — package doc admission.go:1 shift-queueing-out-of-the-goroutine-scheduler-into-reorderable-WorkQueues, slots-vs-tokens grantKind :54 concurrency-for-CPU vs rate-for-IO-because-compaction-debt-lands-later, requester/granter :178/:198, WorkQueue (tenant, WorkPriority int8 ladder admissionpb.go:23 LowPri=MinInt8…UserHighPri=50, FIFO ts) work_queue.go:303/Admit :813, kvSlotAdjuster AIMD on **runnable-goroutines-per-CPU sampled at 1 ms** kv_slot_adjuster.go:16/:46 = queuing-time detection in scheduler clothing, ioLoadListener L0 file/sub-level thresholds io_load_listener.go:69/:77 = topic 4's write-stall signals promoted to node-wide policy), experiments crate compiles: sim.rs PROVIDED (open-loop arrivals, client-timeout-but-server-does-the-work-anyway work amplification, retries at arrival+timeout, outage trigger, Policy trait admit/allow_retry/observe_queuing — 3 exact-arithmetic tests pass incl. offered=1600=exactly-2× in the collapsed window and vulnerable-without-trigger-is-invisible) — tokenbucket.rs (retry budget: burst-then-deny, steady-rate 10-in-1s, idle-does-not-accumulate) and admission.rs (DagorGate: healthy-admits-all, overload-sheds-lowest-first-never-prio-0, additive recovery) are `todo!()` stubs — 6 tests fail as todo panics; overload_bench lane 1 RUN (table above), lane 2 (retry budgets 15 vs 25 QPS straddling the 20 QPS headroom — one heals, one never) and lane 3 (DAGOR-lite at 2× overload: goodput + per-priority success + admitted p99, vs no-control FIFO starving everyone) armed behind catch_unwind; notes.md records the 2,800-queued/560-locked/260-per-s-growth arithmetic + predictions for lanes 2-3; M35 log: per-query priority + queuing-time cursor on the executor (1 s/2000-query windows), -BUSY-style fast reject with retry-after hint, plan-time memory gate (DENYOOM per query), targets ≥80% of saturated throughput at 2× overload + lane 1 reproduced-then-fixed on the real engine. Same verified-facts-then-agents-write workflow as topics 33/34; no new clones (redis/cockroach already under ~/repos). -- 2026-07-26 — **topic 34 Debugging & Production Diagnosis added** (new topic, added to PLAN.md this session; grew out of "how do I debug a database in production"): study guide (**coordinated omission measured** — bench lane 1 run on a virtual clock, deterministic: 1M ops, service 1 µs, 100 ms stall every 100K ops, arrivals every 10 µs — closed-loop reports p50/p99/p99.9/p99.99 all **1.0 µs** (only max sees a stall) while open-loop reports **p99 = 90.0 ms, p99.9 = 99.0 ms, p99.99 = 99.9 ms** = a **90,000× lie at p99**; each stall queues ~10K arrivals, 9 stalls → ~9% of requests carry decaying queueing delay, so the worst 1% are exactly the ≥90 ms victims — provable arithmetic, not noise; three-failure-currencies diagram wrong-answers/too-slow/crashed → replay/measurement/forensics; rr one-diagram; redis 3-tier cost table always-on-one-compare / armed-160-sample-rings / on-demand-doctors; RocksDB PerfLevel mermaid), 4 reading guides (rr ATC'17 verified against arXiv:1705.05937 PDF — record boundary = user/kernel interface, nondeterminism = syscall results + async-event timing, RCB the only deterministic HW counter so execution point = (RCB, registers), seccomp-bpf + RR-page + 2-byte-syscall→5-byte-call rewrite avoids 4 ctx switches/syscall, <2× slowdown, one-thread-at-a-time so weak-memory unobservable; Gregg flame graphs CACM 59(6) 2016 — x-axis is alphabetical merge not time, width = sample fraction, on-CPU vs off-CPU for lock/fsync waits; redis code-read with 13 verified anchors — slowlogPushEntryIfNeeded slowlog.c:103 with :104 negative-disables/:105 >=-logs, slowlogCreateEntry :28 arg/string trimming, latencyStartMonitor/latencyAddSampleIfNeeded latency.h:50/:63 zero-cost-when-off macro discipline (default 0 config.c:3271), LATENCY_TS_LEN=160 latency.h:17, same-second max-coalescing latency.c:82, LATENCY DOCTOR createLatencyReport latency.c:182, MEMORY DOCTOR object.c:1421, watchdog sigalrmSignalHandler debug.c:2643→logStackTrace :2115; RocksDB code-read with 10 verified anchors — PerfContext perf_context.h:305 thread-local via get_perf_context() :342, PerfLevel ladder perf_level.h:27 kDisable=1/kEnableCount=2/kEnableWait=3/kEnableTimeExceptForMutex=4 (agent corrected brief's kEnableWaitForMutex against the clone), PERF_TIMER_GUARD perf_context_imp.h:27 compiled out under NPERF_CONTEXT, PerfStepTimer RAII perf_step_timer.h:13, HistogramBucketMapper 109 buckets histogram.h:21/:84, StatisticsImpl/recordTick statistics_impl.h:42/statistics.cc:549 as the global tier), experiments crate compiles: workload.rs PROVIDED (StallModel virtual clock, closed_loop the-liar vs open_loop charging completion−intended, exact-arithmetic tests incl. lat[101]=991_000 — 3 provided tests pass) — histogram.rs (LogHistogram: linear below 2^sub_bits then 2^sub_bits sub-buckets/octave, contracts = est≥true within 1/32 relative error + merge-equals-bulk + memory-never-grows) and slowlog.rs (redis semantics exactly: >=-threshold logs, negative disables, ring evicts oldest, ids monotonic across reset) are `todo!()` stubs — 6 tests fail as todo panics; debug_bench lane 1 RUN (table above), lane 2 (histogram record ns/op + percentile error vs sort-everything on 10M latency-shaped samples) and lane 3 (**the observability tax**: hot-loop ns/op bare vs +clock-pair vs +histogram vs +slowlog — M34's overhead budget) armed behind catch_unwind; notes.md predicts lanes 2-3 + records the lane-1 arithmetic; M34 log: GRAPH.SLOWLOG port (FalkorDB src/slow_log/slow_log.c exists in C) + parse/plan/execute/serialize step timers behind a PerfLevel dial, level 0 provably free via lane 3, full level <5% on M11 suite, before-shot = lane 1 reproduced on the real engine under an induced stall. Same verified-facts-then-agents-write workflow as topic 33; no new clones (redis/rocksdb/FalkorDB already under ~/repos). -- 2026-07-22 — **topic 33 Temporal Graphs added** (new topic, added to PLAN.md this session): study guide (**the static-condensation lie measured** — bench lane 1 run: 2000 nodes, contacts uniform over a 10K-tick horizon, static BFS vs time-respecting reachability for 20 sources: at 4K contacts static claims 25,031 reachable pairs but only 137 have a time-respecting witness = **99.5% false positives**, falling 97.5% → 56.9% → **0.0% at 64K contacts** where temporal saturates to all 39,980 static pairs — the transition is sharp, static reach is the T→∞ limit; contact (u,v,t,λ) model + valid/transaction/bitemporal axes ASCII; reachability-is-not-transitive so "shortest" splits into earliest-arrival/latest-departure/fastest/shortest; storage-menu table snapshot-per-t / event-log-first-Raphtory / anchor+delta-AeonG / MVCC-as-history keyed on AT-TIME cost + anchor/delta mermaid), 4 reading guides (Wu et al. VLDB'14 — condensing lies, four minima that need four algorithms, Dijkstra's subpath invariant dies to a cheap-prefix-misses-the-bus counterexample, one-pass O(n+M) earliest-arrival over a time-sorted stream in Rust, dominance lists for fastest/shortest, time-expanded O(M) DAG as the materialized-view alternative; Paranjape/Benson/Leskovec WSDM'17 — δ-temporal motifs, the 36-motif derivation, window-scan DP with cnt[i][j] fragment counters and the expire-shortest-first/insert-longest-first correctness orders, stars-cheap-triangles-O(m√m), blocking-vs-non-blocking fingerprints; AeonG VLDB'24 verified against arXiv:2304.12212v2 — per-VERSION lifespan ω in transaction time, FOR TT AS OF/FROM..TO scoped to MATCH, VP/VE/EP three-clocks split, GC-as-migration Algorithm 1 riding the reaper thread = the 9.74% headline, KV SkipList keys type+Gid+ω with A/D anchor/delta bit, adaptive anchoring Eq 1 three bands, legal check Eq 2 + both-store consults, 5.73× storage / 2.57× latency numbers; Raphtory code-read on fresh clone with 8 verified anchors — EventTime(i64,usize) tiebreaker timeindex.rs:28 solving exactly Wu's λ=0 tie-order problem in the type system, TimeIndex/TCell size-adaptive enum ladders, TPropCell time→offset into columnar PropColumn, WindowedGraph derives-Copy = BETWEEN as a zero-copy lens + TimeOps::window composing on every view type, db4 segments as the batch-into-arrays correction; caught stale name: TimeIndexEntry renamed EventTime), experiments crate compiles: events.rs PROVIDED (gen_contacts λ=1 sorted, static_reachable the-liar, earliest_arrival_oracle as deliberately-Bellman-Ford fixpoint so lane 2's one-pass speedup is a measurement not a tautology, replay_at_time naive AT-TIME oracle — 3 provided tests pass) — temporal_reach.rs (one-pass earliest_arrival, matches-oracle-on-3-random-streams + respects-start-time + λ=0-chains contracts) and snapshot.rs (AnchorDeltaStore append/at_time/replay_len, matches-full-replay + anchor-spacing-bounds-replay` sections (first sentence defines the concept assuming zero DB-internals background, then a real-numbers example / ASCII diagram / the guide's existing code sample, then why-it-matters; each step uses only terms defined in earlier steps, terms of art defined parenthetically at first use), then the navigation section ("How to read the paper (with the concepts in hand)" for papers, "Where each step lives in the code" with anchors grouped by step for code reads), Questions/Takeaway/References verbatim at the end; all existing assets (diagrams, code samples, file:line anchors, tie-backs) preserved and reorganized, H1 titles unchanged so SUMMARY.md links held. Executed by 17 parallel agents (2-3 topics each; two stalled mid-batch — topics 03 and 09 refinished by follow-up agents); exemplars hand-written first (reading-drepper.md for paper reads, reading-turso-btree.md for code reads). Verification: all 186 guides pass structure checks (≥4 steps, exactly one problem statement and References), 185 SUMMARY.md link titles match on-disk H1s with 0 mismatches, all 49 mermaid diagrams parse under mermaid 11.6.0 (jsdom harness), fence-and-backtick-aware angle-bracket scan clean (2 false positives from multi-line backtick spans), mdBook build green. -- 2026-07-12 — **restructure rollout: all 179 remaining reading guides across topics 00-25 and 27-32 rewritten as self-contained chapters** (topic 26 was the pilot, previous entry), executed by 8 parallel agents (4 topics each) against the same spec: concept-first H1 titles replacing "Reading guide — ...", Sources blocks replaced by 2-4 sentence framing leads, one inline Rust-ish code sample of the core algorithm added where the guide lacked one (skips documented for pure surveys / guides already carrying equivalent code), all existing content kept (diagrams, line-anchor tables, questions, tie-backs), `## References` appended with Papers (arXiv) + Code (GitHub) links carrying the old reading advice, filenames unchanged to avoid link churn; the 32 topic READMEs' guide lists updated to the new titles; one agent (topics 16-19) hit context limits with 2 files left — reading-umbra-tidy-tuples.md (retitled "Umbra & copy-and-patch: the war on compile latency" + copy-and-patch memcpy/patch-holes sample + References) and reading-sqlite-vdbe.md (References) finished by hand; SUMMARY.md link titles regenerated centrally by script from the actual on-disk H1s rather than agent reports (179 updated); verification: zero old-style H1s, zero Sources blocks, zero guides missing References, fence-and-backtick-aware bare-angle-bracket scan across all 186 guides found one genuine hazard (`HashMap` in topic 7's ae guide, backticked), mdBook build green. The whole book now reads as chapters instead of pointers. -- 2026-07-12 — book-quality pass, three moves: (1) paper audit against dbscholar's citation-PageRank ranking (rmarcus.info — pulled the underlying data.json, 11,867 SIGMOD/VLDB/CIDR/PODS papers) — resources/papers.md gains a "Modern systems & directions" section and 6 topic READMEs gain "Further references" (Kung-Robinson OCC '81, Calcite, Spark SQL, Kipf/Neo/Bao learned optimization, Photon, Velox, GAMMA, Dremel, Lakehouse+Delta Lake, MillWheel, CockroachDB); (2) all `~/repos/...` code references linkified to their GitHub repos (scripted, fence-aware: 143 links across 115 files) so the online book's code pointers resolve; (3) **restructure demo on topic 26** — all 7 reading guides rewritten as self-contained chapters: concept titles ("HyperLogLog: count distinct in 12 KB" not "Reading guide — ..."), framing lead instead of a Sources block, an inline Rust code sample of each core algorithm (HLL add/merge + Ertl count skeleton, blocked-bloom 6-probe loop with golden-ratio remix, cuckoo kick loop with the XOR involution, PGM shrinking-cone add_point, roaring galloping intersect, BRIN one-sided range prune, Morton interleave64 magic masks), and a "## References" section at the bottom (papers with arXiv links, code with GitHub links); filenames kept (`reading-*.md`) to avoid link churn; SUMMARY.md + README titles updated; mdBook build verified locally (mdbook-mermaid install + build green). If the format lands, roll it out to the other 32 topics. -- 2026-07-12 — PLAN.md expansions backfilled into the four already-scaffolded topics (the plan-only commit 2fb095a now has matching study material): topic 7 gains "Bolt: the third answer" — RESP/pgwire/Bolt framing-typing-streaming table (Bolt's PULL{n}/DISCARD = protocol-level backpressure, §4's problem solved at the wire) + reading-bolt-packstream.md anchored on FalkorDB's *removed* Bolt server read frozen via `git show 0b11a00b3^:src/bolt/` (#2170, 2026-07-08): session-sequence ASCII with handshake bolt_api.c:803/version-clamp :845-864, RUN-executes-but-PULL-streams :467-482/:504-521 decoupling, PackStream marker nibbles bolt.c:11/:21/:36 + graph types Node 0x4E/Rel 0x52/Path 0x50 in the type system, why-was-it-removed as a question, M7 stretch = Bolt listener beside RESP; topic 13 gains "The query-language landscape" — 6-language table (Cypher/GQL/SQL-PGQ/SPARQL/Gremlin/Datalog) on model/matching/composability/pushdown + reading-query-languages.md (SIGMOD '22 GQL+PGQ paper, count-2-paths-in-a-triangle under homomorphism/isomorphism/trail as the same-pattern-three-answers demo, family-tree mermaid, kuzu Cypher.g4 anchor, M13 rule: keep the AST GQL-shaped — quantified path patterns + explicit path-mode field); topic 20 gains "Parallelism: OpenMP inside SuiteSparse, rayon in Rust" — saxpy3's costed static scheduling (coarse/fine tasks GB_AxB_saxpy3.c:22-48, nthreads-from-flopcount slice_balanced.c:418, parallel flopcount pass :219) vs rayon work-stealing (join/mod.rs:93 inline+push+steal, registry.rs:248 crossbeam_deque Stealer) + reading-openmp-vs-rayon.md with the static-vs-stealing trade table, no-native-Rust-GraphBLAS note (crates are FFI), new M20 checkbox: document each OpenMP→rayon mapping decision; topic 26 gains "Geo indexes: 2D keys through 1D indexes" — valkey GEO as geohash-in-a-zset (interleave64 geohash.c:52 → 52-bit Morton score, geohashEstimateStepsByRadius helper.c:64, 9-cell scan geo.c:375 + haversine verify = bloom's candidate-then-verify control flow) + reading-geo-indexes.md (Z-order seams vs Hilbert, Guttman R-tree/GiST, S2 prefix-containment vs H3 hexagons, M26 mapping: Morton key through the existing sorted property index). SUMMARY.md gains the 4 new guides so they appear in the book. -- 2026-07-11 — topic 32 scaffolded (added to the plan this session, so the scaffold-all-topics run is complete again): study guide (**the HTAP problem measured** — bench lane 1 run: 1M-row store behind one coarse lock, fixed 2 s window per mode: writes alone = 11,438,647 writes at p99 333 ns; writes + a free-running full scanner = **69 writes** with p99 **7.49 seconds** — not slowdown, *starvation*: std Mutex is unfair, the scanner re-wins the lock after each ~0.6 ms scan (3261 scans) and the parked writer never gets in; interference at its worst is zero writes, and the coarse lock is deliberate — mitigations ARE the topic; the freshness/isolation/cost trilemma triangle; the architecture-menu table HANA-delta+main / HyPer-fork() / TiFlash-learner / F1-Lightning-CDC / pg_duckdb-offload keyed on one-copy? freshness isolation; the changelog-is-the-glue mermaid tying topic 27's thesis to every split design; the same-fold-four-costumes thread: topic 4 LSM minor compaction = HANA delta merge = TiFlash segmentMergeDelta = FalkorDB delta-matrix flush), 4 reading guides (TiDB VLDB '20 — columnar-copy-as-Raft-LEARNER so the replica costs no write-quorum latency, freshness-is-a-WAIT with tiflash LearnerRead.cpp:35 doLearnerRead + :61 waitIndexTimeout as the anchor, one-planner-two-engines via tidb find_best_task.go:535/:1841/:1878 TiFlash-paths-retained-so-cost-not-topology-decides, learner-log vs CDC-changelog trade question; TiFlash DeltaTree — Segment.h:84 delta-over-stable ASCII with MemTableSet/DeltaValueSpace.h:65 as a-little-LSM-inside-the-delta, Delta/MinorCompaction.h why-compact-the-delta-at-all, DeltaIndex.h:27 as the structure that makes delta+stable merge reads cheap (the thing our scan_sum_a lacks), DeltaMergeStore.h:668 segmentMergeDelta with our merge_preserves_scans test as its correctness condition, MVCC-versions-in-both-layers GC question linking topic 31's causal stability; HyPer ICDE '11 + HANA SIGMOD Record '12 paired as the one-copy family — fork() CoW-page ASCII (snapshot cost ∝ dirtied pages = MVCC-where-the-version-chain-is-the-page-table, GC is exit()), snapshot-ages-until-re-fork = lane 3's apply interval in OS clothing, HANA delta+main = our replica.rs minus segmenting, HANA's-trilemma-corner-is-exactly-what-lane-1-measures; F1 Lightning VLDB '20 + Özcan SIGMOD '17 survey — CDC-fed HTAP with zero OLTP changes, **safe-timestamp = applied_lsn productionized** (reads never wait, served stale-but-consistent at the max fully-applied ts — the opposite choice from doLearnerRead), Changepump ordering question = topic 27 changelog + topic 29 Spanner timestamps, the survey's copies×engines quadrant with every cell trading the same three currencies), experiments crate compiles: row.rs PROVIDED (RowStore = rows + every-write-appended-to-log changelog + scan oracle + skewed_key + percentile — 2 provided tests pass) — replica.rs (ColumnarReplica delta+main: apply/scan_sum_a/merge_delta with delta-overrides-main + highest-lsn-per-key-wins + merge-preserves-scans-and-sorts contracts, freshness_is_visible via applied_lsn gap — TiFlash DeltaTree in miniature, 4 tests), learner.rs (read_wait over an apply schedule: first-batch-covering-read_index, Some(0) if already applied, None = waitIndexTimeout — doLearnerRead as arithmetic, 3 tests) are `todo!()` stubs — 7 tests fail as todo panics; htap_bench lane 1 RUN (table above; original fixed-200K-writes design serialized into minutes behind ~0.6 ms scan lock-holds — switched to fixed-2s-window per mode, making the writes-completed collapse the headline number), lanes 2 (scan row-vs-delta-heavy-vs-merged + freshness max-lsn-gap vs batch 1K/10K/100K) and 3 (learner-read wait distribution vs apply interval 1/10/100, 50K reads demanding lsn==now) armed behind catch_unwind; notes.md predicts lanes 2-3 + records the starvation surprise + the RwLock exercise; M32 log: Lightning-shaped not TiFlash-shaped (no consensus group until M15, decoupling = zero primary changes), M27's changelog feeds a delta-matrix replica, router advertises applied_lsn safe timestamps + freshness-bound routing with read_wait-or-fallback, before-shot recorded: 69 writes/2s when analytics shares the copy, success = restoring the 11.4M with scans elsewhere. Cloned tiflash + tidb. -- 2026-07-11 — topic 31 scaffolded (the last topic): study guide (consensus-vs-CRDT as the same problem in opposite currencies — agree-on-an-order vs design-so-order-doesn't-matter, 1-RTT vs 0-RTT, unavailable-in-minority vs available-under-any-partition; SEC via join-semilattice mermaid; CvRDT/CmRDT table with where-each-lives-in-this-crate; the zoo table clock/lww/counter/orset/rga/graph with the one idea per structure; **LWW's lie measured** — lane 1 run: two replicas, 20K writes each, LWW map: 10 hot keys + sync-every-write loses **94.98%** of writes, 1000 keys + sync-every-100 loses **88.34%**, even 100K keys + rare sync loses **12.45%** — "eventually consistent" without conflict semantics is not a semantics, priced; sequence-CRDT integration ASCII with the interleaving dragon; code-reading table across all five cloned repos), 4 reading guides (Shapiro SSS'11 + INRIA RR-7506 — SEC's three clauses, the CvRDT⇔CmRDT equivalence proof, the catalog reading map ending at §4's graphs where concurrent addEdge∥removeVertex is declared application-specific = the dangling-edge problem M31 inherits, causal-stability GC question; Kleppmann arc JSON-CRDT '17 + move-op '21 + Local-First Onward!'19 — ops-address-identities-not-paths with automerge op_set2/op.rs:52 succ as deletion-by-successor-ops, the concurrent-move duplicate/cycle problem and its undo/redo total-order fix, the does-move-survive-in-graphs M31 design question; sequence CRDTs in production — yrs block.rs:160 ID=our-Dot / :1302 Item with origin+right_origin=YATA's pair vs RGA's single parent / :1415 Item::integrate as the loop our rga.rs apply implements plus run-coalescing splits, diamond-types merge.rs:142 self-described "bastardization" + yjsspan.rs:29 INSERTED/NOT_INSERTED_YET retreat/advance = only-be-a-CRDT-at-merge-time, Loro/Fugue maximal-non-interleaving with the letter-soup demo, automerge-vs-loro bench as scratch-project exercise per deps convention; cr-sqlite as THE-database-goes-multi-master — crsql_as_crr clock-row-per-CELL diagram, local_writes/mod.rs:83-133 db_version bookkeeping = Lamport-clock spine, compare_values.rs version-tie-broken-by-VALUE-comparison = deterministic convergence with zero clock trust, changes_vtab.rs replication-endpoint-as-virtual-table, delete-wins-for-rows vs our add-wins-for-nodes tension question, the M31 change-feed schema design question), experiments crate compiles: clock.rs PROVIDED (Dot + VClock tick/covers/merge/partial_cmp-None-defines-concurrent, modeled on automerge clock.rs:109/:145) + lww.rs PROVIDED (register/map with (ts,replica) total order; merge counts its own discards so lane 1 can price the lie) — 6 provided tests pass — counter.rs (G/PN with semilattice-law tests + why-PN-is-two-G-Counters), orset.rs (add-wins over Dots: add-tags-fresh-dot / remove-kills-observed-dots, concurrent-add-beats-remove + remove-covers-all-observed-tags + seeded-permutation convergence), rga.rs (insert-after-parent + skip-larger-(counter,replica)-siblings + tombstones-still-anchor, idempotent apply, delete∥insert-after-it convergence), graph.rs (OR-Set nodes/edges + LwwMap props composition; dangling-edge-hidden-NOT-deleted test: edges()-filters-to-visible-endpoints, re-add-resurrects; props keyed by node id survive remove/re-add vs automerge's key-by-creation-op as an exercise) are `todo!()` stubs — 18 tests fail as todo panics; crdt_bench lane 1 RUN (table above; also caught state-based-sync's quadratic cost live — sync_every=1 ships the whole map per write, workload shrunk and the delta-CRDT motivation written into the bench comment), lanes 2-4 armed behind catch_unwind (OR-Set gossip storm + tombstone census, RGA 50K-char trace + tombstone bloat, graph dangling storm 100-removes∥500-edge-adds + resurrection count); notes.md flags lane 1's honest caveat (counts merge-time discards only — locally-overwritten writes don't show, so it's a lower bound); M31 log: node/edge identity must be Dots not user ids (cr-sqlite's auto-increment-PK trap), dangling policy locked hide-not-delete, props LWW-with-HLC (topic 29), anti-entropy v1 whole-state → v2 db_version-watermark deltas, deliverable = same workload through M15 Raft vs active-active with latency histogram + concrete-conflict table. All 32 topics scaffolded. -- 2026-07-10 — topic 30 scaffolded: study guide (the-shape-of-the-problem ASCII — regular append-mostly writes vs range+selector+aggregation reads; the baseline finding measured: delta+varint lands at a shape-blind **11.00 B/sample** because raw f64 values dominate — whatever the codec does about timestamps is a rounding error until it attacks the value bytes, hence XOR; Gorilla→Prometheus/VM→IOx lineage mermaid with Monarch and BtrDB as the bracketing extremes; five-system table on codec / time organization / label index / out-of-order policy; TSDB=LSM-keyed-by-time thesis with retention=drop-the-oldest-level), 4 reading guides (Gorilla VLDB '15 + prometheus chunkenc/xor.go — prediction-error framing with the dod/XOR ASCII, paper bucket table vs prometheus's retuned 14/17/20 buckets (:195-208) as buckets-are-workload-parameters, writeVDelta :226 window reuse, :396 tDelta+=dod as the whole model in one line, no-random-access-by-design, entropy-floor bit accounting question; prometheus tsdb — full architecture ASCII head/WAL/2h-blocks/exponential compaction, head_append.go:436/:481/:688-693 as the exact head.rs contract (ErrOutOfOrderSample vs ErrTooOldSample), OutOfOrderTimeWindow head.go:168 quarantine design, MemPostings postings.go:60/:403 = topic 23's inverted index with labels as terms, the two famous failure modes high-cardinality + churn; VictoriaMetrics + InfluxDB 3 paired as two-rebuttals — VM partition.go:75 rawRows→parts LSM-said-out-loud, nearest_delta2.go:15 byte-aligned varint batches with optionally-lossy precisionBits vs Gorilla's exact bits, index_db.go:124 tagFilters cache + churn invalidation, vs IOx WAL→Arrow QueryableBuffer→Parquet-on-object-store (influxdb3_wal/lib.rs:75-98 SnapshotTracker, queryable_buffer.rs:41) as topic 28's landing-zone applied to metrics, vertical-integration-vs-commodity-formats trade table, how-much-of-Gorilla's-win-was-really-sorting question; Monarch VLDB '20 + BtrDB FAST '16 — monitoring-must-not-depend-on-what-it-monitors ⇒ RAM-first lazy durability, push-vs-pull, distribution-typed values as the schema cure for cardinality, query pushdown; BtrDB's aggregate tree — min/mean/max/count per 64-way node ⇒ query cost ∝ pixels not samples, downsampling as the index structure itself, CoW versions), experiments crate compiles: gen (scrape jitter, gauge/counter/constant/random shapes, OOO arrivals, label sets with the unique-instance cardinality bomb) + bits (MSB-first BitWriter/Reader + sign_extend so the stub is the algorithm not bit plumbing) + baseline (zigzag varint delta) PROVIDED — 7 provided tests pass, lane 1 RUN (table above, decode ~270-330 Msamples/s — byte-aligned codecs have a real throughput edge over bit-packed Gorilla, the actual design axis is ratio-vs-decode-speed) — gorilla.rs (paper dod buckets + XOR leading/trailing windows; bit-exact roundtrip incl. bucket edges, constant ≤~2 bits/sample, gauge beats raw 3×, random must FAIL to compress >8 B/sample — the codec wins on regularity not magic), head.rs (in-order fast path + bounded OOO window + TooOld refusal + LWW merge flush that feeds the in-order-only encoder — prometheus semantics exactly), index.rs (MemPostings-style (name,value)→sorted-ids + shortest-list-first k-way intersect, brute-force oracle, cardinality-bomb-counted test) are `todo!()` stubs — 15 tests fail as todo panics; tsdb_bench lanes 2-4 (gorilla ratios per shape, OOO tax sweep 0-50%, selector latency at 100K series) armed behind catch_unwind; notes.md predicts all lanes; M30 log: history chunks per (entity, attribute), `MATCH ... AT TIME t` = latest_write_before ≤ t so M29's MVCC read path generalizes to time-travel, storage split by age (custom hot chunks → M28 Parquet cold), Gorilla dod survives for changelog timestamps but property values need dictionary+RLE not XOR, BtrDB-shaped rollup tree over the M27 changelog for graph-evolution queries. -- 2026-07-10 — topic 29 scaffolded: study guide (the-problem-priced motivation table — measured conflict probability of the bank workload itself: 0.3% of 8-txn batches collide at zipf θ=0.5 but **29.9% at 0.9, 86.2% at 1.1, 99.6% at 1.3** — contention is the common case at real-workload skew; design-space mermaid rooted at textbook 2PC with the three escapes labeled on the edges — move-the-decision-into-the-data (Percolator), replicate-the-coordinator (Spanner), remove-runtime-agreement (Calvin), decompose+batch (FDB); one-table five-system summary keyed on concurrency control / clock / cross-shard atomicity / blocking window; Percolator-in-six-lines with THE-COMMIT-POINT marked), 4 reading guides (Percolator OSDI '10 + TiKV — three-column-families ASCII (data/lock/write with write-CF-as-commit-index), lifecycle sequenceDiagram with the any-reader-resolves note, TiKV walk actions/prewrite.rs:37 (pessimistic_action + secondary_keys args as post-paper hardening) → commit.rs:64 with the :57 duplicate-commit-returns-Ok idempotency arm → check_txn_status.rs:92/:241 + MissingLockAction :458 as production resolve_lock → cleanup.rs:24 Rollback records our sim skips → latch.rs/scheduler.rs local-vs-distributed conflict split, txn_status_cache; Spanner OSDI '12 + HLC OPODIS '14 paired — bound-the-ERROR-vs-bound-the-SKEW fork diagram, commit-wait derivation, HLC rules + the l≤max-pt anti-Lamport-drift bound, uncertainty-interval restarts, CRDB walk hlc.go:38/:411/:471/:517 (UpdateAndCheckMaxOffset crashes the node — maxOffset is a promise) + txn_coord_sender.go:113 interceptor stack + txn_interceptor_committer.go:128 parallel commits with STAGING-is-implicitly-committed :195-205 as Percolator's-resolve-idea-shaving-a-latency-round; Calvin SIGMOD '12 — agree-on-inputs-not-outcomes diagram, sequencer/scheduler/executor layers, deterministic-locking-kills-both-deadlock-and-2PC, OLLP reconnaissance for dependent txns with the graph-traversals-are-the-ultimate-dependent-txn M29 question; FoundationDB SIGMOD '21 — unbundled roles ASCII, ConflictSet.cpp:947 detectConflicts over the :224 SkipList as the-whole-SI-check-in-one-data-structure, CommitBatchContext :504 batch-is-the-unit, masterserver-is-barely-a-counter, ResolverBug.cpp injectable-wrong-answers as DST culture beyond crash injection, vs-Calvin and vs-Percolator design reads), experiments crate compiles: kv.rs PROVIDED (the three Percolator column families as HashMaps/BTreeMap, strictly-monotonic TSO, latest_write_before range scan, 2-shard cluster, Zipf transfer workload — 4 provided tests pass) — tpc.rs (2PC coordinator with 4-point CrashPoint injection + recovery-from-durable-state-only; blocking_window_demonstrated test names the flaw), percolator.rs (get/prewrite/commit_primary/commit_secondaries/resolve_lock with roll-forward-iff-primary-committed tests, no-lock-leak on failed prewrite, total-conserved-after-rollback), hlc.rs (send/recv rules; monotonic-under-backward-clocks, l-bounded-by-max-pt over 1000 skewed messages, concurrent-events-collide-without-node-id-tiebreak as an assert_eq teaching test) are `todo!()` stubs — 14 tests fail as todo panics; txn_bench lane 1 RUN (conflict table above), lanes 2 (abort rate vs θ + bank invariant) and 3 (20K-txn crash storm, crash every 100th cycling 4 points, blocked_aborts counts the blocking window empirically) armed behind catch_unwind; notes.md predicts lane 2/3 numbers before implementation; M29 log: shard by node id, cross-shard edge = prewrite both adjacency sides with primary on u, supernodes are the Zipf head — per-shard adjacency segments turn the WW hotspot into scatter-gather reads, every protocol step gets a kill point with no-dangling-half-edges as the invariant. -- 2026-07-10 — topic 28 scaffolded: study guide (the latency ladder measured and priced — local NVMe p50 0.10 ms vs raw S3 p50 14.17 / p99 112.99 ms = 140× median, **940× tail**, and why everyone moved anyway: $/GB, 11-nines durability = replication-is-someone-else's-problem, scale-to-zero; the 2008→Snowflake→Aurora→Socrates→Neon→SlateDB lineage mermaid; Neon's four-box data flow with the safekeepers-commit-fast / pageserver-serves-pages split; five-system design-space table with what-crosses-the-network as the axis; WAL-rule-promoted-to-architecture rosetta linking topic 27's Kafka thesis), 5 reading guides (Aurora SIGMOD '17 — the-log-is-the-database, 6-copy 4/6 AZ+1 quorums over 10 GB protection groups, 35× network amplification killed, VDL-replaces-2PC question, commit=log-quorum-ack + recovery-without-REDO-at-compute; Socrates SIGMOD '19 — durability≠availability as THE decomposition, XLOG landing zone vs page servers vs XStore mapped onto topic 5's WAL lifecycle and onto Neon components, RBPEX = buffer pool made restart-durable; Snowflake SIGMOD '16 + Building-a-DB-on-S3 SIGMOD '08 paired — the prescient paper's three blockers (eventual consistency/no CAS/request cost) vs what fixed each (strong consistency 2020, conditional PUT 2024, immutability routed around all three), micro-partitions as CoW-clone-by-file-list, min/max pruning = topic 26's BRIN at cloud scale; Neon code walk — get_rel_page_at_lsn pgdatadir_mapping.rs:258 → Timeline::get :1227 → LayerMap::search layer_map.rs:448 as an LSM over (page, LSN), walredo.rs:173 = REDO on the read path in a sandboxed Postgres, branch_timeline_impl tenant.rs:4985 O(1) branches + the timeline.rs:4548 ancestor walk our stub reimplements, branch-aware GC retain-point question; SlateDB+Quickwit S3-first code walk — tablestore.rs:835 block-granular ranged GETs, cached_object_store part cache = our cache.rs in production form, **fence.rs:105 CAS-epoch fencing = consensus outsourced to S3 conditional PUT**, clone.rs:38 zero-copy clones, quickwit bundle hotcache footer + TimeoutAndRetryStorage :37 hedging with the AWS-recommends-it citation, pathology→countermeasure convergence table), experiments crate compiles: virtual-latency sim (charged-not-slept lognormal S3 with 2% 8× stragglers, NVMe, scripted Fixed model for exact contract tests) + block store + zipf + percentiles PROVIDED — 6 provided tests pass, tier_bench provided lanes RUN (numbers above) — LruBlockCache+TieredReader (touch-protects, 1/8-cache zipf hit-rate >50%, block-sharing hits), hedged_get (scripted 50ms-primary/1ms-backup/10ms-deadline ⇒ exactly 11ms, p99-halves-at-<10%-extra-GETs on straggler-heavy S3), BranchStore::get (parent-prefix visibility, sibling isolation, PITR historical branch points, 100-deep chains, branching-copies-nothing proven by version_count) are `todo!()` stubs — 13 tests fail as todo panics; notes.md predicts cache-fixes-the-median-hedging-fixes-the-tail and flags the O(n)-eviction-scan wall-time trap; M28 log: L0+WAL stay local (landing-zone lesson), manifest-CAS fencing not leases, branch at SST-list not page granularity, image-layer materialization deferred until ancestor walks profile hot. -- 2026-07-10 — topic 27 scaffolded: study guide (recompute-is-the-enemy with the priced motivation table — full recompute per 100-change batch: triangles 97.2 ms / wedge self-join 894.3 ms / re-BFS 24.7 ms vs µs-scale incremental targets; the one algebraic idea as a table — LINEAR ops stream deltas statelessly, BILINEAR joins need arranged inputs via Δ(A⋈B)=ΔA⋈B+A⋈ΔB+ΔA⋈ΔB, NONLINEAR distinct/aggregates need integrals; DBSP I→Q→D mermaid; timestamps/watermarks section: timely frontiers are proofs where Flink watermarks are heuristics; four-system comparison timely/DBSP/Materialize/RisingWave), 5 reading guides (Naiad+timely — could-result-in pointstamp protocol, MutableAntichain frontier.rs:380/update_iter :533, ChangeBatch :16 progress-updates-are-Z-set-shaped, worker.rs:235 step as the topic-7 event loop one layer up, the rosetta table frontier=vacuum-watermark; differential — consolidation.rs:24 = our from_updates verbatim, arrangements as LSM-of-batches with advance=compaction, join_traces join.rs:69 with the fuel/effort loop :348-395 as operator-level cooperative yielding, iterate.rs:192 Variable + bfs.rs:101-107 as the 40 lines our stub can't do, why deletion-in-recursion needs lattice times; DBSP — Q^Δ=D∘Q∘I with the chain rule as the compositional bombshell, feldera anchors z1.rs:221/integrate.rs:85/differentiate.rs:38/join.rs:123-350/delta0.rs, nested-circuits-vs-lattice trade, the M27 mapping: delta matrix DP−DM=ΔA and wedges need NO new state because the integrals ARE the adjacency matrices; Materialize+RisingWave — dogs^3 delta_join.rs:47 + half_join :315/:402 avoiding intermediate arrangements, indexes-are-arrangements-are-memory, RisingWave Op enum stream_chunk.rs:45 as Z-set-weights-as-protocol, hash_join.rs:158 degree tables :269 as hand-rolled weight bookkeeping vs one consolidation rule, barrier checkpoints, single-writer-gets-the-hard-parts-free table; Kafka NetDB'11 — dumb-broker/smart-consumer, offset=LSN rosetta, log-compaction=arrangement-advance=LSM-GC same operation three communities, exactly-once = where-do-offsets-live, M27's raw-log-vs-result-delta subscriber decision), experiments crate compiles: ZSet with consolidation + the distinct-is-not-linear load-bearing test, churn generator with set-semantics guard, full-recompute oracles (sorted-intersect triangles, hash join with weight multiplication, BFS) PROVIDED — 6 provided tests pass, ivm_bench baselines RUN (numbers above) — delta_join+IncrementalJoin (algebra-exact vs join(A+ΔA,B+ΔB)−join(A,B), 30-batch drift-free, deletes-retract), IncrementalTriangles (oracle-tracking under churn, K4-minus-edge=−2, <4K probes for 20 changes on 40K edges), SemiNaiveReach (insert-only BY DESIGN — deletion is differential's lattice territory, documented; matches re-BFS per batch, ≤4 relaxations/edge EVER, intra-component edges free) are `todo!()` stubs — 9 tests fail as todo panics; notes.md flags the honest suspicion that IncrementalJoin's Vec-merge state integration may dominate and re-derive why arrangements exist. -- 2026-07-10 — topic 26 scaffolded: study guide (indexes-are-bets framing with the measured motivation table — point-miss binary search 167 ns / BTreeMap 218 / HashSet 24 at 224 MB, vs blocked bloom's ~15-25 ns at 12 MB target; three-families ASCII filters/sketches/learned; bloom math block with the reproduce-it FPR derivation; bloom→blocked→cuckoo→xor→ribbon lineage mermaid; HLL sparse-30-bytes-to-dense-12KB story; PGM ε-window thesis), 6 reading guides (bloom→ribbon over RocksDB code — FastLocalBloomImpl bloom_impl.h:144 golden-ratio probe remix vs LegacyBloomImpl :364, CacheLocalFpRate :42 as the Poisson-crowding honesty function, ribbon as banded GF(2) solve with StandardBanding ribbon_impl.h:471 / num_starts_=slots−kCoeffBits+1 :504 and the build-can-fail-vs-monotone question; cuckoo+xor — partial-key involution getAltHash cuckoo.c:122, RedisBloom's LSM-of-subfilters growth CuckooFilter_InsertFP :256 vs the paper's fail-at-MAX_KICKS, delete-a-false-positive-corrupts-someone-else contract, xor peeling 1.23× vs bloom 1.44× vs ribbon 1.10×; HLL — hllPatLen :467 / Ertl tau+sigma :1016/:1033 replacing HLL++'s empirical bias tables, ZERO/XZERO/VAL sparse opcodes :380 with promote-at-3KB-or-rank>32, merge-is-a-semilattice AVX2 :1116; learned indexes — Kraska RMI's no-error-bound flaw, PGM_SUB/ADD_EPS pgm_index.hpp:32-33 + optimal hull PLA piecewise_linear_model.hpp:96/:154-190 vs our simpler shrinking cone, ALEX gapped arrays predict_position alex_nodes.h:1448 + exponential search :1462 with the degrades-in-space-vs-time-vs-write-amp scoreboard; roaring internals extending topic 23 — Store enum store/mod.rs:28-31, ARRAY_LIMIT=4096/RUN_MAX_SIZE=2048 as pure arithmetic container.rs:9-11, 3×3 pairwise kernel dispatch, three-adaptive-encodings table roaring/HLL-sparse/GIN-varbyte; postgres indexam as the classical baseline — _bt_search nbtsearch.c:100 + Lehman-Yao moveright :211, GIN varbyte ginCompressPostingList ginpostinglist.c:196 + pending-list-as-mini-LSM, BRIN bringetbitmap brin.c:301 as the one-sided filter that's 10,000× smaller than bloom when clustering holds), experiments crate compiles: splitmix64/hash2/fastrange PROVIDED (avalanche + coverage tests pass), filter_bench motivation lanes RUN — **binary search miss 167 ns ≈ 23 dependent misses, BTreeMap 218, HashSet 24 ns at 224 MB, hit≈miss 169** (the walk is the cost, not the compare) — BlockedBloom (no-FN + FPR<2.5%@10bpk + <4×-theory + halves-8→16 tests), CuckooFilter (no-FN at 90% load, FPR<1%@12-bit, delete-leaves-others-intact — the test bloom can never pass, graceful-full-failure), Hll (err<3% at 1K/100K/5M, merge registers EXACTLY equal union's), and LearnedIndex (window ≤2ε+2 always contains true pos, uniform-1M <2K segments, ε holds on hostile powers-of-2+quadratic mix) are `todo!()` stubs — 15 contract tests fail as todo panics, notes.md has predict-before-measure table and the M26 call that learned indexes are NOT in scope (node IDs are dense — a plain array is already the perfect model). -- 2026-07-10 — topic 25 scaffolded: study guide (message-passing-is-SpMM with receipts — the message_and_aggregate table showing GCNConv = `spmm(adj_t, x)` at gcn_conv.py:273 and SAGEConv = `spmm(..., reduce=mean)` while GAT can't fuse because attention recomputes the matrix values per forward; associativity-as-query-plan — (AX)W vs A(XW) swaps which term carries the big dimension, 90× on Cora; GraphRAG closed-loop mermaid graph→embeddings→M14 vector index→hybrid Cypher), 7 reading guides (node2vec KDD '16 — second-order walk figure, p/q as BFS↔DFS knobs, the alias-table O(m·avg_deg) memory trap with rejection sampling as the fix, PyG Node2Vec.loss node2vec.py:135 as the SGNS reference; Kipf-Welling GCN — renormalization trick, gcn_norm anchors gcn_conv.py:45-71, GCN-forward-is-a-query thesis, oversmoothing as why-2-layers; GraphSAGE — sampling as a page budget (B·10·25 fan-out math), mean+lin_r≈concat, inductive-is-the-only-write-friendly-variant; GAT — SDDMM+softmax+SpMM kernel decomposition = topic 24's masked SpGEMM, a_src/a_dst per-node split as factor-out-of-join, materialized-vs-computed view line running exactly between GCN and GAT; PyG message-passing machinery — 10-stop code walk, COO gather-scatter materializes an m×d temp (145 MB on our bench) vs CSR spmm's zero temporaries = materialize-the-join vs pipeline-the-aggregate, message()-as-arbitrary-callable = Ligra's F-with-CAS tradeoff third community; TransE — relations as translations, symmetric-relation collapse, link-prediction-is-an-ANN-query so M14 serves KG completion natively; GraphRAG-SDK with systems eyes — vector_store.py:344 queryNodes / :219 SET embedding write path, relationship_expansion's ANN+k-MATCHes as the k+1-round-trip join to push down, multi_path's client-side cosine rerank, router-as-planner-with-no-cost-model, four systems smells), experiments crate compiles: CSR + SBM generator (ground-truth labels; O(m) inter-block sampling not O(n²) Bernoulli) + ring-of-cliques + dense Mat/glorot/softmax + SpMM + row-norm adjacency + uniform walks + dense GCN oracle PROVIDED and run — SBM 16,384 vertices/566K edges: uniform walks 42.8 Msteps/s, **SpMM 21.2 GFLOP/s = 81% of dense matmul's 26.2** (64-wide feature rows amortize the gather — fat RHS forgives sparsity, the number that makes M25 plausible) — node2vec_walks (4 tests: degree-stationary dist, p=q=1 ≡ uniform, q orders exploration on ring-of-cliques, p orders backtrack rate), train_skipgram (SBM intra-block cosine must beat inter by 0.2), gcn_norm+gcn_forward (dense-oracle 1e-4, sorted rows, transform-before-aggregate) are `todo!()` stubs. -- 2026-07-10 — topic 24 scaffolded: study guide (per-source vs whole-graph algorithm map; frontier-world vs algebraic-world mermaid with the honest trade — per-vertex tricks like Afforest's edge skipping vs LAGraph's batched matrix frontiers and atomics-free bulk ops; rmat-vs-uniform baseline table making skew the headline), 6 reading guides (GAP arXiv:1508.03619 + gapbs anchors — sssp.cc:44's redundant-relaxation-beats-bookkeeping bet, bc.cc:76 succ bitmap, cc.cc:69/:106/:129 Afforest's three phases, tc.cc WorthRelabelling, and the 5-graph matrix as topic 22's change-anything-different-number; Meyer-Sanders delta-stepping as the Dijkstra↔Bellman-Ford dial with gapbs thread-local-bins vs LAGr_SSSP's MIN_PLUS tmasked vxm + three implementation traps; Brandes '01 with the dependency-recurrence derivation exercise + gapbs-vs-LAGr_Betweenness table — the batched ns×n matrix frontier amortizes what frontier code cannot; Ligra PPoPP '13 — edgeMapData ligra.h:235-272, the m/20 threshold :238, dense/sparse/denseForward, PageRank-degenerates-to-SpMV lesson, m/20 = Beamer α/β = dot-vs-saxpy; Louvain→Leiden Sci Rep '19 — the disconnected-communities bug as topic 21's greedy-destructive trap, refinement as egg's keep-both-forms, ΔQ accumulator = SPA, aggregation = S·A·Sᵀ SpGEMM, determinism-needs-seeding for CALL algo.community; LAGraph analytics — FastSV7 mngp/hooking-as-one-mxv :102 + FASTSV_SAMPLES :335, TriangleCount's six formulations with the :44 urand-flips-to-saxpy exception, PageRankGAP-vs-PageRank as benchmark-specs-fork-implementations, and FalkorDB's proc_pagerank.c:197 already calling LAGr_PageRank = M24's pattern exists, re-plumb it), experiments crate compiles: weighted CSR + RMAT/uniform generators + heap Dijkstra with pop counter + pull PageRank + degree-ordered triangle count + union-find CC + O(n³) definitional BC oracle PROVIDED and run — RMAT scale 16 vs uniform same n/m: **15,645,988 vs 5,428 triangles (2,883×)** in 376/158 ms, PR 8-vs-6 iters (hubs slow L1 decay), Dijkstra 343K pops = 1.74×n stale-entry tax, 18,844 components at avg_deg 16 (RMAT's leaf quadrant strands vertices) — delta_stepping (bucketed, extremes-must-still-be-exact test + relaxation counters), brandes (must match the O(n³) oracle exactly on n=128, then sample GAP-style), and afforest (partition-equal + <50%-of-m edges-inspected bound) are `todo!()` stubs; RMAT skew assertion needed scale-aware bounds (19.1% top-1% share at scale 12, 36.6% at 16). -- 2026-07-10 — topic 23 scaffolded: study guide (inverted-index anatomy ASCII — analyzer → FST term dict → TermInfo{doc_freq, postings_range} → 128-doc Δ-bitpacked blocks with {last_doc, max_score} skip data; write-path mermaid making Lucene-segments-are-an-LSM explicit — tiered LogMergePolicy works for text because queries fan out anyway; the two speed tricks: compression-with-random-access + score upper bounds), 6 reading guides + 2 cross-linked (Zobel-Moffat CSUR '06 design-space map — TAAT vs DAAT, capped accumulators as 2006's WAND, merge-based construction = LSM before Lucene; Robertson-Zaragoza BM25 derivation ladder — eliteness ⇒ tf saturation at K1+1 ⇒ the static ceiling WAND needs, mapped to tantivy bm25.rs:8-59 with the 1-byte-fieldnorm quantization question; Ding-Suel SIGIR '11 block-max WAND with pivot diagram + four implementation traps (θ seeding, livelock-on-failed-refinement, k-boundary ties, docs-evaluated metric); Roaring — 4096 crossover derivation, kernel matrix, containers = GraphBLAS sparse↔bitmap lattice at 64K granularity; tantivy code walk — compression/mod.rs:3 128-blocks, skip.rs:93/:175/:186 SkipReader/block_max_score, term_info.rs:9-13, fst_termdict, block_wand_union.rs:8-24 find_pivot_doc, log_merge_policy.rs:20-24, 90-minute read order; RediSearch redisearch_rs Rust-rewrite — newly cloned — InvertedIndex core.rs:30/:75 chained varint IndexBlocks vs tantivy's immutable bitpacked segments, Encoder-as-type-parameter monomorphizing 11 codecs, gc_marker/unique_id cursor validation ↔ delta-matrix wait, new-block-on-delta-overflow, and the finding that RediSearch has NO block-max WAND — scored unions walk everything), experiments crate compiles: zipf corpus (term id = rank, df(t0)=99.9%) + tf-counting index builder with per-128-block max-BM25 metadata + saturating-BM25 + exhaustive TAAT oracle PROVIDED and run — 100K docs/7.9M postings built in 335 ms, common∧rare [t0 t12000] top-10 walks 99,964 postings in 6.34 ms at ~32 ns/posting (hash accumulate dominates — Q1's lesson again) while the rare term carries ~93% of the winning score = the WAND poster child measured, and vec two-pointer dense∧sparse AND costs O(|dense|) 52 µs for 172 hits = the roaring motivation measured — block-max `wand_topk` (recipe in doc comment, must match oracle top-k while scoring <25% of the postings) and mini-Roaring (array/bitmap containers, 3 density-crossing oracle tests) are `todo!()` stubs. -- 2026-07-10 — topic 22 scaffolded: study guide (OLTP↔OLAP benchmark map + the number-is-four-choices mermaid workload/data/harness/metric; choke points one-liner table — Q1 tiny-group agg = expression bench, Q6 2%-scan = the GB/s headline, Q9 = optimizer punisher, TPC-C = the D_NEXT_O_ID hot counter institutionalized; benchmarking-sins checklist linking topic 0's fair-benchmarking guide), 4 new reading guides + 3 cross-linked existing ones (Boncz TPCTC '13 choke-point taxonomy with duckdb dbgen/queries dir open + hidden messages — uniform data is why JOB exists, Q1's 4-6 groups make hash-agg invisible; YCSB SoCC '10 + go-ycsb zipfian.go:92-165 anchors — zetan/eta/alpha math, the two fast paths, scrambled-fnv rationale, coordinated-omission warning; OLTP-Bench VLDB '13 + benchbase TPCC anchors — keying/think times :85-100, NURand C_LAST load-vs-run constants :94-116, why nobody runs TPC-C honestly = 12.86 tpmC/warehouse; DuckDB tpch extension — dbgen as streaming TABLE FUNCTION tpch_extension.cpp:17-99 with answers/ shipped next to queries/ = benchmark-as-oracle, run-real-TPC-H-here recipe), experiments crate compiles: dbgen-lite lineitem + row-at-a-time Q1/Q6 oracles + YCSB A-F driver over BTreeMap with ns-percentile Hist PROVIDED and run — Q1 oracle 5.6 GB/s effective (HashMap per row even at 6 groups: CP1.2 measured), Q6 branchy oracle 15.7 GB/s (2% selectivity = perfectly-predicted branch, the crater is hiding at 50%), YCSB uniform A-F 2.88/4.15/3.72/4.40/1.11/2.85 Mops/s with E's scans 4× a point read — Zipfian/Scrambled generators (the actual YCSB math with head-frequency-vs-theory statistical contract tests) and q1_flat/q6_branchless columnar lanes are `todo!()` stubs; fixed a nearest-rank percentile off-by-one in the harness itself (harness bugs are results bugs). -- 2026-07-10 — topic 21 scaffolded: study guide (tool-per-guarantee table proptest→TLC→SMT→Lean with cost axis; e-graph = union-find + hashcons + congruence closure with egg's deferred rebuild = delta-matrix wait = LSM compaction — batch the invariant repair; equality-saturation loop mermaid; TLA+ spec-as-math with the measured state counts; Z3 DPLL(T) diagram + the euf_egraph.h:23 comment where Z3 cites egg back; Perceus RC as the Arc::make_mut compiler pass), 5 reading guides (AWS CACM '15 — 35-step S3 bug, exhaustively-testable-pseudo-code pitch, small-scope hypothesis; egg POPL '21 with full source anchors — egraph.rs:970 add / :1147 union / :1416 rebuild / :1346 process_unions fixpoint, machine.rs Bind/Scan/Compare pattern VM = topic 19's bytecode interpreter, extract.rs greedy find_best vs lp_extract, e-graph ≈ Cascades memo; Z3 TACAS '08 + src/ast/euf anchors — backtracking trail + justifications = WAL for unions, e-matching triggers = index choice; Specifying Systems + Ongaro's raft.tla — newly cloned — with the un-model-an-assumption exercise that re-derives terms; Beans + Perceus borrowed-vs-owned + reuse tokens with the proof-vs-TLC-vs-proptest calibration exercise), experiments crate compiles on egg 0.9: expr IR + hand-ordered fixpoint rewriter PROVIDED and run — ~30% cost reduction, ~2 µs/firing, and the planted trap measured: (a*2)/2 → strength-reduce fires before div-reassoc → stuck at (a<<1)/2 cost 5 where egg should reach cost 1 — egg_optimize is a `todo!()` stub with trap + never-worse-than-hand tests; **TLA+ WalReplication spec written AND model-checked** (tla2tools.jar downloaded, java 17): SyncCommit=TRUE → Durability holds over 1080 distinct states depth 14 in <1 s; SyncCommit=FALSE → TLC finds the 5-state data-loss trace Append→Commit→Crash→Failover after 123 states — the postgres synchronous_commit=off story, found exhaustively. -- 2026-07-10 — topic 20 scaffolded: study guide (format lattice hypersparse→sparse→bitmap→full with the actual switch tests from GB_convert_sparse_to_bitmap_test.c and GB_conform; one-mxm-four-engines mermaid — dot3 iterates the MASK so work ∝ nnz(M) vs saxpy3's coarse/fine × Gustavson/hash task scheduler with its flopcount pre-pass = cudf size/retrieve five years early; push-vs-pull BFS as vxm-vs-mxv with LAGraph's shipped α=8/β1=8/β2=512; delta matrices as LSM-over-matrices — DP=memtable, DM=tombstones, wait=minor compaction, delta_mxm's `(A*(M+DP))` fold), 6 reading guides (Davis TOMS '19+'23 — zombies/pending as the library's own deltas, iso matrices, 32-bit indices; SuiteSparse internals with saxpy3.c:22-60 scheduling-essay and hash>m/16⇒Gustavson anchors; Gustavson '78 + Buluç-Gilbert — SPA design space, symbolic/numeric two-phase; Beamer SC '12 direction-optimizing with the ICPP '18 linear-algebra translation; LAGraph — BFS template switch block anchors, ANY_SECONDI parent-without-comparisons, six triangle-count formulations, PageRankGAP; FalkorDB delta_matrix with fresh eyes — header state-table as spec, transposed twin, transpose-as-masked-copy sync, over-masking question — LAGraph newly cloned), experiments crate compiles: CSR+RMAT/uniform/path generators, SpMV, hash-SpGEMM, scalar BFS, hypersparse PROVIDED and run — SpMV 16-19 GB/s single-thread (gather tax vs 30 GB/s streaming), hash SpGEMM ~60-75 Mflop/s (15 ns/flop = the accumulator cost the SPA stub should crush), **hypersparse 50× index memory and 171× full-sweep** on the 10M-nodes/100K-edges FalkorDB shape — dense-SPA Gustavson and push/pull/direction-optimizing BFS (with per-level trace + path-graph-never-pulls test) are `todo!()` stubs. -- 2026-07-10 — topic 19 scaffolded: study guide (the spectrum tree-walker → bytecode VM → copy-and-patch → IR JIT → LLVM as a compile-latency-vs-run-speed trade with each system placed on it; produce/consume compiles the PIPELINE not the operators — push inverts control so tuples stay in registers; three JIT grains compared — postgres per-query, Umbra per-pipeline with adaptive Flying-Start→LLVM tiering, GraphBLAS per-kernel-specialization cached forever; DuckDB's deliberate no-JIT with the VLDB '18 tie as the counter-argument; M19 = expressions only, gate on measured cost), 6 reading guides (Neumann VLDB '11 pipelines/breakers + produce-consume inversion; SQLite VDBE — vdbe.c:1049 switch over 199 opcodes, register machine, OP_Yield coroutines as flattened-bytecode's free resumability; Umbra Tidy Tuples single-pass IR + copy-and-patch musttail stencils; postgres llvmjit_expr.c opblocks-per-EEOP-step + the jit_above_cost estimate-gate failure taxonomy + deform-JIT-as-the-real-win; GraphBLAS jitifyer encodify-hash → PreJIT → memory table → dlopen → invoke-cc ladder with FalkorDB cache-warming implications; cranelift-jit-demo declare→define→finalize→transmute ladder + per-node CLIF emission table for the stub), experiments crate compiles on cranelift 0.116: Expr enum + seeded generator, AST interpreter, and column-at-a-time vectorized lane PROVIDED and run — interp ~2.1 ns/node flat, vectorized 6-12× over interp across depths 2-10 (topic 11's number reproduced; both linear in nodes, no y-intercept until compile time adds one) — jit.rs compile() is a `todo!()` stub with bit-exact-vs-interpreter tests, jit_bench prints the three-way table with compile µs + e2e winner and survives the stub via catch_unwind. -- 2026-07-10 — topic 18 scaffolded: study guide (GPU-for-DB-people translation table — SIMT = topic 17's predication in hardware, coalescing = columnar × 32, shared memory = cache blocking made explicit; the bus decides the architecture — Crystal's regime A ship-per-query vs regime B device-resident, rewritten for Apple unified memory; libcudf size/retrieve two-phase + cooperative-groups probing = SwissTable at warp scale; Gunrock advance/filter + load-balance menu; CAGRA = HNSW with SIMT-hostile parts deleted by construction), 6 reading guides (Crystal SIGMOD '20 tile model + fair-CPU-baseline lesson; wgpu compute examples ladder incl. the hello_compute doc-comment our bench proves; libcudf join size/retrieve + shared-mem-until-spill groupby with cuco/cooperative-groups anchors — newly cloned; Gunrock Essentials bfs.hxx enactor + advance.hxx thread/block/merge_path dispatch anchors — newly cloned; CAGRA ICDE '24 + single-CTA kernel/shared-mem-hashmap/bitonic-topk anchors — cuvs newly cloned; Faiss GPU billion-scale paper — WarpSelect register k-select, memory-tier table), experiments crate compiles AND RUNS on Metal: GpuCtx + workgroup-reduction sum kernel PROVIDED with per-phase timings, gpu_bench crossover sweep run on Apple M3 Pro — **CPU wins at every size up to 16M elements** (~1.5 ms fixed dispatch floor, flat 16K→1M; even amortized the GPU reads the same unified memory at ~9 GB/s effective vs CPU 30 GB/s — regime B's bandwidth ratio doesn't exist for streaming ops on this machine, which IS the lesson), filter_count (one-atomicAdd-per-workgroup, WGSL skeleton provided) and l2_batch (one-invocation-per-target + row/column-major coalescing experiment) are `todo!()` stubs with exact-match/1e-3 tests, notes.md predicts where arithmetic intensity finally flips the verdict. -- 2026-07-10 — topic 17 scaffolded: study guide (ports×latency mental model — M-series 4 FMA ports × 3cy ⇒ ~12 independent chains, the four autovectorization failures, branchy/branchless/compress filter shapes with AVX-512 vpcompress vs NEON's missing compress, vshrn movemask idiom, FastLanes interleaved layout), 7 reading guides (simdjson VLDB '19 + arm64 nibble-LUT classification / PMULL prefix_xor quote parity / LUT-shuffle compress emulation / flatten_bits over-write-under-advance — newly cloned; polars-compute float_sum STRIPE=16 + pairwise-128 and simd_filter! with per-ISA compress + selectivity-adaptive scalar bit-iteration fallback; hashbrown Group×3 backends + memchr Vector — newly cloned — with the finding that hashbrown's NEON group is 8 BYTES so vceq output already IS the bitmask, no vshrn, while memchr keeps 16 lanes and narrows; SimSIMD under its numkong rename — per-instruction latency/port tables in headers, f64-upcast accumulation, 4-target streaming states = M14's scoring loop, and the FCMLA lesson: the specialized instruction measured 2.3× SLOWER than 4 plain FMAs; SIGMOD '15 selective-store/load primitives + vertical hash probing + gather-costs-a-load-per-lane; FastLanes VLDB '23 1024-lane transposed layout; Mojo SIMD[type,width] parametric-width ladder), experiments crate compiles: dot naive+unrolled-8 PROVIDED and run — 10.89 → 42.12 GB/s, 3.9× from accumulator count alone, zero intrinsics — wide-f32x4 and NEON vfmaq 4-accumulator rungs are `todo!()` stubs; filter branchy+branchless PROVIDED and swept — branchy craters 9× to 1.19 GB/s at 50% selectivity while branchless holds ~12.7 flat, the SIGMOD '15 curve live — NEON count (vcltq+vsubq mask-accumulate) and LUT-compress compact (simdjson trick, f32 edition, all-16-masks test) are stubs; unpack4 scalar PROVIDED at 10.20 GB/s, NEON shift/mask stub; simd_bench catches stub panics so baselines always print. -- 2026-07-10 — topic 16 scaffolded: study guide (every technique = generator + oracle table, DST determinism boundary diagram, PQS/TLP/NoREC comparison, Jepsen/elle, Z3-as-search-engine), 6 reading guides (turso testing/simulator with clock/io/file fault-injection anchors + interaction-plan properties + doublecheck + structured fuzz targets; FDB simulation + BUGGIFY + Antithesis determinism-boundary table; SQLancer oracle base classes — newly cloned — PQS check()/rectification, TLP 3-way partition, NoREC optimized-vs-forced-scan; PQS OSDI '20 + TLP OOPSLA '20 paired; Jepsen redis-raft/Dgraph findings + elle cycle inference; Z3 — newly cloned — TACAS '08 + tactic/solver/smt_context anchors + Cosette symbolic-row rewrite verification), experiments crate compiles: sim_fs (buffered/synced/torn-tail file) + kv (WAL KV with 4 injectable bugs: LostDelete/NoSyncOnCommit/TornWriteAccepted/StaleRead) PROVIDED, dst harness + ddmin shrinker + TLP Kleene-eval checker are `todo!()` stubs with 12 contract tests (all bugs caught ≤200 seeds, zero false positives over 500, deterministic replay, 1-minimal repro, null-blind engine exposed), crash_matrix PROVIDED and run: 5000 seeds/0.02 s per bug — 0.0% false positives, bugs caught at 48.8–99.6% per-seed rates, first failing seed ≤ 3; the matrix caught a real bug in this crate's own recovery (missing WAL tail truncation made torn leftovers join the next batch — 72.7% divergence until fixed), the topic's thesis self-demonstrated. -- 2026-07-10 — topic 15 scaffolded: study guide (topology menu as WHO-can-ack axis, Raft state-machine mermaid + election/log-matching/§5.4.2 three-way split, write-path comparison valkey/WAIT/raft, consistency ladder + ReadIndex, hash slots vs ranges), 6 reading guides (Raft ATC '14 with Fig 8 worked by hand; valkey replication.c shared repl buffer + PSYNC replid/offset + REPL_STATE_ handshake + WAIT + FAILOVER with line anchors; tikv raft-rs — newly cloned — RawNode/Ready contract + step_* dispatch + Progress tracking + maybe_commit-is-§5.4.2; qdrant consensus.rs raft-for-metadata-only split with the data path outside raft; VSR Revisited round-robin views + no-disk durability + TigerBeetle disk-can-lie; DDIA ch. 5/8/9 anomaly catalog + fencing tokens + linearizability), experiments crate compiles: sim.rs deterministic lockstep network PROVIDED (seeded delivery, partition/heal — topic 16 DST preview), raft.rs `todo!()` stub with 5 safety-pinning tests (one leader, one-per-term across 10 seeds, replicate-to-all, minority-commit-freeze, stale-leader truncation), partition_test timeline binary, repl_lag PROVIDED and run: follower fsync policy AS ack latency — every-1 = 339 entries/s at 2973 µs p50 (F_FULLFSYNC) vs never = 18568/s at 6 µs, the topic 5 ladder measured as replication lag. -- 2026-07-10 — topic 14 scaffolded: study guide (recall-vs-QPS curve framing, HNSW-as-skip-list ASCII, quantization ladder u8/PQ/binary with oversample+rescore, IVF + DiskANN families, filtered-search menu with percolation), 6 reading guides (qdrant GraphLayers builder/serve split + visited pool + the per-query algorithm choice HNSW/ACORN/plain via estimate_cardinality + measured percolation at build; qdrant quantization crate u8 affine dot-expansion / PQ ADC LUTs / binary xor_popcnt + get_oversampled_top; usearch — newly cloned — node-tape layout + paper-default constants + striped locks (helix-db dropped: public repo no longer ships engine source); HNSW paper with skip-list lens; Jégou PQ with SDC/ADC + IVFADC residuals-as-FOR; DiskANN Vamana robust-prune α-slack + PQ-steers/f32-ranks SSD layout), experiments crate compiles: brute-force oracle PROVIDED and run (185 QPS at recall 1.0 over 100K×128-d — the floor), hnsw (Alg 1/2/4, level draw, ef knob) and quant (affine u8 + symmetric distance + rescore pipeline) are `todo!()` stubs with 10 contract tests (self-query top-1, recall@10 ≥ 0.9, sorted results, log level distribution, α/2 error bound, rescored recall ≥ 0.95), ann_bench sweeps ef 16..256 + oversampling 1/2/4. -- 2026-07-10 — topic 13 scaffolded: study guide (adjacency representation menu with CSR ASCII, four-architecture table neo4j/memgraph/kuzu/FalkorDB across store/Expand/pattern-match/MVCC/updates, delta-overlay-as-LSM observation, pointer-chasing cost analysis, WCOJ/AGM section, LDBC referee), 6 reading guides (GraphBLAS 4 sparsity formats + dot-vs-saxpy mxm + masks-as-pushdown + FalkorDB Delta_Matrix M/DP/DM state machine — neo4j/kuzu/GraphBLAS newly cloned; neo4j 15 B node / 34 B rel fixed records + doubly-linked rel chains = one miss per edge; memgraph skip-list vertex + small_vector edges + PointerPack'd delta MVCC; kuzu columnar CSR node groups persistent+transient + Intersect WCOJ + factorization; AGM bound / Generic Join / EmptyHeaded with the `C=A²` equivalence; LDBC SNB correlated-power-law datagen + updates-during-reads), experiments crate compiles: adj_list oracle PROVIDED and run (1M-node/16M-edge preferential-attachment: 3.5 µs/query random vs 295 µs supernodes — the 85× graph-shaped tail, max degree 6565), csr (counting-sort build + slice two_hop) and matrix (masked-SpMV two_hop) are `todo!()` stubs with oracle-agreement + exact-layout + cycle-self-exclusion tests, hop_bench cross-checks via checksums. -- 2026-07-10 — topic 12 scaffolded: study guide (row-vs-column ASCII, lightweight-encoding zoo table incl. FSST, analyze→score→compress lifecycle, zone-map pruning diagram, Arrow-vs-Parquet boundary, MergeTree/DuckDB/Pinot architecture table), 6 reading guides (DuckDB compression framework + 4-mode bitpacking + fetch_row-shapes-the-menu + CheckZonemap; ClickHouse MergeTree — newly cloned — parts/granules/sparse-index/marks two-offset trick + merge-time work; arrow-rs + parquet-rs — newly cloned — buffer recipes, RLE-hybrid, two compression layers; C-Store + SIGMOD '06 process-compressed thesis; BtrBlocks sampling cascade + FSST symbol tables; ClickHouse VLDB '24 with ClickBench-on-DuckDB exercise), experiments crate compiles: RLE/Dict/BitPacked `todo!()` stubs with exact-size + maximal-runs + FOR-width + O(1) random-access contract tests, scan_bench PROVIDED (100M values × 3 shapes, raw vs encoded scans incl. RLE sum-without-decode and dict codes-only sum — "raw-equiv GB/s > memory bandwidth" is the compression-IS-performance headline to verify). -- 2026-07-10 — topic 11 scaffolded: study guide (Volcano→X100→HyPer mermaid, selection vectors + vector-type flags, morsel-driven parallelism diagram, vectorized hash join/agg internals), 6 reading guides (DuckDB DataChunk/2048 + pipeline executor push-pull hybrid + join-HT salt-in-pointer probe; postgres ExecProcNode self-replacing dispatch + execExprInterp computed-goto; polars-stream Morsel/MorselSeq/SourceToken + float_sum masked-SIMD multi-accumulator + DataFusion ExecutionPlan streams and intern-then-flat-arrays GroupedHashAggregateStream; X100 CIDR'05 U-curve; VLDB'18 compiled-vs-vectorized scorecard — memory-bound probes favor vectorized, the M11 architecture argument; SIGMOD'14 morsels), experiments crate compiles: one query three engines — Volcano PROVIDED and run (180.7 M rows/s; found LLVM DEVIRTUALIZING the statically-known `Box` chain, 202→180 after black_box — a compiler will silently turn your Volcano into a compiled engine), vectorized (batches + selection vectors + flat group array) and fused branchless kernel are `todo!()` stubs with oracle-agreement tests incl. partial-final-batch and mask-sign-extension traps, exec_bench sweeps selectivity 5/50/95. -- 2026-07-10 — topic 10 scaffolded: study guide (parse→bind→logical→rewrite→join-order→physical pipeline mermaid, rewrite-rule menu, Selinger DP vs DuckDB DPccp+greedy-fallback, cardinality three-lies table, Selinger-vs-Cascades memo ASCII), 5 reading guides (DuckDB optimizer.cpp 25-pass pipeline + plan_enumerator DPccp with greedy escape hatch :234 + cost=output-cardinality-only; postgres allpaths.c standard_join_search + geqo threshold 12 + DEFAULT_EQ_SEL 0.005; sqlparser-rs Pratt parse_subexpr + DataFusion fixpoint-of-rules vs DuckDB ordered passes — sqlparser/datafusion/polars newly cloned; Selinger '79 vs Cascades with M10 architecture-choice question; Leis VLDB'15 JOB — cardinality error 10²–10⁴ dwarfs cost model 2× and search 1.2×, graph-JOB design exercise), experiments crate compiles: toy cost-based planner `todo!()` stubs (parse_and_plan naive left-deep → push_down → greedy reorder_joins → estimate with 1/NDV + independence + containment) with contract tests incl. join_order_flips_with_stats, explain binary PROVIDED for side-by-side DuckDB EXPLAIN comparison. -- 2026-07-10 — topic 9 scaffolded: study guide (latch vs lock table, memory-ordering cheat sheet + publication idiom, latch-coupling→OLC→lock-free ladder, epoch reclamation diagram, Bw-tree cautionary arc, false sharing), 4 reading guides (postgres lwlock.c packed u32 + recheck-after-enqueue lost-wakeup dance; crossbeam-epoch pin/defer/try_advance — newly cloned; RocksDB InlineSkipList CAS+splices vs memgraph lazy-locking skiplist with accessor-id GC — memgraph newly cloned; Bw-tree ICDE'13 + SIGMOD'18 reality check + Leis OLC), experiments crate compiles: lock-free ConcurrentSet `todo!()` stub over crossbeam-epoch with 5 contract tests (same-key/remove races exactly-one-winner, reader-survives-removal-churn UAF canary), scaling shootout PROVIDED (global mutex / 16-shard / crossbeam SkipSet / yours, 1→16 threads), false_sharing PROVIDED and run — packed 63 M inc/s vs pad128 3707 M (59×), and pad64 still 2.2× slower than pad128: Apple M-series coherence granularity is 128 B, x86-style 64 B padding only half-fixes it. -- 2026-07-10 — topic 8 scaffolded: study guide (anomaly-per-isolation-level table, doctors write-skew walkthrough, 2PL/OCC/MVCC comparison, postgres tuple-header + visibility flowchart, HOT chain, Hekaton contrast), 6 reading guides (postgres heapam.c/heapam_visibility.c HeapTupleSatisfiesMVCC + HOT + prune/vacuum with line anchors; RocksDB optimistic vs pessimistic txns over one base class — memtable-only OCC validation, point lock manager; surrealdb kvs layer — newly cloned — versioned reads + putc as portable OCC; Berenson '95 history notation + SI dethroned; SSI VLDB'12 dangerous structure + the single-writer M8 shortcut question; Hekaton + Wu/Pavlo 5-axis menu), experiments crate compiles: Mvcc `todo!()` stub with 8 contract tests including write_skew_HAPPENS_under_SI (test passes when the anomaly occurs) and Serializable-mode prevention via read-set validation, txn_bench PROVIDED (global Mutex baseline vs MVCC, 3 mixes incl. 64-key hot set, abort counts). -- 2026-07-10 — topic 7 scaffolded: study guide (RESP wire anatomy, event-loop mermaid beforeSleep→poll→read→execute→buffer, three threading models table, backpressure: querybuf/output-buffer kills vs pgwire portals), 4 reading guides (redis ae.c + networking.c parse/reply path with line anchors; valkey 8 io_threads.c SPSC inboxes + tagged job pointers + memory_prefetch.c batch-MLP; pgwire Parse/Bind/Execute/Sync portals + qdrant dual tonic servers — both newly cloned; C10K → thread-per-core arc with the shared↔sharded plane exercise), experiments crate compiles: RESP2 parse/encode `todo!()` stub with 8 format-fixing tests (incomplete-input-keeps-bytes, binary-safe bulks, pipelining), tokio server PROVIDED (16-shard store, parse-all-then-flush-once pending-writes trick) — benches vs real redis via redis-benchmark -P 1/-P 64 + flamegraph once resp.rs is implemented. -- 2026-07-10 — topic 6 scaffolded: study guide (translation-cost table hash/swizzle/MMU, miss-path mermaid, three shapes of approximate-LRU, swip state diagram, mmap CIDR-'22 checklist), 6 reading guides (postgres bufmgr.c packed-atomic state + CLOCK + buffer rings; DuckDB eviction queue with dead nodes + 4096-insert purge — newly cloned; LeanStore swips/cooling/hybrid latches — newly cloned; redis zmalloc per-thread padded counters + turso CLOCK page cache bonus; mmap paper with LMDB rebuttal; LeanStore+vmcache paper arc), experiments crate compiles: CLOCK BufferPool `todo!()` stub with contract tests (pinned-never-evicted, dirty-writeback, scan-pressure survival), pool_vs_mmap binary (1GiB file, 4× memory budget, Zipf, tail-latency focus), eviction bench PROVIDED and run — CLOCK 67.0% vs strict-LRU 66.3% hit rate at 20× less time per access (32ms vs 678ms per 1M trace): the "nobody ships strict LRU" lesson, measured. -- 2026-07-10 — topic 5 scaffolded: study guide (WAL rule, four-designs axis LMDB→turso→postgres→redis-AOF, fsync ladder table, group-commit mermaid), 5 reading guides (postgres xlog.c — newly cloned — reserve-then-copy/XLogFlush-recheck/FPI with line anchors; turso WAL checksum chain + salts; redis aof.c/rdb.c with the AOF-as-LSM mapping + FalkorDB angle; ARIES three passes/CLRs; Aether four-bottleneck taxonomy), experiments crate compiles: fsync_ladder PROVIDED and run (this Mac: fsync 21µs vs F_FULLFSYNC 3.0ms — 140×, the macOS weak-fsync gap is real), Wal `todo!()` stub with format-fixing tests (torn tail, uncommitted-txn invisibility, commit_many = 1 fsync), crash_test kill-9 harness (100 rounds, acked-key + atomicity checks), commit_throughput bench (per-commit vs group 8/64/512). -- 2026-07-10 — topic 4 scaffolded: study guide (memtable→SST lifecycle mermaid, SST block anatomy, leveled/tiered/lazy RUM table, stall triggers, Monkey intuition), 6 reading guides (lsm-tree crate — newly cloned, fjall delegates to it — + RocksDB compaction/table with line anchors; Monkey, Dostoevsky, RocksDB TODS '21, compaction design-space VLDB '21), experiments crate compiles: mini-LSM with provided Bloom (tests pass) + Memtable, SST writer/reader + Lsm engine `todo!()` stubs with correctness tests (tombstone-across-compaction, WA>1 check), write_amp binary measuring the full RUM position of leveled vs tiered. -- 2026-07-10 — topic 3 scaffolded: study guide (slotted page anatomy, 3-sibling balance mermaid, LMDB double-meta COW commit diagram), 5 reading guides (turso btree deep + SQLite btree.c + LMDB mdb.c with line anchors from fresh clones; Graefe survey selective-read map, SQLite file-format hex-dump exercise), experiments crate compiles: slotted Page + DiskBTree `todo!()` stubs with format-fixing tests, bench vs redb (point/scan) + prefix-truncation stress case (32B keys, 24B shared prefix). -- 2026-07-10 — topic 2 scaffolded: study guide (chaining vs open addressing cache stories, incremental-rehash mermaid, skiplist/rax ASCII, dense-filter/fat-payload pattern table), 7 reading guides (redis dict/zset/rax, hashbrown SwissTable, RocksDB InlineSkipList — line numbers from local clones; ART paper, CppCon SwissTable talk), experiments crate compiles: skiplist + incremental_map `todo!()` stubs with tests (the build work is the learning work), benches vs hashbrown/BTreeMap/crossbeam-skiplist, rehash_spike binary (HdrHistogram per-insert max/p99.9). -- 2026-07-10 — topic 1 started: study guide (two-family write/read paths, amplification vocabulary, RUM triangle), 8 reading guides (fjall/turso/tidesdb/rocksdb code + O'Neil/Comer/RUM/Hellerstein papers, line numbers from fresh shallow clones), engine_shootout scaffold (fjall vs redb behind a common trait, db_bench workload names, durability parity) compiles + smoke-tested (space-amp binary at 20K keys shows fixed-overhead floor, not amplification — re-run at 1M+). Topic 0 plan audit: fixed phantom CMU-lecture reference in PLAN.md, added missing roofline-thinking section to topic 0 README §4. -- 2026-07-10 — topic 0 finished: cache_ladder (after fixing a self-caching bug: restarting the pointer chase at 0 measured an 8MB hot path — fixed by carrying the walker across iterations; true ladder 1.0 ns L1 / 5–9 ns L2 / ~110 ns DRAM+TLB), lookup_shootout (HashMap flat at ~7–9 ns thanks to MLP; binary search wins ≤1e4; linear scan never beats hashing at n≥100 — folklore busted), flamegraph captured (21% SipHash in HashMap lookups), reference baselines recorded in capstone/BASELINES.md. Topic 0 + M0 done. -- 2026-07-10 — topic 0 started: study guide + 3 experiment benches (cache_ladder, lookup_shootout, branch_misprediction); capstone workspace scaffolded with `workload` crate (seeded Zipfian generator, ~11M ops/s). First measured result: branchy filter 8.1x slower on shuffled vs sorted data; branchless flat at 15 Gelem/s. Repo published to github.com/AviAvni/database-learning-path. -- 2026-07-10 — repo initialized: plan, capstone design, resources. +## 2026-07-28 — repo-wide audit: verify.sh coverage, CI, and the conventions backfilled + +**Repo-wide review and fix pass — no new topics, but the measurement spine rebuilt.** Prompted by a full audit of all 44 packages, 230 reading guides, 45 crates and the book build. What the audit found and what was done: + +**A benchmark reporting a physically impossible number.** Topic 12's `scan_bench` printed `raw sum 800.0 MB 0.000 s **19047619.0 GB/s**` — roughly 20,000x this machine's memory bandwidth. Cause: `time()` took best-of-3 reps of a *pure* fold, so LLVM hoisted the whole computation out of the repetition loop and reps 2-3 timed nothing. Fixed with `black_box` on the input inside each timed closure, plus a `MIN_CREDIBLE_SECS` guard that prints `n/a — below timer resolution` instead of a figure. Now measures **24.4 / 50.0 / 57.0 GB/s** across the three column shapes (150 GB/s peak on an M3 Pro, so one core gets ~a third of the bus). The spread across shapes is run-to-run noise on identical work — recorded as such, with the observed 24-76 GB/s range, rather than presented as a property of the data. This is topic 0's first failure mode found in the repo's own code, and it is written up in the topic and in README rather than quietly deleted. + +**`verify.sh` covered 10 of 44 topics while CLAUDE.md claimed it ran every measured lane.** Root cause was mechanical, not missing work: topics 34-43 caught the exercise-lane panic and printed `[stub — ...]`, while topics 2-31 let a `todo!()` abort the process (rc=101) after lane 1 had already printed. Ported a quiet `stub_lane` helper (suppresses the panic hook, catches, prints one marker line) through every bench binary; topics 17-18 already had it, 19-22 got a `STUBBED` flag plus one summary line, 23-30 only needed hook suppression. **verify.sh now runs 42 bin lanes across 41 topics and exits 0** (plus 3 criterion lanes for topic 0 behind `--criterion`), with `--list` and `--criterion` added. Topic 8's `txn_bench` was the one genuine FAIL the new coverage exposed (`run_mvcc` aborting the whole table); it now prints the global-lock baseline with `—` in the MVCC column. + +**Two topics had no measurable lane at all, so two were written.** Topic 7's only binary was the RESP server, which cannot run until the reader implements `resp.rs`; added `loopback_bench`, which measures the topic's actual thesis with no protocol parsing and no store — **44,088 ops/s at P=1 rising to 12,321,414 at P=256, a 279.5x swing on identical zero-work requests**, slightly super-linear against the 2/P syscall floor because larger writes amortize per-byte costs too, and per-request latency *improving* 22.68 → 0.08 µs (client-side batching is not the usual throughput-for-latency trade). Topic 3's only bench interleaved the reader's `DiskBTree` with redb and so died on the first stub; added `btree_baseline`, which prices the fanout arithmetic (8 B keys: **185 leaf cells, fanout 255, height 3 at 1e6**; 32 B keys: 88 / 102 / 4 — a **2.5x** interior-slot cost, which is what suffix truncation buys back) against redb measured warm. **The height ladder is a negative result and the best thing in the topic**: lookups climb **367 → 423 → 862 → 1101 ns** from 1e4 to 4e6 keys while height stays pinned at 3 from 1e6 onward. Height sets how many pages a lookup touches; cache residency sets what a touch costs, and at 270 MB the pages are not resident. Long keys: **733 → 882 ns and 67.9 → 135.3 MB** (1.20x slower, 1.99x bigger) — redb absorbs most of the predicted fanout loss, which is itself the finding. + +**CI never ran any Rust.** The only workflow built the book, so the repo's central claim had zero automated protection. Added [verify.yml](.github/workflows/verify.yml): `./verify.sh --summary` plus a `-D warnings` build of all 45 crates, both cached against `../.dlp-target`. Deliberately *not* `cargo test` (stub tests are the specification and must fail on a fresh clone) and deliberately *not* clippy (style lints across 44 independent teaching crates are a different argument). Fixed the 8 warning-emitting crates to make the gate pass: `#[allow(dead_code, reason = "...")]` on stub scaffolding, `#[allow(unused_variables, reason = "used once the todo!() is implemented")]` on stub bodies, and three genuine fixes (an unused import, two needless `mut`, one unused closure arg). + +**book/ was 7.2 GB, of which 6.7 GB was copied Rust build artifacts.** `src = "."` in book.toml makes mdbook copy every non-markdown file under the root, and it honours neither `.gitignore` nor `.mdbookignore` (both tested — hidden directories are copied too), so all 44 `target/` dirs were duplicated into the rendered book on every build. Fixed with a root [.cargo/config.toml](.cargo/config.toml) setting `target-dir = "../.dlp-target"`, outside the clone — verified that cargo resolves a relative `target-dir` against the config file's parent, and that nested crates inherit it. **16 GB reclaimed; book/ is now 49 MB**, and the 44 crates share one dependency build instead of 44. Also un-ignored and committed **45 `Cargo.lock` files**: for a repo whose pitch is that a seeded figure reproduces exactly, floating transitive versions were the one thing that could silently change a generator's output. + +**The documented conventions held for topics 26-43 and not for the earlier ones.** Backfilled, after correcting two things the audit had initially over-counted: +- **`notes.md` measured baselines (12 added, plus topic 9 refreshed).** The audit first flagged "empty measurement columns in topics 1-25", which was wrong — those cells are the reader's prediction worksheet and are *supposed* to be empty. The real gap was narrower: topics 1-8 and 10-15 had no record of the provided lane's output at all. Added `## Baseline (provided lane, Apple M3 Pro, measured 2026-07-28)` sections with the real numbers and the analysis, and for topics 4 and 10 (whose benches measure only the reader's code) a `## No provided baseline in this topic — and why` section giving the arithmetic or the external oracle to predict against instead. Topic 9's recorded 636 ms / 59x false-sharing figures no longer reproduce — re-measured at **202.7 ms and 17.8x** — so both are now recorded with an explicit note that contended-line throughput wants a range, not a point. +- **Numbers-first README openers (20 added).** Again narrower than first counted: 12 of topics 0-31 already opened with measured output under other names ("Our motivation numbers first", "The problem, priced"). The 19 that did not, plus topic 12, now do. +- **`## Done when` in all 230 reading guides (114 added).** The gap was a clean band: topics 13-30 had zero, topic 0 had zero, topic 1 had 4 of 8. Each new checklist is specific to its guide's `### Step` sections and questions, and where a topic has a lane the checklist ties back to the measured number. +- **The pinned-commit convention was aspirational: exactly one guide had a real SHA**, against a few thousand `file:line` anchors. Rather than scatter thousands of SHAs that would drift independently, added [tools/pin-table.py](tools/pin-table.py) and one authoritative table at the end of `resources/codebases.md` recording **84 reference clones with verified HEAD, date and origin**, ranked by mention count. Only clones with a resolvable HEAD and origin are emitted — unverifiable ones are reported and left out rather than guessed at. The generated block is stripped from its own corpus so `--check` is idempotent. + +**New reader-facing pages.** [FINDINGS.md](FINDINGS.md) — the repo's differentiator was four rows in README; it is now **42 rows**, one measured headline per topic with the command that re-derives it, plus a section on how to read it (which rows contradict their own topic's tidy story, which baselines refuse to be weak, and the observation that topics 34-43 are all measurement failures rather than system failures) and a mermaid dependency DAG with the three threads worth following deliberately (the fsync wall at **337 / 341 / 14 ms** in topics 5, 15 and 28; skew in 13, 24, 36, 20; and the measurement itself lying in 0, 12, 34, 39-43). + +**Presentation fixes.** PROGRESS.md showed 43 of 44 topics as `todo`, accurate for the *study* but read as "unbuilt" to anyone arriving from the book — split into **Package** (all 44 done) and **Studied** columns with the distinction stated up front. SESSION-LOG.md was 132 KB under a single heading: added **50 `## date — topic NN — title` headings** for anchors and sidebar navigation, verified byte-identical entry text afterwards. capstone/README.md claimed milestones map to topics 0-31 (PROGRESS has M0-M43) and did not say that **M0 is the only one built** — both fixed. `drafts/` was untracked and unmentioned; committed with a README stating it is not part of the book and that the attribution rule still applies. CLAUDE.md and CONTRIBUTING.md updated so every rule they state is now actually true of the repo, including the new stub-degradation rule and the mdbook `src = "."` wrinkle. + +Verified at the end: **42/42 lanes PASS, 0 FAIL**, all 45 crates build under `-D warnings`, `mdbook build` clean with 66 mermaid blocks intact and 50 SESSION-LOG anchors, zero broken relative links across 292 markdown files, and SUMMARY.md titles matching every on-disk H1. + +## 2026-07-27 — topic 43 — Network & IT-Ops Dependency Graphs + +**topic 43 Network & IT-Ops Dependency Graphs added** (sixth and last of the graph use-case deep dives; the 38-43 expansion is complete): study guide (**the alert storm and the gray failure measured** — bench lane 1: synthetic microservice topology, 4 frontends / three tiers of 10-16-20 / 5 shared infra leaves, 152 configured edges of which **113 are reachable** (an unreachable-configuration finding in itself), 40,000 requests, with a planted **gray failure** on the most-depended-on infra leaf — `infra-0` is SLOW on 55% of calls rather than failing, and its callers time out, so the errors are manufactured one hop ABOVE the cause; result: **34 of 55 services alert above a 5% error rate and the broken service is not one of them**, its own error rate is **0.0040 = exactly the baseline**, it ranks **35 of 55 by failure count and 41 of 55 by error rate** (error-rate ranking puts the three front ends at the top, i.e. it points at the services furthest from the fault), and **all five infra leaves sit at 0.0040-0.0041, statistically indistinguishable** — no sorting of any per-node column can separate them, which is the entire argument for the topic; lane 2 reference — localization across five topologies: per-node baselines average **rank 36.4 (failure count) and 44.0 (error rate) with 0/5 top-3**, while a **correlation-weighted random walk and Sherlock's Ferret at k=1 both average rank 1.0 with 5/5 top-1**, at 22.8 ms and 21.4 ms respectively; the instructive ablation is that a **backward-only walk ranks the cause 3rd instead of 1st** because it drains into the leaves with no way to climb out, so the forward and self edges are what let the correlation weights bite; and the detail that makes the Ferret implementation work is **clamping the fitted severity to [0,1]** — a severity is a probability, so a candidate simply not on enough requests would need one above 1 to explain the observed rates; lane 3 reference — Dapper sampling as **two different questions with two different answers**: edge recall stays at **1.000 all the way down to 39 traces (1/1024)** while rare-path recall collapses **1.000 → 0.249 → 0.062 → 0.016 → 0.004 → 0.001**, and the mean latency stays within **5.8%** while the **p99 error reaches 25.6%** — one sample, three verdicts depending on whether the question is aggregate, rare-event or tail, with the honest caveat stated in the output that edge recall saturates this early only because this topology has little path diversity), 4 reading guides (Dapper 2010 read in full — the ubiquity/continuous-monitoring requirements that force negligible overhead, trace trees with clock skew handled by causality rather than NTP, the three instrumentation points (thread-local context, the common control-flow library, the single RPC framework) that make transparency possible in <1000 lines of C++ and <800 of Java, **out-of-band collection for two reasons** (in-band trace data would dwarf sub-10 KB RPC responses and bias analyses; in-band assumes perfectly-nested RPCs, which middleware violates) at a cost of a bimodal p98 collection latency, the overhead budget (**204 ns root span / 176 ns non-root / 9 ns unsampled annotation / 40 ns sampled / <0.3% of a core / 426 bytes per span / <0.01% of network traffic**) and the 9-vs-40 ns split that made 70%-of-spans annotation coverage possible, **Table 2's sampling cost (+16.3% latency at 1/1, +2.12% at 1/16, −0.20% at 1/1024 inside experimental error)**, the "if a notable execution pattern surfaces once it will surface thousands of times" argument together with its own caveat about low-volume services, adaptive sampling by rate-per-unit-time with the probability recorded alongside the trace, and the critical detail that collection-time sampling hashes the **trace id** so whole traces are kept or dropped — a shredded trace has no causal structure left; Sherlock SIGCOMM'07 read in full — the **(P_up, P_troubled, P_down)** three-state model with *troubled* defined as "servers or links continue to function but users perceive poor performance" (differential observability, a decade early), the three node types (root-cause / observation / meta) and **three meta-nodes** with their truth tables — noisy-max (with probability `1−d` the child escapes its parent's state), selector (a noisy-max node would give a client a **25% chance of being up with both load-balanced servers down**), failover — the **always-troubled / always-down pseudo-causes at 0.001** ("1 in 1000 failures are caused by a component not in our model") and router-path edges at 0.9999 as explicitly-priced model error, the **O(3ⁿ) → O(n)** propagation reduction for noisy-max nodes, and **Ferret**: 3^r assignment vectors cut to at most **(2r)^k** by Observation 3.1 ("it is very likely that at any point in time only a few root-cause nodes are troubled or down", error "vanishingly small for k = 4 onwards") plus **two orders of magnitude** from Observation 3.2, scored by fitting two Gaussians (200 ms vs 2 s) to historical response times with a null-hypothesis significance test, over a dependency graph *discovered* from packet co-occurrence within a **10 ms dependency interval** discounted at (10ms)/I; Pivot Tracing SOSP'15 read in full as **the database paper hiding in an operations topic** — the two failures of ordinary monitoring ("one size does not fit all", with the Apache issue-tracker evidence and HBase's "all users pay the 10% overhead"; and crossing boundaries), the query language and the **happened-before join `Q1 ⋈ Q2` over Lamport's →**, the five advice primitives OBSERVE/UNPACK/FILTER/PACK/EMIT woven at runtime with no jumps or recursion and guaranteed termination, **baggage** as a per-request tuple container propagated across thread/process/machine boundaries so joins evaluate **in situ** rather than centrally (Magpie's strategy is Figure 6a), and **Table 3's rewrite rules pushing projection, selection and aggregation down to the source tracepoints for a 600 → 6 tuples/s reduction** — predicate pushdown and join placement, in a tracing system; Huang et al. HotOS'17 *Gray Failure* — the observer/app/ground-truth model and the four-cell table whose fourth cell is **differential observability**, why every redundancy mechanism is inert under it (they are all keyed on the observer's view), three structural reasons detection is hard, and the escalation argument that makes a gray failure the trigger of a topic-35 metastable failure), experiments crate `opsgraph-experiments` (`services.rs` PROVIDED — the topology generator, the gray-failure workload with slow-dependency-plus-caller-timeout propagation, traces carrying paths/edges/latency, both per-node baselines, the symptom correlation, and `participation` = P(service on path | entry frontend), which is the deliberately weak observable Ferret must work from; `rca.rs` stub — `random_walk_rca` with three edge types and `sherlock_single_fault`; `sampling.rs` stub — whole-trace `sample`, `edge_recall`, `rare_path_recall`; **4 provided tests pass, 9 fix the contract for the stubs**, including that a gray failure must not trip its own alert, that all infra error rates stay within 0.01 of each other, that the walk beats both baselines and a backward-only walk is strictly worse, that the ranking is stable across five seeds, and that sampling keeps whole traces), and capstone M43 (trace ingest as an incrementally-maintained dependency graph with sketched edge weights, both localization procedures over the topic-18 CSR, and a happened-before join operator in the query engine with Pivot Tracing's pushdown rewrites — deliverable numbers include **top-1 accuracy under sampling**, the question the whole topic converges on and which none of the four papers answers). Cross-topic threads worked: 38/42 (personalized PageRank a third time, same justification), 34 (this is topic 34 at cluster scale), 37 (fan-out arithmetic explains the storm; hedging works where failure detection does not), 35 (gray failure as trigger, retry storm as sustaining loop), 10 (Pivot Tracing's Table 3 is an optimizer), 26 (edge weights must be sketches — the p99 row is why), 40 (the same graph question with the arrows reversed), 27 (trace ingest is a stream, the dependency graph a materialized view over it), 21 (Sherlock's model is tuned, not verified). + +## 2026-07-27 — topic 42 — Recommendations & Social Graphs + +**topic 42 Recommendations & Social Graphs added** (fifth of the six graph use-case deep dives): study guide (**the popularity trap measured** — bench lane 1: synthetic bipartite interaction graph, 3000 users x 6000 items, 30 communities, Zipf(1.1) popularity tail, 60,000 training edges and 6,000 held-out engagements; the bestseller list gets **hit-rate@50 = 0.340** with a personalization score of only **0.155** (and that only because each user's own items are filtered out — everybody is handed the same list), while Pixie's unmodified Algorithm 1 reaches 0.403 but **45% of every returned list is the bestseller list again**, because an unbiased walk's stationary distribution goes as degree — Pixie's own complaint from §3.1, "low degree nodes with fewer edges contribute less signal ... smaller boards are more likely to produce highly relevant recommendations"; lane 2 reference — the Pixie ablation over 300 users x 8 query pins x 30,000 steps: going from one query pin to eight with sub-linear step allocation takes hit rate **0.403 → 0.823** (the biggest single win, and the least clever idea), **early stopping runs in 35% of the steps at 2.2× the speed keeping 0.793 top-50 overlap with hit rate unchanged** — almost exactly the paper's "84% overlap at a third of the runtime" — and **the multi-hit booster shows NO gain at all**, 0.823 unboosted vs 0.803 boosted at one interest per user and 0.563 vs 0.547 at three, which is the more instructive result: the arithmetic is right (the unit test pins (√2+√2)²=8 against a single-source 4) but the generator does not contain Equation 3's premise, since it draws its held-out item from the same distribution as the training items, so a published trick's *domain assumption* has to be measured on your own data before you ship it (exercise 4 builds a graph where the premise holds); lane 3 reference — link prediction on a collaboration graph grown with preferential attachment + triadic closure where a random guess is right **0.314%** of the time (right inside Liben-Nowell's 0.147–0.475% band): preferential attachment **1.9×**, common neighbours **20.7×**, Jaccard **25.6×**, Adamic/Adar **22.3×** — the degree-only measure barely beats chance, exactly as in the paper), 4 reading guides (Pixie WWW'18 read in full — the 30–50%-engagement argument for real-time over batch, Algorithm 1 in twenty lines, the four innovations (user-feature biasing with `PersonalizedNeighbor` as a *subrange* operator, weighted query sets with Equation 1's sub-linear allocation `s_q = |E(q)|·(C − log|E(q)|)` where C must be the graph-wide maximum or the top query pin gets zero steps, Equation 3's multi-hit boost, and per-walk early stopping on n_p pins reaching n_v visits), the language-biasing table (En→Slovak target-language content **2.13% → 42.55%**, En→Japanese 16.35% → 80.33%), hit rate 6.3/23.1/52.2% at top-10/100/1000 against content-based 2.1/4.6/10.5%, A/B lifts of +48% on homefeed, the **pruning result that F1 peaks 58% above the unpruned graph at 20% of the edges**, and the implementation section's `edgeVec` object pool + open-addressed visit counter sized to N + **HugePages cutting page-table entries 512×**; GraphJet VLDB'16 read in full — four generations (Cassovary → Hadoop RealGraph → MagicRecs → GraphJet) and what killed each, the single-server bet with its "ten billion edges is a mere 80 GB" arithmetic and the challenge to distributed graph research, MagicRecs' reformulation of temporal edge detection as an **intersection of adjacency lists**, the five-method API and the two deliberate omissions (no deletes because interactions are point events, no timestamps as a space/quality trade), **temporally-partitioned index segments** with only the newest writable and whole-segment discard as coarse pruning, id mapping by double hashing where the hash IS the internal id (hence the power-of-two table chain) with edge type bit-packed to leave 2²⁹ ids, **edge pools whose slice sizes double** (`P_r` holds `n/2^{r−1}` slices of `2^r` edges; degree 25 → `P1(1),P2(2),P3(0),P4(0)`) justified as an allocator that assumes preferential attachment, single-writer/multi-reader with memory barriers instead of locks, background relayout of sealed segments for contiguous iteration, the **alias method** for O(1) degree-weighted cross-segment sampling, full vs subgraph SALSA (the subgraph fits in cache and needs only a left-to-right index, ~half the memory, at the cost of second-order paths), the deployment numbers (**1M edge insertions/s**, 500 rec/s per server at **p50 19 / p90 27 / p99 33 ms**, O(10⁹) edges in **<30 GB**, >99.99% over 30 days), and **§7.3's rejection of Redis `LPUSH` as an adjacency-list store for two named reasons — no memory-allocation optimization and no pruning mechanism — which is a two-item feature list for a Redis-module graph engine**; TAO ATC'13 read in full — the three failures of lookaside caching (inefficient edge lists, distributed control logic, expensive read-after-write), the two data shapes and four association queries, **creation-time locality** ("most of the data is old, but many of the queries are for the newest subset") forcing newest-first association lists and prefix caching, which in turn forces **refill rather than invalidate** (invalidating truncates a cached prefix and discards edges), sharding associations by `id1` so every query is one server — which is *why* there is no multi-hop traversal — the leader/follower hierarchy sized by **read misses being 25× as frequent as writes**, hot-spot handling by shard cloning and access-rate-triggered client-side caching, slab allocation with per-type arenas and **association counts packed into 14 bytes** for 20% more cache entries, and the production envelope (**96.4% read hit rate**, `assoc_get` 1.0 ms p50 hit vs 143 ms p99 miss, writes 12.1 ms in-region vs 74.4 ms from 58 ms away, **4.9 × 10⁻⁶ failed queries over 90 days**) plus the two workload tails that must be in any honest benchmark — **1% of `assoc_count` results ≥512K** and **64% of non-empty ranges returning exactly one edge**; Liben-Nowell & Kleinberg read in full — the training/test interval setup with κ=3 Core filtering, why **factor-improvement-over-random** is the only interpretable metric when raw accuracy is 0.147–0.475%, the measure catalogue (common neighbours, Jaccard, **Adamic/Adar's `1/log|Γ(z)|` hub discount**, preferential attachment, Katz, hitting/commute time normalized by the stationary distribution *because otherwise popular nodes dominate — the popularity trap from a third direction*, rooted PageRank, SimRank), Figure 3's table in which **preferential attachment scores 4.7–15.2× against common neighbours' 18.0–47.2× and Adamic/Adar's 16.8–54.8×**, "there is no single clear winner among the techniques", and the three meta-approaches — low-rank approximation, unseen bigrams, and a **clustering step that deletes low-confidence edges and recomputes, which is Pixie's graph pruning arrived at fifteen years earlier**), experiments crate `social-experiments` (`graphs.rs` PROVIDED — the bipartite interaction graph with communities, a Zipf tail, configurable interests per user and held-out engagements, plus the Liben-Nowell collaboration graph with preferential attachment + triadic closure and a train/test split, the baselines `popularity_topk` and `basic_random_walk` (= Pixie Algorithm 1), and the metrics `hit_rate` / `personalization` / `popularity_overlap` / `evaluate` with factor-over-random; `pixie.rs` stub — `allocate_steps`, `walk_per_query`, `multi_hit_boost`, `pixie_walk` with early stopping; `linkpred.rs` stub — the four measures; **2 provided tests pass, 8 fix the contract for the stubs**, including the boost arithmetic being exact and leaving single-source scores unchanged, every query pin getting ≥1 step with a step ratio strictly below the degree ratio, early stopping keeping ≥70% top-100 overlap in strictly fewer steps, two users from different communities sharing <50% of their top-50, Adamic/Adar's discount being arithmetically exact on a hand-built hub-vs-specialist case, and preferential attachment losing to both common neighbours and Adamic/Adar), and capstone M42 (a temporally-bounded bipartite interaction store with GraphJet's index segments and doubling edge pools over M31's storage, a Pixie-shaped walk procedure with sub-linear allocation and early stopping, and a TAO-shaped `assoc_range`/`assoc_time_range`/`assoc_count` API with time-ordered lists and cached counts — plus the benchmark GraphJet §7.3 explicitly invites, a Redis adjacency-list baseline measured on both counts it names). Cross-topic threads worked: 38 (one random-walk primitive, three seedings), 23/39 (Adamic-Adar = IDF = FRAUDAR column weights), 9 (single-writer deletes the whole latch hierarchy), 12 (sealed-segment relayout as LSM compaction; TAO's 14-byte count as a columnar instinct), 26 (alias method, bit-packing), 6 (TAO's cache is buffer management), 36 (shard-by-id1 and cloning vs migration), 40 (TAO vs Zanzibar on hot spots), 25 (low-rank approximation is where embeddings come from). + +## 2026-07-27 — topic 41 — On-Chain & Crypto Analytics + +**topic 41 On-Chain & Crypto Analytics added** (fourth of the six graph use-case deep dives): study guide (**haircut taint diffusion measured** — bench lane 1: synthetic UTXO chain with planted ground truth, 400 entities / 20,400 transactions / 40,400 outputs / **30,342 addresses (76 addresses per entity — the pseudonymity illusion)**, one stolen coinbase worth 0.25% of all the money; haircut tainting ends up flagging **3657 of 3734 UTXOs (97.9%) and 3553 of 3627 addresses (98.0%)**, of which 658 are <0.1% tainted, 2997 are 0.1–5%, and **exactly two are above 5%** — the total is conserved to the satoshi, haircut does not invent money, it just stops being information; the real-chain version from Anderson et al.: the 2012 Linode theft of 46,653 BTC taints **16,855,619 addresses (93% of all of them) under haircut vs 245,120 (1.35%) under FIFO**, Flexcoin 2014 taints 10,421,112 (57%) vs 15,265; lane 2 reference — the three policies on the same theft: **poison flags 394.67× the stolen amount** (it re-counts each descendant output's full value, so the total explodes with fan-out), **haircut 1.00× spread over 97.9% of the UTXO set**, **FIFO 1.00× concentrated in 0.9% (32 UTXOs, one holding 22.5% of the flagged value)** — same conservation law, 114× narrower answer, at **3.1M transactions/s** because the whole algorithm is a queue splice; lane 3 reference — the clustering collapse curve: Heuristic 1 (co-spend) holds **precision exactly 1.000 at every change-reuse rate** because it keys on a property of the protocol, while Heuristic 2 (one-time change address) buys recall 0.041 → 0.397 and then goes **precision 1.000 → 0.661 → 0.502 → 0.089 → 0.009** as one change address in {∞, 100, 50, 20, 10} is reused, with the largest cluster growing **93 (1%) → 366 (3%) → 476 (4%) → 1894 (16%) → 7991 (71% of all addresses)** — union-find makes every false merge transitive and permanent, which is why a safe heuristic at recall 0.04 beats an effective one at precision 0.09), 4 reading guides (Meiklejohn et al. IMC'13 read in full — the 2013 parse (231,207 blocks / 16,086,073 txs / 12,056,684 keys), Heuristic 1's safety argument ("these entities would need to reveal their private keys to each other") taking 12M keys to **5,579,176 clusters**, Definition 4.3's four conditions with condition 4 as the heuristic's conscience (decline when two outputs are both fresh) and condition 3 explained by 23% of transactions using self-change, the **false-positive ladder 13% → 1% (excluding the Satoshi Dice payout pattern) → 0.28% (wait a day) → 0.17% / 7,382 addresses (wait a week)** = precision bought with latency, the **1.6M-key super-cluster** containing Mt. Gox + Instawallet + BitPay + Silk Road and its two named causes, and the leverage argument (2,197 clusters named covering 1.8M addresses = **1,600× manual tagging**) plus Satoshi Dice at ~60% of all activity; Anderson/Shumailov/Ahmed/Rietmann *Bitcoin Redux* WEIS'18 read in full with the RustyTaintChain @ 4e12fd0 code read — `nemo dat quod non habet` and why bitcoin being a commodity rather than money keeps theft victims' claims alive, poison/haircut/FIFO Figures 1–3, **Clayton's Case (1816)** as the precedent, the losslessness argument ("the transaction processes it in a lossless way... we can trace a bitcoin's heritage backwards as well as tracing taint forwards"), `TaintPart{name: u16, value: u64}:52` / `extract_taint:142` (the three branches — queue dry, run fits, run straddles the cut and must be split) / `combine_taints:174` (collisions between crime sources — why `name` is a u16 not a bool) / `reduce_taint:250` (run-length coalescing or the queue fragments forever), the mixer inversion ("one black coin and nine white coins into a laundry isn't ten white coins, but ten black ones — people designing money laundering mechanisms have been using quite the wrong metrics of quality"), and §5's self-undermining finding that most victims' coins never touched the chain at all because exchanges settle off-chain; BlockSci USENIX Sec'20 read in full — the design chain **append-only ⟹ static snapshots ⟹ ACID unnecessary ⟹ in-memory analytical database** and the "infinite COST" conjecture, Figure 2's transaction record (32-bit ids, 60-bit value + 4-bit address type in one word) with inputs/outputs stored **inline at a deliberate 19% space cost bought for sequential locality** (Table 4: 50.09 GB current vs 40.50 normalized vs 69.26 at 64-bit ids, on 489M txs / 1.198B inputs / 1.302B outputs), the snapshot illusion (disk table grows, each instance pins a block height, past state reconstructible because append-only), memory mapping giving zero-synchronisation parallelism because there is exactly one writer (load ~4 min, **full parallel pass 0.9 s on 16 vCPUs**, parse 5.5 h), the parser's bloom filter + multi-use address cache exploiting **88% of inputs spending outputs <4000 blocks old** and **8.6% of addresses used more than once accounting for 51% of occurrences**, union-find address linking in "a few minutes" yielding **474M clusters / 380M singletons / 809 over 20k / one supercluster >17M addresses**, the fluent DSL as a miniature query planner (7–11× over the helper method, 3–5× off hand C++), and **Table 3 benchmarked against Neo4j, Memgraph and RedisGraph — FalkorDB's own ancestor** (calculate fee: BlockSci 0.57 s vs Neo4j 303.69 vs RedisGraph did-not-finish vs Memgraph 187.02; but Neo4j-with-index *beats* single-threaded BlockSci on `Tx locktime > 0`, 0.05 vs 0.31 s) read row-by-row as a spec for what a graph engine must add to win scan-shaped queries back; Weber et al. KDD'19 read in full — the Elliptic data set (203,769 nodes / 234,355 edges / 166 features / 2% illicit / 21% licit / 49 time steps with **no edges between time steps**), the 94-local vs 72-aggregated feature split making the comparison "learned vs hand-built one-hop aggregation", and the uncomfortable result **Random Forest illicit-F1 0.788 (0.796 with GCN embeddings concatenated) beating GCN 0.628**, Skip-GCN 0.705, EvolveGCN 0.720 — plus the **dark market shutdown at time step 43 that breaks every method even when retrained after every step with fresh ground truth**, and why micro-F1 >0.92 for every method is meaningless at 2% base rate), experiments crate `chain-experiments` (`chain.rs` PROVIDED — synthetic UTXO chain *with ground truth the real blockchain does not come with*: `address_entity` per address, one stolen coinbase, planted co-spending / change addresses / recipient address reuse, no fees so taint conservation is exactly testable; `taint.rs` — `haircut` provided, `poison` / `extract_taint` / `fifo` stubbed; `clustering.rs` — `UnionFind` and the O(addresses) pair-precision/recall scorer provided, `multi_input_clusters` / `change_output` (Definition 4.3) / `full_clusters` stubbed; **3 provided tests pass, 10 fix the contract for the stubs**, including FIFO conserving the stolen amount exactly, haircut touching >5× as many UTXOs, poison flagging >10×, every policy staying inside the descendant set with FIFO ⊆ poison, `extract_taint` splitting a straddling run rather than rounding, co-spend precision being exactly 1.000, Definition 4.3 declining every two-fresh-output transaction, and the 5%-reuse collapse below precision 0.2), and capstone M41 (incremental FIFO taint queues in the property layer with run-length coalescing, a maintained union-find cluster index re-pointing M39's machinery, and a BlockSci-shaped columnar transaction store so the two layouts can be compared on Table 3's queries). Cross-topic threads worked: 39 (clustering IS entity resolution — same union-find, hand-written conditions vs learned weights), 40 (both score a graph the adversary reads; prefer protocol properties and lossless measures), 12 (BlockSci's inline layout as the columnar argument), 32 (Table 3 read row-by-row is an HTAP brief), 1 (the taint policies as a RUM triangle), 33 (Elliptic's missing cross-time edges delete time-respecting paths by construction), 25 (build the aggregates and measure before reaching for a GNN), 36 (the infinite-COST conjecture rests on graph data resisting partitioning). Repos cloned for the code reads: `~/repos/RustyTaintChain` @ 4e12fd0, `~/repos/BlockSci` @ 14ccc93. + +## 2026-07-27 — topic 40 — Security & Attack Graphs + +**topic 40 Security & Attack Graphs added** (third of the six graph use-case deep dives): study guide (**the list-vs-graph gap measured** — bench lane 1: synthetic AD-shaped directory, 2000 users / 400 groups / 1000 computers, five edge kinds (`MemberOf`, `AdminTo`, `HasSession`, `GenericAll`) with a planted over-privileged group, planted service-account groups and planted policy violations; the console answer is **5 direct tier-zero members, 8 with the one nested group expanded, and it never moves**, while attack-path reachability with 1% of users (20) in the over-privileged group goes **39 (1.9%) at zero sessions → 1969 (98.5%) at 100 sessions → 2000 (100%) at 500** — the cascade is that each newly exposed user's own sessions drag in everyone who is local admin on those machines, so **exposure is a function of collection time, not of how much privilege exists**; separately, with the gateway shut, **one Domain Admin token left on an ordinary workstation takes exposure from 8 users to 2000**; mean shortest attack path 6.03 hops, worst 8; lane 2 reference — **choke points are dominators**: in the reverse graph rooted at tier zero, node d dominates u iff every attack path from u crosses d, so d's dominator subtree IS its blast radius, pricing every single-node remediation in **0.8 ms vs 543 ms** for 3400 individual reachability re-runs (Cooper–Harvey–Kennedy iterative dominators, exact agreement with the delete-and-recompute oracle on every node in both regimes) — and the finding that matters more than the speedup, **tiering is what makes a graph have choke points**: identical 2000-user exposure, but the tiered directory has a group with a **1992-user (99.6%) blast radius** and greedy cuts 2000 → 8 → 5, while the flat one has **no single node whose removal frees a single user** and cutting the whole gateway set one node at a time reads 2000 → 2000 → 2000 → 2000 → 2000 → 8, so remediation is a set problem and the all-zeros dominator pass IS the report; lane 3 reference — Zanzibar Check by pointer chasing costs **19 → 559 tuple reads and 0.46 → 11.28 µs** as group nesting goes 2 → 32 while the Leopard-style flattened closure stays **4 → 12 probes at ~0.01 µs**, flat in depth, for a **1.7× entry tax** (6672 tuples → 11393 entries, closure quadratic in chain depth) and galloping intersection beats a linear merge by **>1000×** on a 1-vs-500,000 pair), 4 reading guides (BloodHound code read against ~/repos/bloodhound @ 1968388 — 104 `StringKind` node/edge kinds at `graphschema/ad/ad.go:28`, the four purposeful partitions `Relationships`/`ACLRelationships`/`PathfindingRelationships` (63 traversable kinds, the attacker's alphabet) / `PostProcessedRelationships` (**31 kinds that are derived, not collected** — `AdminTo`, `CanRDP`, `DCSync`, ADCS `ESC1..ESC13` — a materialized view refreshed by the four-stage pipeline at `analysis.go:346`), principal sets as **roaring bitmaps** (`cardinality.Duplex[uint64]`, `post.go:244`) and parallel BFS with `CheckedAdd` on a thread-safe bitmap as the visited set (`membership.go:81`), `tiering.go:37` `IsTierZero`, `agt.go` selector expansion diffed against previous state; Ammann/Wijesekera/Kaushik CCS'02 read in full — monotonicity ("the attacker never needs to backtrack"), the Sheyner numbers it replaced (**5 hosts, 8 exploits → 5,948 nodes / 68,364 edges / 2 hours / 229-bit state space** vs **at most 229 nodes** monotone), no negation in preconditions + `preConds ∩ postConds = ∅` ⟹ `markAttributes` is O(|A|²·|E|) converging in ≤|A| layers, `findMinimal`/`findAll`/`findShort` with Results 1–3, minimum attacks NP-complete but minimal easy, the 3-host example (60 attributes, 30 instantiated exploits, only 8 attributes ever change value), and **§2.3's three-sentence cut-set paragraph** which lane 2 makes precise; plus MulVAL CCS'06 read in full — logical attack graphs as tabled-Datalog derivation graphs (derivation nodes = AND, fact nodes = OR, primitive vs derived facts), XSB tabling for cycles and memoization, Theorems 1–3 (O(N²) derivation steps / O(N²) graph size / O(δN²) = O(N² log N) build), "useless edges" as a why-provenance test, **1000 fully-connected hosts on a Pentium 4** where Sheyner's tool blew up at 10 (Fig 14) and Sheyner's own 10-host/5-vuln case producing a **10-million-edge** graph in 15 min; Zanzibar ATC'19 read in full with SpiceDB @ 8422483 anchors — the relation-tuple grammar and why the user slot holds a userset, the three rewrite leaf kinds (`_this` / `computed_userset` / `tuple_to_userset`), Check as ∃-tuple ∨ ∃-userset-with-recursive-Check with concurrent leaf evaluation and subtree cancellation, **Leopard** (`GROUP2GROUP` ancestor→descendants, `MEMBER2GROUP` user→direct parents, membership = `O(min(|A|,|B|))` skip-list-seek intersection = topic 23's galloping intersect doing authorization; **1.56M QPS median, <150 µs median / <1 ms p99**, offline snapshot pipeline + Watch-fed incremental layer at ~500 updates/s, one tuple change → tens of thousands of index events), **zookies and the new enemy problem** (Example A neglecting ACL update order, Example B applying an old ACL to new content; the `≥` semantics is what lets Safe requests outnumber Recent by two orders of magnitude), hot spots §3.2.5 (consistent-hashed distributed cache forming "cache trees", **timestamp quantization to 1 or 10 s** so cache keys collide, lock table against stampedes, hot-object prefetch) and the surprise that a **10% check cache hit rate prevents 500K internal RPC/s** of hot-spotting, scale (>2 trillion tuples / ~100 TB / >10M QPS / Check Safe p50-p95-p99 = **3.0 / 9.46 / 15.0 ms** / >99.999% for 3 years) and the SpiceDB map of which parts are inherent vs Google-shaped (`graph/check.go:99→165→304→539→567`, `membershipset.go` set algebra **with caveats** so a result can be "maybe", `lookupsubjects.go:430` reverse arrow traversal, `dispatch/keys/computed.go:58` a `uint64` over a *canonicalized* expression, `singleflight.go:47` = Zanzibar's lock table verbatim, `defaultConcurrencyLimit = 50`); SLEUTH USENIX Sec'17 read in full — provenance graphs, the dependency-explosion problem, a main-memory dependence graph at **<10 bytes/event** vs ~250 B/edge for Neo4j-class stores and ~3 KB for STINGER/NetworkX (32-bit ids, events stored inside subjects, variable-length encoding down to 4-byte subject-event and 16-bit object-event records, delta timestamps, **6-byte bidirectional edges**, 38M events in **329 MB**, <100 ns decode), the tag design (t-tags benign-authentic/benign/unknown × c-tags secret/sensitive/private/public, and the **split of code vs data t-tags worth 1305× against 4.68×** for a single tag), four objective-based detection policies, **backward analysis as Dijkstra with tag-derived edge costs** (unknown→benign = 0, benign→benign = high, unknown→unknown = 1, stopping as soon as an entry point joins the shortest-path tree) and forward analysis pruning 100–500×, Table 11's end-to-end **38.5M events → 130** (297,100×) with a 54,517× average, and Table 7's 174 correct / 0 incorrect / 2 missed across eight DARPA campaigns where >99.9% of events were benign), experiments crate `attack-experiments` (lane 1 provided in `ad_graph.rs` with `AdConfig::tiered()` as the clean-directory preset; `chokepoint.rs` stub — `immediate_dominators` + `blast_radius`, with `exposure`, `rank_chokepoints` and the `blast_radius_naive` delete-and-recompute oracle provided; `authz.rs` stub — `check_pointer` with cycle protection and optional memoization, `LeopardIndex::build`, `intersect_galloping`, with the store generator and linear-merge straw man provided; **3 provided tests pass, 9 fix the contract for the stubs**, including exact dominator-vs-oracle agreement on every node in both directory regimes, index-equals-pointer-chasing on every user × group pair, cycle termination, and galloping-beats-merge by >1000×), and capstone M40 (edge-kind-filtered variable-length reachability as a Cypher procedure over M31's storage with the traversable-kind mask as a roaring set, a one-pass dominator choke-point procedure over the topic-18 CSR, and a Zanzibar-shaped `check(subject, resource#relation)` with a maintained closure index on the property layer). Cross-topic threads worked: 26/23 (roaring principal sets, galloping intersect), 27 (derived edges and the Leopard closure as materialized views; MulVAL's graph as a Datalog derivation), 1 (lane 3 is a RUM triangle), 37 (SpiceDB's bounded scatter-gather, and Zanzibar hedging to Spanner/Leopard but *never* between its own servers), 18 (CSR traversals), 12 (SLEUTH's encoding as the columnar argument), 33 (provenance as a contact sequence), 39 (both topics score a graph against an adversary who reads the score). + +## 2026-07-26 — topic 39 — Fraud Rings & Identity Graphs + +**topic 39 Fraud Rings & Identity Graphs added** (second of the six graph use-case deep dives; per user, 39-43 proceed without per-topic review): study guide (**camouflage kills row scores, measured** — bench lane 1: 5000×5000 Zipf(0.7)×Zipf(0.8) background, 50k edges, planted 25×100 block at density 1.0; precision@|fraud users| at camo/fraud-edge {0, 0.5, 1, 2}: degree-rank **0.00/0.28/0.60/0.76** (misses economical fraud, lights up only once camouflage inflates the row), obscurity-rank **0.52/0.00/0.00/0.00** (mirror image — dies the moment camo buys popular columns) — both are functions of the fraudster's own row and he tunes camo ≈ 0.5 to slip between them; FRAUDAR column-weighted peeling reference holds **F = 1.00 in every regime** while unweighted g degrades **1.00/0.95/0.69/0.65** (camo glues the block to the power-users × hit-products core); peel of a 100k×50k-node / **1,019,984-edge** graph in **~0.2 s** at F = 1.00; Fellegi–Sunter lane: 15,000 records (5000 entities × 3 dups, 5 fields pools [200 500 3650 200 2000] typo [.10 .07 .03 .12 .05]) — naive **112,492,500 pairs → blocked 271,012 (415×)** via two passes (last name OR dob), sampled **u = [0.0052 0.0021 0.0003 0.0051 0.0006] ≈ 1/pool**, EM **m = [0.80 0.86 0.94 0.78 0.90]** vs analytic (1−t)² [0.81 0.87 0.94 0.77 0.90], p = 0.184, link at 12 bits: **precision 0.989 / recall 0.992 in 48 ms**), 4 reading guides (FRAUDAR KDD'16 read in full — axioms, g(S)=f(S)/|S|, column weights 1/log(d+5), greedy peel O(|E| log |V|), Theorem 2 ½-approximation, Theorem 3 camouflage-resistance (camo lands on honest columns, block columns never change), F above 0.95 for 200×200 injected blocks under all four camo attacks, Twitter 41.7M users/1.47B edges → 4031×4313 block at 68% density with 57% hand-labeled fraud vs 12-25% controls; Winkler 2006 survey pp. 1-22 — FS decision rule R=P(γ|M)/P(γ|U) with T_λ/T_μ + clerical band proved optimal, per-field log2(m/u) weights, exact matching misses over 25% of census matches → Jaro-Winkler comparators, EM (Winkler 1988), multi-pass blocking 10¹⁷ → 10¹² pairs keeping 99.5% of matches, BigMatch 100M×4B at ~100k pairs/s with 10 passes in one data pass, 1990 census clerical 3000×3mo → 200×6wk; FlowScope AAAI'20 read in full — laundering = dense multi-step flow on k-partite X→W→Y, f_i=min(in,out), g=(1/|S|)Σ[(1+λ)f_i−λq_i] λ=4 so parking/camouflage LOWER the score, CBank 6.13M accounts/43.98M transfers with a labeled real ring (4 sources/12 mules/2 destinations ≈452M yuan): FAUC 0.761/0.843 vs FRAUDAR 0.529/0.704, F1 ≥ 0.9 down to 76M vs 180M injected — covered as guide + exercise 5, no stub; splink code read @ 04189f5 with 14 verified anchors — linker.py:66 façade, training.py:163 estimate_u_using_random_sampling / :231 one-EM-session-per-blocking-rule, expectation_maximisation.py:225 (E :18 / M :193), comparison_level.py:148 with match weight log2(m/u) :426 + _tf_adjustment_sql :667, graded levels comparison_level_library.py:406/:458/:493, predict.py:203 prior+weights → 1/(1+2^(−mw)), blocking.py:747 passes as SQL self-joins, clustering.py:43 → connected_components.py:121, dialects.py:24 one model on DuckDB/Spark/SQLite/PostgreSQL :270/:402/:532/:674), experiments crate (review_graph.rs PROVIDED — Zipf background + planted block + Zipf(1.5) popularity-biased camouflage + both naive rankers, 3 tests green incl. obscurity 0.75 → under 0.3 at camo 2; fraudar.rs + er.rs stubs with 6 contract tests: log-weighted F ≥ 0.9 with and without camo / unweighted F below 0.7 at camo 2 (measured 0.643) / g(returned) ≥ g(planted)/2; u within 0.005+expect of 1/pool / per-pass EM p,m within 0.05 of labeled empirical with the blocked field NaN / match-weight gap over 20 bits / blocking ≥ 20× / precision ≥ 0.95 recall ≥ 0.9; **the design discovery: a fixed-u EM over the unioned blocked candidates degenerates to fitted p → 1.0** (every candidate agrees on a blocking key by construction, class U cannot explain it) — the fix IS splink's API shape, one session per pass excluding its own blocking field, m averaged; margins that keep the contracts honest: block density 1.0 required (0.9 → log F 0.702), camo 4 breaks even log weights (0.619), 20×80 wide-short block caps per-column camo at 20 edges ≈ 3.06 weighted degree below block g 4.97, threshold 12 bits clears the coincidence patterns dob+city ≈10.3 / dob+first ≈10.6 / last+phone ≈10.95 where 8 bits chains precision down to 0.85; reference verified 9/9 then reverted, 0 warnings, bench prints lane 1 + `[stub …]` banners via catch_unwind), PLAN §39, capstone M39 (dense-block peel as a procedure over M31 storage reading weighted degrees off the topic-18 CSR + write-time identity resolution with blocking-key indexes, FS weights in the property layer, incremental union-find; targets: ~5M edges/s peel on 10M-edge synthetic, per-insert resolution latency at 1M records with two blocking indexes, precision/recall vs lane 3's 0.989/0.992). Same verified-facts-then-agents-write workflow; splink newly cloned, FRAUDAR/FlowScope/Winkler PDFs to /tmp, all read. + +## 2026-07-26 — topic 38 — GraphRAG & Agent Memory + +**topic 38 GraphRAG & Agent Memory added** (first of the six approved graph use-case deep dives, FalkorDB's core market; pilot — review before 39-43): study guide (**the path-finding collapse measured** — bench lane 1: mean rank of the true answer among 17 candidates, chance = 9.0 — mention-count ranking (vector RAG's shape) **1.00 at 1 hop → 9.21 at 2 hops → 8.71 at 3**; BFS distance **9.51/8.95/9.15** at all hops — coverage without association; PPR reference restores **1.00/1.00/1.00** since restart mass from both seeds SUMS at the meet node; one PPR query, 100k nodes / ~400k directed edges, 30 power iterations = **56.6 ms**; bi-temporal store reference: 10k entities × 10 job changes → **100,000 edges kept, 10,000 current, as-of scan 0.09 ms** — nothing deleted, any moment answerable), 4 reading guides (HippoRAG NeurIPS'24 read in full — hippocampal index analogy, 2-step OpenIE, synonymy τ=0.8, PPR damping 0.5, node specificity |Pᵢ|⁻¹; R@2/R@5 MuSiQue 40.9/51.9, 2Wiki 70.7/89.1, HotpotQA 60.5/77.7; 10-30× cheaper 6-13× faster than IRCoT; AR@5 2Wiki 37.1→75.7; Südhof path-finding case; Microsoft GraphRAG 2404.16130v2 read in full — 600-token chunks/gleanings, exact-match dedup with duplicate-count edge weights, hierarchical Leiden (graspologic), degree-ordered bottom-up community summaries, shuffled map-reduce with 0-100 helpfulness; Podcast 8,564 nodes/20,691 edges + News 15,754/19,520, indexing 281 min gpt-4-turbo; comprehensiveness win 72-83%, C0 = 26,657 tokens ≈ 2.6% of TS, 9-43× fewer; Claimify 34.18 vs 25.23 claims/answer; Zep 2501.13956 read in full — episode/entity/community tiers, §2.1 bi-temporal four timestamps, LLM edge invalidation keeps expired edges, dynamic label propagation, φ→ρ→χ retrieval; DMR 94.8 vs MemGPT 93.4, LongMemEval 60.2→71.2% with latency 28.9→2.58 s and context 115k→1.6k tokens, temporal +38.4%, regression single-session-assistant −17.7%; GraphRAG-SDK code read @ f42ab3d — fixed 9-step IngestionPipeline pipeline.py:35 with mandatory lexical graph + concurrent mentions∥index :175, 2-step GLiNER-then-LLM extraction graph_extraction.py:89, 4-strategy resolution ladder up to embedding+LLM llm_verified_resolution.py:192, vector+fulltext indices INSIDE FalkorDB vector_store.py:35, survivor-pattern dedup :228, rule-based router router.py:19, 9-step MultiPathRetrieval multi_path.py:48 with four parallel chunk paths and cosine rerank top_k=15), experiments crate (kg.rs PROVIDED — synthetic path-finding instances, 3 tests green; ppr.rs + temporal.rs stubs with 6 contract tests: PPR is a distribution + chain decay + meet-node rank 1; invalidate-without-delete t_invalid=Some(200)/t_expired=Some(205) + event-time reconstruction + late-fact known-vs-true split; bench lanes 2-3 print `[stub …]` until solved; reference verified 9/9 then reverted, 0 warnings), PLAN §38, capstone M38 (PPR as graph procedure over topic-18 CSR, bi-temporal versioning; targets: PPR recall@5 ≈ 1.0 where direct mention is chance, under 100 ms on 100k nodes, as-of within 2× current-only). + +## 2026-07-26 — topic 37 — Distributed Query Execution + +**topic 37 Distributed Query Execution added** (new topic, added to PLAN.md this session; second of the two approved scaling topics, completing the pair with 36): study guide (**the fan-out tail measured** — bench lane 1 run: analytic table P(any slow)=1−(1−p)ⁿ — at p=1/100: **1.0% for n=1 → 63.4% at n=100 → 99.3% at 500 → 100% at 1000**; at p=1/10,000 still **18.1% at n=2000** — fan-out exponentiates rarity into certainty, the component's p99 becomes the service's median at n≈70 since 0.99⁷⁰≈0.5; simulated 100-leaf scatter-gather at 1-in-100 slowness, 20k queries: **one-leaf p50/p95/p99 = 5.6/9.6/10.0 ms, wait-for-all-100 = 1000/1000/1000 ms, wait-for-95% = 9.6/9.9/9.9 ms** — the paper's Table 1 shape reproduced, good-enough results delete the tail; exchange-as-iterator ASCII, hedge timeline ASCII, DataFusion-vs-DistSQL production shapes), 4 reading guides (Volcano exchange TR CS/E 89-007/SIGMOD'90 verified against the PDF read in full — anonymous inputs so parallelism is one more iterator, packets through shared-memory queues, master/slave propagation-tree forking + primed processes, end-of-stream counted per producer 3×4=12, §4.4 broadcast-by-pinning + merging exchange must keep producers' records separate + exchange-in-the-middle makes flow control obsolete + fork-vs-reuse is a run-time switch, §4.5 two-level buffer locking never-hold-pool-lock-during-I/O + restart removes hold-and-wait = deadlock-free + ~100-instruction spin-locks + read-ahead/write-behind daemon, §4.6 vs GAMMA shared-memory/top-down/bushy vs shared-nothing/bottom-up/left-deep, §5 Sequent Symmetry 12×80386 numbers: **20.28 s single-process vs 28.00 s no-fork = 25.73 µs/record/exchange, forked 4-process pipeline 16.21 s beats single-process, packet sweep 171 s at 1 rec/packet → 94 at 2 → 15.0 at 50 → 13.7 at 83** = batching is a 12× swing, vectorization's argument made with processes; Tail at Scale CACM'13 verified against the PDF read in full — variability sources incl. SSD-GC-×100-reads, 63%/18% arithmetic, Table 1 real service 1/5/10 ms leaf → 40/87/140 ms at 100% vs 12/32/70 ms at 95% with slowest-5%-of-requests = half the p99, **hedged requests at p95 delay ≈5% extra load, BigTable 1000 keys/100 servers hedge-after-10 ms: p99.9 1,800→74 ms at +2% requests**, tied requests with cross-server cancellation + ≤1 ms stagger Table 2: idle p99.9 98→61 ms (−38%), with-terasort 159→108 ms (−32%), tied+terasort≈idle-unhedged at <1% disk overhead, probe-first loses 3 ways (staleness/estimation/herding), micro-partitions ~20/machine = 5% shed steps, latency-induced probation via shadow requests, canary requests on every Google fan-out, mutations easy: Paxos quorums inherently tail-tolerant; DataFusion RepartitionExec code-read with 20 verified anchors — RepartitionExec repartition/mod.rs:1150 + preserve_order :1160 = the merging exchange with per-(input,output) spill channels :398-538, BatchPartitioner :560 with **pinned seed-0 REPARTITION_RANDOM_STATE :592** so same-key-same-partition always (joins depend on it), partition_iter :825 routes whole batches round-robin but rows by hash via create_hashes :854 + strength-reduced % :675, distributor_channels.rs channels() :55 + Gate :62 + send :131 = N unbounded buffers with one global gate that parks senders only when ALL are non-empty (prevents distribution deadlocks in join plans), Partitioning enum partitioning.rs:117; **real finding: EnforceDistribution retired into EnsureRequirements** ensure_requirements/mod.rs:159, enforce_distribution.rs:18/:76 are helpers + the retirement note; cockroach DistSQL code-read with 18 verified anchors — checkSupportForPlanNode distsql_check.go:214, mustWrapNode :312 for no-processor-equivalent nodes, **PartitionSpans distsql_physical_planner.go:971 = the topic-36 bridge: range ownership becomes the parallel plan**, createPhysPlan :3604, OutputRouterSpec data.proto:149 with PASS_THROUGH/MIRROR/BY_HASH/BY_RANGE :152-:160 = Volcano's routing policies as a protobuf enum, Flow flowinfra/flow.go:72/Setup :272/Run :566, Outbox colrpc/outbox.go:50/:218/:323 + Inbox inbox.go:57/:212/:333 = exchange's two halves over gRPC with Inbox.Next an ordinary iterator — anonymous inputs surviving a network hop, hashRouter rowflow/routers.go:538 + vectorized HashRouter colflow/routers.go:443), experiments crate compiles: fanout.rs PROVIDED (two-mode leaf 1-10 ms fast/1000 ms stall, closed form, scatter_gather max + 95%-frac variant — 3 tests pass: 63.4%/18.1% exact arithmetic, simulation within ±2% at 20k trials, leaf-tail-becomes-service-median) — exchange.rs (Exchange::partition round-robin-cursor/splitmix64-hash to k outputs: deterministic-and-complete + balance-within-one-row + merge_sorted k-way keeps the multiset sorted) and hedge.rs (request_with_hedge fire-second-copy-only-past-delay: 10 ms hedge cuts p99.9 ≥10× + extra load <10% + zero-delay-doubles-requests) are `todo!()` stubs — 6 tests fail as todo panics; **reference solution verified 9/9 then reverted**: round-robin 229.6 M rows/s, hash 543.0 M rows/s balance 1.002, 8×500k merge 80.6 M rows/s, hedge@10 ms **p99.9 1000→18.3 ms at +0.5% requests** (hedge@0 = +100%, the degenerate case) — the paper's 1,800→74 ms shape; distq_bench lane 1 RUN (tables above), lanes 2-3 armed behind catch_unwind; notes.md predictions vs measurements + all verified anchors + PDF facts; M37 log: scatter-gather over M36's slots + in-engine exchange (hash for join build, round-robin for scans, per-producer end-of-stream counting) + hedged reads on slot replicas with p95 delay through topic-35's admission layer, targets = near-linear scale-up 1→2→4→8, p99.9 with one stalled shard within 2× no-stall when hedging, hedge overhead ≤5%. Same verified-facts-then-agents-write workflow; no new clones (datafusion/cockroach under ~/repos), Volcano + tail-at-scale PDFs to /tmp, both read in full. + +## 2026-07-26 — topic 36 — Sharding, Partitioning & Rebalancing + +**topic 36 Sharding, Partitioning & Rebalancing added** (new topic, added to PLAN.md this session; first of the two approved scaling topics — 37 distributed query execution is next): study guide (**mod-N's failure measured** — bench lane 1 run on 1M hashed keys: growing N→N+1 moves **80.0% at 4→5, 83.4% at 5→6, 88.9% at 8→9, 94.1% at 16→17** vs the ring's ideal 1/(N+1) = 20.0/16.7/11.1/5.9% — the closed form is exact, k mod N == k mod N+1 iff k mod N(N+1) < N (CRT), so movement = N/(N+1) and *worsens as you grow*; and the skew hashing provably can't fix: Zipf traffic on 16 hash shards, 10k keys/500k samples, hottest shard carries **9.5% at s=0.8 (1.5× the 6.25% ideal), 14.7% at s=1.0 (2.4×), 23.8% at s=1.2 (3.8×)** because a hash maps one key to one shard — only range splitting *between* keys or hot-key replication answers it; Dynamo strategy 1→2→3 table (fixed-partitions+movable-ownership beats boundaries-follow-node-identity: strategy-1 bootstrap "almost a day", strategy-3 metadata 3 orders smaller, partition-as-file), ring ASCII, redis MOVED-vs-ASK mermaid, cockroach split/merge/rebalance trigger table, edge-cut vs vertex-cut ASCII), 4 reading guides (Dynamo SOSP'07 verified against the PDF — MD5 128-bit ring, vnodes, preference list skipping same-physical-node, R+W>N with production (3,2,2), sloppy quorum + hinted handoff, per-range Merkle anti-entropy, §6.2's three partitioning strategies with Fig 8's efficiency numbers, imbalance 20%-low-load vs 10%-high, 99.94% one-version reads; PowerGraph OSDI'12 verified — α≈2 natural graphs, Twitter in-degree α=1.7 and 1%-of-vertices≈half-the-edges, Thm 5.1 random edge-cut = 1−1/p (87.5% at p=8), Thm 5.2 replication from the degree distribution with gains growing as α falls, Thm 5.3 vertex-cut ≤ ghosts of any edge-cut, greedy Cases 1-4, coordinated vs oblivious; redis cluster code-read with 14 verified anchors — CLUSTER_SLOTS=16384 cluster.h:23, keyHashSlot CRC16 & 0x3FFF + hash-tag carve-out :59, getNodeByQuery cluster.c:1191 → clusterRedirectClient :1443, CLUSTER_REDIR_ASK :1397 vs MOVED :1432 (MOVED updates the client slot map, ASK is one-shot + needs ASKING :1680), migrating_slots_to/importing_slots_from cluster_legacy.h:343-344, SETSLOT state machine cluster_legacy.c:6072-6075; cockroach rebalancing code-read with 12 verified anchors — RangeMaxBytes 512 MB zone.go:257, load splits at 2500 QPS replica_split_load.go:34 / 500 ms CPU :52, split_queue.go:145/:194, merge_queue.go:138, the Decider's windowed per-key sketch split/decider.go:155/:222/:329 with PopularKeyCount/NoSplitKeyCount as honest failure counters, AllocatorAction allocator.go:125, StoreRebalancer store_rebalancer.go:114/:218 lease-transfers-first — range splits are *semantic*, between keys, the answer to the Zipf row), experiments crate compiles: placement.rs PROVIDED (splitmix64, modn_movement, Zipf harmonic-CDF sampler, hot_shard_share — exact-80% + hashed-keys + Zipf-hot-shard tests) + graphs.rs PROVIDED (planted-partition + preferential-attachment generators, edge_cut, random baseline — random-cut≈0.875=Thm-5.1 test) — 4 provided tests pass, zero warnings — hashring.rs (consistent-hash ring with vnodes: ≈1/(N+1)-movement-all-to-the-new-node + remove-moves-only-its-keys + more-vnodes-tighter-balance contracts) and partitioner.rs (one-pass LDG greedy: score = placed-neighbors × (1−|P|/C), balanced-within-slack + beats-random-by-40% + deterministic contracts) are `todo!()` stubs — 6 tests fail as todo panics; shard_bench lane 1 RUN (tables above), lane 2 (ring 4→5 movement ≈20% vs mod-N 80%, removal moves only the removed node's share, balance vs vnodes 1/8/64/512) and lane 3 (edge-cut at k=8 random-vs-greedy on community + power-law graphs) armed behind catch_unwind; notes.md predictions vs measurements + all verified anchors + PDF facts; M36 log: slot = hash(vertex_key) & 0x3FFF with hash tags, edges live with source vertex, MOVED/ASK-style redirects + per-slot migration state machine with dual routing, migration runs at topic-35's lowest admission priority, targets = movement ≈1/(N+1), edge-cut + replication beat random on power-law, p99 during live migration within 2× steady state. Same verified-facts-then-agents-write workflow; no new clones (redis/cockroach under ~/repos), Dynamo + PowerGraph PDFs to /tmp. + +## 2026-07-26 — topic 35 — Overload Control & Resource Governance + +**topic 35 Overload Control & Resource Governance added** (new topic, added to PLAN.md this session; grew out of "more topics for maintaining a database in production"): study guide (**metastable failure measured** — bench lane 1 run on a deterministic virtual-clock queueing sim reproducing HotOS'21 Fig 2: 300 QPS server, clients time out at 1 s and retry once, single 10 s outage at t=30 s — at **280 QPS offered load the outage queues ~2,800 requests, every timeout fires a retry, offered load locks at 560 QPS and goodput is still 0 at t=199 s** (160 s after the trigger ended, provably forever: queue grows 260 req/s); at **140 QPS the identical trigger + identical 280 QPS storm heals at t=161 s** because 280 is below the 300 QPS capacity — the dividing line is **hidden capacity = capacity/(1+retries) = 150 QPS**, and recovery takes ~2 min for a 10 s outage because drain rate = headroom = 20 QPS; metastable stable/vulnerable/metastable lifecycle ASCII, work-amplification table (retry ×2, look-aside cache ×10 at 90% hit rate, failover herds, slow error paths), detection ladder response-time-recursive-vs-CPU-busy≠overloaded-vs-**queuing-time-local** + CoDel min-sojourn, DAGOR cursor mermaid, cockroach slots-vs-tokens table, redis edge-surfaces table), 4 reading guides (metastable HotOS'21 verified against sigops PDF — trigger vs sustaining loop, root cause = the loop not the trigger, Fig 2's 280/560/300 arithmetic, stable below 150 / recovery needs retries below 20 QPS, 3000-QPS-app-on-300-QPS-db cache example, Facebook link-imbalance 2-years-undiagnosed one-line MRU-pool fix, "emergent behavior… one cannot write a unit or integration test", Kraken live-traffic testing, trigger intensity 151-vs-299, reproduction needs Tene-honest load gen → topic 34; DAGOR SoCC'18 verified against arXiv:1806.04075 — subsequent overload Def 1 with 0.5×0.5=25% random-shedding math, **avg queuing time 20 ms over 1 s/2000-req windows** explicitly not response time (recursive false positives: DAGOR_r sheds at 630 QPS where DAGOR_q reaches the 750 QPS saturation) and not CPU, business priority hash table Login-highest/Pay-above-IM-100×-complaints copied down the call tree, 128 hourly-rotated user sublevels fixing τ/τ−1 oscillation (session priority rejected: users re-rolled by logout/login), Algorithm 1 α=5% multiplicative-down β=1% additive-up on admit counts via priority-histogram prefix sums, collaborative shedding piggybacks the cursor upstream so rejects cost the overloaded server nothing, ~50% higher success than CoDel/SEDA on M², fairness uniform M¹–M⁴; redis code-read with 16 verified anchors — EVPOOL_SIZE 16 evict.c:36, evictionPoolPopulate :134 sampling maxmemory-samples=5 config.c:3223, getMaxmemoryState :384, performEvictions :532 before each command, OOM gate is_denyoom_command server.c:4391 → performEvictions()==EVICT_FAIL :4485 → rejectCommand oomerr :4498 reject-before-work, -BUSY server.c:2130 after busy_reply_threshold script.c:150, output-buffer limits checkClientOutputBufferLimits networking.c:5151 / async close :5215 as slow-consumer backpressure, CLIENT PAUSE pauseActions server.c:4850 — single thread can't shed by priority so every surface converts an unbounded queue (memory/replies/time) into a bounded fast error; cockroach admission code-read with 13 verified anchors — package doc admission.go:1 shift-queueing-out-of-the-goroutine-scheduler-into-reorderable-WorkQueues, slots-vs-tokens grantKind :54 concurrency-for-CPU vs rate-for-IO-because-compaction-debt-lands-later, requester/granter :178/:198, WorkQueue (tenant, WorkPriority int8 ladder admissionpb.go:23 LowPri=MinInt8…UserHighPri=50, FIFO ts) work_queue.go:303/Admit :813, kvSlotAdjuster AIMD on **runnable-goroutines-per-CPU sampled at 1 ms** kv_slot_adjuster.go:16/:46 = queuing-time detection in scheduler clothing, ioLoadListener L0 file/sub-level thresholds io_load_listener.go:69/:77 = topic 4's write-stall signals promoted to node-wide policy), experiments crate compiles: sim.rs PROVIDED (open-loop arrivals, client-timeout-but-server-does-the-work-anyway work amplification, retries at arrival+timeout, outage trigger, Policy trait admit/allow_retry/observe_queuing — 3 exact-arithmetic tests pass incl. offered=1600=exactly-2× in the collapsed window and vulnerable-without-trigger-is-invisible) — tokenbucket.rs (retry budget: burst-then-deny, steady-rate 10-in-1s, idle-does-not-accumulate) and admission.rs (DagorGate: healthy-admits-all, overload-sheds-lowest-first-never-prio-0, additive recovery) are `todo!()` stubs — 6 tests fail as todo panics; overload_bench lane 1 RUN (table above), lane 2 (retry budgets 15 vs 25 QPS straddling the 20 QPS headroom — one heals, one never) and lane 3 (DAGOR-lite at 2× overload: goodput + per-priority success + admitted p99, vs no-control FIFO starving everyone) armed behind catch_unwind; notes.md records the 2,800-queued/560-locked/260-per-s-growth arithmetic + predictions for lanes 2-3; M35 log: per-query priority + queuing-time cursor on the executor (1 s/2000-query windows), -BUSY-style fast reject with retry-after hint, plan-time memory gate (DENYOOM per query), targets ≥80% of saturated throughput at 2× overload + lane 1 reproduced-then-fixed on the real engine. Same verified-facts-then-agents-write workflow as topics 33/34; no new clones (redis/cockroach already under ~/repos). + +## 2026-07-26 — topic 34 — Debugging & Production Diagnosis + +**topic 34 Debugging & Production Diagnosis added** (new topic, added to PLAN.md this session; grew out of "how do I debug a database in production"): study guide (**coordinated omission measured** — bench lane 1 run on a virtual clock, deterministic: 1M ops, service 1 µs, 100 ms stall every 100K ops, arrivals every 10 µs — closed-loop reports p50/p99/p99.9/p99.99 all **1.0 µs** (only max sees a stall) while open-loop reports **p99 = 90.0 ms, p99.9 = 99.0 ms, p99.99 = 99.9 ms** = a **90,000× lie at p99**; each stall queues ~10K arrivals, 9 stalls → ~9% of requests carry decaying queueing delay, so the worst 1% are exactly the ≥90 ms victims — provable arithmetic, not noise; three-failure-currencies diagram wrong-answers/too-slow/crashed → replay/measurement/forensics; rr one-diagram; redis 3-tier cost table always-on-one-compare / armed-160-sample-rings / on-demand-doctors; RocksDB PerfLevel mermaid), 4 reading guides (rr ATC'17 verified against arXiv:1705.05937 PDF — record boundary = user/kernel interface, nondeterminism = syscall results + async-event timing, RCB the only deterministic HW counter so execution point = (RCB, registers), seccomp-bpf + RR-page + 2-byte-syscall→5-byte-call rewrite avoids 4 ctx switches/syscall, <2× slowdown, one-thread-at-a-time so weak-memory unobservable; Gregg flame graphs CACM 59(6) 2016 — x-axis is alphabetical merge not time, width = sample fraction, on-CPU vs off-CPU for lock/fsync waits; redis code-read with 13 verified anchors — slowlogPushEntryIfNeeded slowlog.c:103 with :104 negative-disables/:105 >=-logs, slowlogCreateEntry :28 arg/string trimming, latencyStartMonitor/latencyAddSampleIfNeeded latency.h:50/:63 zero-cost-when-off macro discipline (default 0 config.c:3271), LATENCY_TS_LEN=160 latency.h:17, same-second max-coalescing latency.c:82, LATENCY DOCTOR createLatencyReport latency.c:182, MEMORY DOCTOR object.c:1421, watchdog sigalrmSignalHandler debug.c:2643→logStackTrace :2115; RocksDB code-read with 10 verified anchors — PerfContext perf_context.h:305 thread-local via get_perf_context() :342, PerfLevel ladder perf_level.h:27 kDisable=1/kEnableCount=2/kEnableWait=3/kEnableTimeExceptForMutex=4 (agent corrected brief's kEnableWaitForMutex against the clone), PERF_TIMER_GUARD perf_context_imp.h:27 compiled out under NPERF_CONTEXT, PerfStepTimer RAII perf_step_timer.h:13, HistogramBucketMapper 109 buckets histogram.h:21/:84, StatisticsImpl/recordTick statistics_impl.h:42/statistics.cc:549 as the global tier), experiments crate compiles: workload.rs PROVIDED (StallModel virtual clock, closed_loop the-liar vs open_loop charging completion−intended, exact-arithmetic tests incl. lat[101]=991_000 — 3 provided tests pass) — histogram.rs (LogHistogram: linear below 2^sub_bits then 2^sub_bits sub-buckets/octave, contracts = est≥true within 1/32 relative error + merge-equals-bulk + memory-never-grows) and slowlog.rs (redis semantics exactly: >=-threshold logs, negative disables, ring evicts oldest, ids monotonic across reset) are `todo!()` stubs — 6 tests fail as todo panics; debug_bench lane 1 RUN (table above), lane 2 (histogram record ns/op + percentile error vs sort-everything on 10M latency-shaped samples) and lane 3 (**the observability tax**: hot-loop ns/op bare vs +clock-pair vs +histogram vs +slowlog — M34's overhead budget) armed behind catch_unwind; notes.md predicts lanes 2-3 + records the lane-1 arithmetic; M34 log: GRAPH.SLOWLOG port (FalkorDB src/slow_log/slow_log.c exists in C) + parse/plan/execute/serialize step timers behind a PerfLevel dial, level 0 provably free via lane 3, full level <5% on M11 suite, before-shot = lane 1 reproduced on the real engine under an induced stall. Same verified-facts-then-agents-write workflow as topic 33; no new clones (redis/rocksdb/FalkorDB already under ~/repos). + +## 2026-07-22 — topic 33 — Temporal Graphs + +**topic 33 Temporal Graphs added** (new topic, added to PLAN.md this session): study guide (**the static-condensation lie measured** — bench lane 1 run: 2000 nodes, contacts uniform over a 10K-tick horizon, static BFS vs time-respecting reachability for 20 sources: at 4K contacts static claims 25,031 reachable pairs but only 137 have a time-respecting witness = **99.5% false positives**, falling 97.5% → 56.9% → **0.0% at 64K contacts** where temporal saturates to all 39,980 static pairs — the transition is sharp, static reach is the T→∞ limit; contact (u,v,t,λ) model + valid/transaction/bitemporal axes ASCII; reachability-is-not-transitive so "shortest" splits into earliest-arrival/latest-departure/fastest/shortest; storage-menu table snapshot-per-t / event-log-first-Raphtory / anchor+delta-AeonG / MVCC-as-history keyed on AT-TIME cost + anchor/delta mermaid), 4 reading guides (Wu et al. VLDB'14 — condensing lies, four minima that need four algorithms, Dijkstra's subpath invariant dies to a cheap-prefix-misses-the-bus counterexample, one-pass O(n+M) earliest-arrival over a time-sorted stream in Rust, dominance lists for fastest/shortest, time-expanded O(M) DAG as the materialized-view alternative; Paranjape/Benson/Leskovec WSDM'17 — δ-temporal motifs, the 36-motif derivation, window-scan DP with cnt[i][j] fragment counters and the expire-shortest-first/insert-longest-first correctness orders, stars-cheap-triangles-O(m√m), blocking-vs-non-blocking fingerprints; AeonG VLDB'24 verified against arXiv:2304.12212v2 — per-VERSION lifespan ω in transaction time, FOR TT AS OF/FROM..TO scoped to MATCH, VP/VE/EP three-clocks split, GC-as-migration Algorithm 1 riding the reaper thread = the 9.74% headline, KV SkipList keys type+Gid+ω with A/D anchor/delta bit, adaptive anchoring Eq 1 three bands, legal check Eq 2 + both-store consults, 5.73× storage / 2.57× latency numbers; Raphtory code-read on fresh clone with 8 verified anchors — EventTime(i64,usize) tiebreaker timeindex.rs:28 solving exactly Wu's λ=0 tie-order problem in the type system, TimeIndex/TCell size-adaptive enum ladders, TPropCell time→offset into columnar PropColumn, WindowedGraph derives-Copy = BETWEEN as a zero-copy lens + TimeOps::window composing on every view type, db4 segments as the batch-into-arrays correction; caught stale name: TimeIndexEntry renamed EventTime), experiments crate compiles: events.rs PROVIDED (gen_contacts λ=1 sorted, static_reachable the-liar, earliest_arrival_oracle as deliberately-Bellman-Ford fixpoint so lane 2's one-pass speedup is a measurement not a tautology, replay_at_time naive AT-TIME oracle — 3 provided tests pass) — temporal_reach.rs (one-pass earliest_arrival, matches-oracle-on-3-random-streams + respects-start-time + λ=0-chains contracts) and snapshot.rs (AnchorDeltaStore append/at_time/replay_len, matches-full-replay + anchor-spacing-bounds-replay` sections (first sentence defines the concept assuming zero DB-internals background, then a real-numbers example / ASCII diagram / the guide's existing code sample, then why-it-matters; each step uses only terms defined in earlier steps, terms of art defined parenthetically at first use), then the navigation section ("How to read the paper (with the concepts in hand)" for papers, "Where each step lives in the code" with anchors grouped by step for code reads), Questions/Takeaway/References verbatim at the end; all existing assets (diagrams, code samples, file:line anchors, tie-backs) preserved and reorganized, H1 titles unchanged so SUMMARY.md links held. Executed by 17 parallel agents (2-3 topics each; two stalled mid-batch — topics 03 and 09 refinished by follow-up agents); exemplars hand-written first (reading-drepper.md for paper reads, reading-turso-btree.md for code reads). Verification: all 186 guides pass structure checks (≥4 steps, exactly one problem statement and References), 185 SUMMARY.md link titles match on-disk H1s with 0 mismatches, all 49 mermaid diagrams parse under mermaid 11.6.0 (jsdom harness), fence-and-backtick-aware angle-bracket scan clean (2 false positives from multi-line backtick spans), mdBook build green. + +## 2026-07-12 — restructure rollout: all 179 remaining reading guides across topics 00-25 and 27-32… + +**restructure rollout: all 179 remaining reading guides across topics 00-25 and 27-32 rewritten as self-contained chapters** (topic 26 was the pilot, previous entry), executed by 8 parallel agents (4 topics each) against the same spec: concept-first H1 titles replacing "Reading guide — ...", Sources blocks replaced by 2-4 sentence framing leads, one inline Rust-ish code sample of the core algorithm added where the guide lacked one (skips documented for pure surveys / guides already carrying equivalent code), all existing content kept (diagrams, line-anchor tables, questions, tie-backs), `## References` appended with Papers (arXiv) + Code (GitHub) links carrying the old reading advice, filenames unchanged to avoid link churn; the 32 topic READMEs' guide lists updated to the new titles; one agent (topics 16-19) hit context limits with 2 files left — reading-umbra-tidy-tuples.md (retitled "Umbra & copy-and-patch: the war on compile latency" + copy-and-patch memcpy/patch-holes sample + References) and reading-sqlite-vdbe.md (References) finished by hand; SUMMARY.md link titles regenerated centrally by script from the actual on-disk H1s rather than agent reports (179 updated); verification: zero old-style H1s, zero Sources blocks, zero guides missing References, fence-and-backtick-aware bare-angle-bracket scan across all 186 guides found one genuine hazard (`HashMap` in topic 7's ae guide, backticked), mdBook build green. The whole book now reads as chapters instead of pointers. + +## 2026-07-12 — book-quality pass, three moves + +book-quality pass, three moves: (1) paper audit against dbscholar's citation-PageRank ranking (rmarcus.info — pulled the underlying data.json, 11,867 SIGMOD/VLDB/CIDR/PODS papers) — resources/papers.md gains a "Modern systems & directions" section and 6 topic READMEs gain "Further references" (Kung-Robinson OCC '81, Calcite, Spark SQL, Kipf/Neo/Bao learned optimization, Photon, Velox, GAMMA, Dremel, Lakehouse+Delta Lake, MillWheel, CockroachDB); (2) all `~/repos/...` code references linkified to their GitHub repos (scripted, fence-aware: 143 links across 115 files) so the online book's code pointers resolve; (3) **restructure demo on topic 26** — all 7 reading guides rewritten as self-contained chapters: concept titles ("HyperLogLog: count distinct in 12 KB" not "Reading guide — ..."), framing lead instead of a Sources block, an inline Rust code sample of each core algorithm (HLL add/merge + Ertl count skeleton, blocked-bloom 6-probe loop with golden-ratio remix, cuckoo kick loop with the XOR involution, PGM shrinking-cone add_point, roaring galloping intersect, BRIN one-sided range prune, Morton interleave64 magic masks), and a "## References" section at the bottom (papers with arXiv links, code with GitHub links); filenames kept (`reading-*.md`) to avoid link churn; SUMMARY.md + README titles updated; mdBook build verified locally (mdbook-mermaid install + build green). If the format lands, roll it out to the other 32 topics. + +## 2026-07-12 — PLAN.md expansions backfilled into the four already-scaffolded topics + +PLAN.md expansions backfilled into the four already-scaffolded topics (the plan-only commit 2fb095a now has matching study material): topic 7 gains "Bolt: the third answer" — RESP/pgwire/Bolt framing-typing-streaming table (Bolt's PULL{n}/DISCARD = protocol-level backpressure, §4's problem solved at the wire) + reading-bolt-packstream.md anchored on FalkorDB's *removed* Bolt server read frozen via `git show 0b11a00b3^:src/bolt/` (#2170, 2026-07-08): session-sequence ASCII with handshake bolt_api.c:803/version-clamp :845-864, RUN-executes-but-PULL-streams :467-482/:504-521 decoupling, PackStream marker nibbles bolt.c:11/:21/:36 + graph types Node 0x4E/Rel 0x52/Path 0x50 in the type system, why-was-it-removed as a question, M7 stretch = Bolt listener beside RESP; topic 13 gains "The query-language landscape" — 6-language table (Cypher/GQL/SQL-PGQ/SPARQL/Gremlin/Datalog) on model/matching/composability/pushdown + reading-query-languages.md (SIGMOD '22 GQL+PGQ paper, count-2-paths-in-a-triangle under homomorphism/isomorphism/trail as the same-pattern-three-answers demo, family-tree mermaid, kuzu Cypher.g4 anchor, M13 rule: keep the AST GQL-shaped — quantified path patterns + explicit path-mode field); topic 20 gains "Parallelism: OpenMP inside SuiteSparse, rayon in Rust" — saxpy3's costed static scheduling (coarse/fine tasks GB_AxB_saxpy3.c:22-48, nthreads-from-flopcount slice_balanced.c:418, parallel flopcount pass :219) vs rayon work-stealing (join/mod.rs:93 inline+push+steal, registry.rs:248 crossbeam_deque Stealer) + reading-openmp-vs-rayon.md with the static-vs-stealing trade table, no-native-Rust-GraphBLAS note (crates are FFI), new M20 checkbox: document each OpenMP→rayon mapping decision; topic 26 gains "Geo indexes: 2D keys through 1D indexes" — valkey GEO as geohash-in-a-zset (interleave64 geohash.c:52 → 52-bit Morton score, geohashEstimateStepsByRadius helper.c:64, 9-cell scan geo.c:375 + haversine verify = bloom's candidate-then-verify control flow) + reading-geo-indexes.md (Z-order seams vs Hilbert, Guttman R-tree/GiST, S2 prefix-containment vs H3 hexagons, M26 mapping: Morton key through the existing sorted property index). SUMMARY.md gains the 4 new guides so they appear in the book. + +## 2026-07-11 — topic 32 scaffolded + +topic 32 scaffolded (added to the plan this session, so the scaffold-all-topics run is complete again): study guide (**the HTAP problem measured** — bench lane 1 run: 1M-row store behind one coarse lock, fixed 2 s window per mode: writes alone = 11,438,647 writes at p99 333 ns; writes + a free-running full scanner = **69 writes** with p99 **7.49 seconds** — not slowdown, *starvation*: std Mutex is unfair, the scanner re-wins the lock after each ~0.6 ms scan (3261 scans) and the parked writer never gets in; interference at its worst is zero writes, and the coarse lock is deliberate — mitigations ARE the topic; the freshness/isolation/cost trilemma triangle; the architecture-menu table HANA-delta+main / HyPer-fork() / TiFlash-learner / F1-Lightning-CDC / pg_duckdb-offload keyed on one-copy? freshness isolation; the changelog-is-the-glue mermaid tying topic 27's thesis to every split design; the same-fold-four-costumes thread: topic 4 LSM minor compaction = HANA delta merge = TiFlash segmentMergeDelta = FalkorDB delta-matrix flush), 4 reading guides (TiDB VLDB '20 — columnar-copy-as-Raft-LEARNER so the replica costs no write-quorum latency, freshness-is-a-WAIT with tiflash LearnerRead.cpp:35 doLearnerRead + :61 waitIndexTimeout as the anchor, one-planner-two-engines via tidb find_best_task.go:535/:1841/:1878 TiFlash-paths-retained-so-cost-not-topology-decides, learner-log vs CDC-changelog trade question; TiFlash DeltaTree — Segment.h:84 delta-over-stable ASCII with MemTableSet/DeltaValueSpace.h:65 as a-little-LSM-inside-the-delta, Delta/MinorCompaction.h why-compact-the-delta-at-all, DeltaIndex.h:27 as the structure that makes delta+stable merge reads cheap (the thing our scan_sum_a lacks), DeltaMergeStore.h:668 segmentMergeDelta with our merge_preserves_scans test as its correctness condition, MVCC-versions-in-both-layers GC question linking topic 31's causal stability; HyPer ICDE '11 + HANA SIGMOD Record '12 paired as the one-copy family — fork() CoW-page ASCII (snapshot cost ∝ dirtied pages = MVCC-where-the-version-chain-is-the-page-table, GC is exit()), snapshot-ages-until-re-fork = lane 3's apply interval in OS clothing, HANA delta+main = our replica.rs minus segmenting, HANA's-trilemma-corner-is-exactly-what-lane-1-measures; F1 Lightning VLDB '20 + Özcan SIGMOD '17 survey — CDC-fed HTAP with zero OLTP changes, **safe-timestamp = applied_lsn productionized** (reads never wait, served stale-but-consistent at the max fully-applied ts — the opposite choice from doLearnerRead), Changepump ordering question = topic 27 changelog + topic 29 Spanner timestamps, the survey's copies×engines quadrant with every cell trading the same three currencies), experiments crate compiles: row.rs PROVIDED (RowStore = rows + every-write-appended-to-log changelog + scan oracle + skewed_key + percentile — 2 provided tests pass) — replica.rs (ColumnarReplica delta+main: apply/scan_sum_a/merge_delta with delta-overrides-main + highest-lsn-per-key-wins + merge-preserves-scans-and-sorts contracts, freshness_is_visible via applied_lsn gap — TiFlash DeltaTree in miniature, 4 tests), learner.rs (read_wait over an apply schedule: first-batch-covering-read_index, Some(0) if already applied, None = waitIndexTimeout — doLearnerRead as arithmetic, 3 tests) are `todo!()` stubs — 7 tests fail as todo panics; htap_bench lane 1 RUN (table above; original fixed-200K-writes design serialized into minutes behind ~0.6 ms scan lock-holds — switched to fixed-2s-window per mode, making the writes-completed collapse the headline number), lanes 2 (scan row-vs-delta-heavy-vs-merged + freshness max-lsn-gap vs batch 1K/10K/100K) and 3 (learner-read wait distribution vs apply interval 1/10/100, 50K reads demanding lsn==now) armed behind catch_unwind; notes.md predicts lanes 2-3 + records the starvation surprise + the RwLock exercise; M32 log: Lightning-shaped not TiFlash-shaped (no consensus group until M15, decoupling = zero primary changes), M27's changelog feeds a delta-matrix replica, router advertises applied_lsn safe timestamps + freshness-bound routing with read_wait-or-fallback, before-shot recorded: 69 writes/2s when analytics shares the copy, success = restoring the 11.4M with scans elsewhere. Cloned tiflash + tidb. + +## 2026-07-11 — topic 31 scaffolded + +topic 31 scaffolded (the last topic): study guide (consensus-vs-CRDT as the same problem in opposite currencies — agree-on-an-order vs design-so-order-doesn't-matter, 1-RTT vs 0-RTT, unavailable-in-minority vs available-under-any-partition; SEC via join-semilattice mermaid; CvRDT/CmRDT table with where-each-lives-in-this-crate; the zoo table clock/lww/counter/orset/rga/graph with the one idea per structure; **LWW's lie measured** — lane 1 run: two replicas, 20K writes each, LWW map: 10 hot keys + sync-every-write loses **94.98%** of writes, 1000 keys + sync-every-100 loses **88.34%**, even 100K keys + rare sync loses **12.45%** — "eventually consistent" without conflict semantics is not a semantics, priced; sequence-CRDT integration ASCII with the interleaving dragon; code-reading table across all five cloned repos), 4 reading guides (Shapiro SSS'11 + INRIA RR-7506 — SEC's three clauses, the CvRDT⇔CmRDT equivalence proof, the catalog reading map ending at §4's graphs where concurrent addEdge∥removeVertex is declared application-specific = the dangling-edge problem M31 inherits, causal-stability GC question; Kleppmann arc JSON-CRDT '17 + move-op '21 + Local-First Onward!'19 — ops-address-identities-not-paths with automerge op_set2/op.rs:52 succ as deletion-by-successor-ops, the concurrent-move duplicate/cycle problem and its undo/redo total-order fix, the does-move-survive-in-graphs M31 design question; sequence CRDTs in production — yrs block.rs:160 ID=our-Dot / :1302 Item with origin+right_origin=YATA's pair vs RGA's single parent / :1415 Item::integrate as the loop our rga.rs apply implements plus run-coalescing splits, diamond-types merge.rs:142 self-described "bastardization" + yjsspan.rs:29 INSERTED/NOT_INSERTED_YET retreat/advance = only-be-a-CRDT-at-merge-time, Loro/Fugue maximal-non-interleaving with the letter-soup demo, automerge-vs-loro bench as scratch-project exercise per deps convention; cr-sqlite as THE-database-goes-multi-master — crsql_as_crr clock-row-per-CELL diagram, local_writes/mod.rs:83-133 db_version bookkeeping = Lamport-clock spine, compare_values.rs version-tie-broken-by-VALUE-comparison = deterministic convergence with zero clock trust, changes_vtab.rs replication-endpoint-as-virtual-table, delete-wins-for-rows vs our add-wins-for-nodes tension question, the M31 change-feed schema design question), experiments crate compiles: clock.rs PROVIDED (Dot + VClock tick/covers/merge/partial_cmp-None-defines-concurrent, modeled on automerge clock.rs:109/:145) + lww.rs PROVIDED (register/map with (ts,replica) total order; merge counts its own discards so lane 1 can price the lie) — 6 provided tests pass — counter.rs (G/PN with semilattice-law tests + why-PN-is-two-G-Counters), orset.rs (add-wins over Dots: add-tags-fresh-dot / remove-kills-observed-dots, concurrent-add-beats-remove + remove-covers-all-observed-tags + seeded-permutation convergence), rga.rs (insert-after-parent + skip-larger-(counter,replica)-siblings + tombstones-still-anchor, idempotent apply, delete∥insert-after-it convergence), graph.rs (OR-Set nodes/edges + LwwMap props composition; dangling-edge-hidden-NOT-deleted test: edges()-filters-to-visible-endpoints, re-add-resurrects; props keyed by node id survive remove/re-add vs automerge's key-by-creation-op as an exercise) are `todo!()` stubs — 18 tests fail as todo panics; crdt_bench lane 1 RUN (table above; also caught state-based-sync's quadratic cost live — sync_every=1 ships the whole map per write, workload shrunk and the delta-CRDT motivation written into the bench comment), lanes 2-4 armed behind catch_unwind (OR-Set gossip storm + tombstone census, RGA 50K-char trace + tombstone bloat, graph dangling storm 100-removes∥500-edge-adds + resurrection count); notes.md flags lane 1's honest caveat (counts merge-time discards only — locally-overwritten writes don't show, so it's a lower bound); M31 log: node/edge identity must be Dots not user ids (cr-sqlite's auto-increment-PK trap), dangling policy locked hide-not-delete, props LWW-with-HLC (topic 29), anti-entropy v1 whole-state → v2 db_version-watermark deltas, deliverable = same workload through M15 Raft vs active-active with latency histogram + concrete-conflict table. All 32 topics scaffolded. + +## 2026-07-10 — topic 30 scaffolded + +topic 30 scaffolded: study guide (the-shape-of-the-problem ASCII — regular append-mostly writes vs range+selector+aggregation reads; the baseline finding measured: delta+varint lands at a shape-blind **11.00 B/sample** because raw f64 values dominate — whatever the codec does about timestamps is a rounding error until it attacks the value bytes, hence XOR; Gorilla→Prometheus/VM→IOx lineage mermaid with Monarch and BtrDB as the bracketing extremes; five-system table on codec / time organization / label index / out-of-order policy; TSDB=LSM-keyed-by-time thesis with retention=drop-the-oldest-level), 4 reading guides (Gorilla VLDB '15 + prometheus chunkenc/xor.go — prediction-error framing with the dod/XOR ASCII, paper bucket table vs prometheus's retuned 14/17/20 buckets (:195-208) as buckets-are-workload-parameters, writeVDelta :226 window reuse, :396 tDelta+=dod as the whole model in one line, no-random-access-by-design, entropy-floor bit accounting question; prometheus tsdb — full architecture ASCII head/WAL/2h-blocks/exponential compaction, head_append.go:436/:481/:688-693 as the exact head.rs contract (ErrOutOfOrderSample vs ErrTooOldSample), OutOfOrderTimeWindow head.go:168 quarantine design, MemPostings postings.go:60/:403 = topic 23's inverted index with labels as terms, the two famous failure modes high-cardinality + churn; VictoriaMetrics + InfluxDB 3 paired as two-rebuttals — VM partition.go:75 rawRows→parts LSM-said-out-loud, nearest_delta2.go:15 byte-aligned varint batches with optionally-lossy precisionBits vs Gorilla's exact bits, index_db.go:124 tagFilters cache + churn invalidation, vs IOx WAL→Arrow QueryableBuffer→Parquet-on-object-store (influxdb3_wal/lib.rs:75-98 SnapshotTracker, queryable_buffer.rs:41) as topic 28's landing-zone applied to metrics, vertical-integration-vs-commodity-formats trade table, how-much-of-Gorilla's-win-was-really-sorting question; Monarch VLDB '20 + BtrDB FAST '16 — monitoring-must-not-depend-on-what-it-monitors ⇒ RAM-first lazy durability, push-vs-pull, distribution-typed values as the schema cure for cardinality, query pushdown; BtrDB's aggregate tree — min/mean/max/count per 64-way node ⇒ query cost ∝ pixels not samples, downsampling as the index structure itself, CoW versions), experiments crate compiles: gen (scrape jitter, gauge/counter/constant/random shapes, OOO arrivals, label sets with the unique-instance cardinality bomb) + bits (MSB-first BitWriter/Reader + sign_extend so the stub is the algorithm not bit plumbing) + baseline (zigzag varint delta) PROVIDED — 7 provided tests pass, lane 1 RUN (table above, decode ~270-330 Msamples/s — byte-aligned codecs have a real throughput edge over bit-packed Gorilla, the actual design axis is ratio-vs-decode-speed) — gorilla.rs (paper dod buckets + XOR leading/trailing windows; bit-exact roundtrip incl. bucket edges, constant ≤~2 bits/sample, gauge beats raw 3×, random must FAIL to compress >8 B/sample — the codec wins on regularity not magic), head.rs (in-order fast path + bounded OOO window + TooOld refusal + LWW merge flush that feeds the in-order-only encoder — prometheus semantics exactly), index.rs (MemPostings-style (name,value)→sorted-ids + shortest-list-first k-way intersect, brute-force oracle, cardinality-bomb-counted test) are `todo!()` stubs — 15 tests fail as todo panics; tsdb_bench lanes 2-4 (gorilla ratios per shape, OOO tax sweep 0-50%, selector latency at 100K series) armed behind catch_unwind; notes.md predicts all lanes; M30 log: history chunks per (entity, attribute), `MATCH ... AT TIME t` = latest_write_before ≤ t so M29's MVCC read path generalizes to time-travel, storage split by age (custom hot chunks → M28 Parquet cold), Gorilla dod survives for changelog timestamps but property values need dictionary+RLE not XOR, BtrDB-shaped rollup tree over the M27 changelog for graph-evolution queries. + +## 2026-07-10 — topic 29 scaffolded + +topic 29 scaffolded: study guide (the-problem-priced motivation table — measured conflict probability of the bank workload itself: 0.3% of 8-txn batches collide at zipf θ=0.5 but **29.9% at 0.9, 86.2% at 1.1, 99.6% at 1.3** — contention is the common case at real-workload skew; design-space mermaid rooted at textbook 2PC with the three escapes labeled on the edges — move-the-decision-into-the-data (Percolator), replicate-the-coordinator (Spanner), remove-runtime-agreement (Calvin), decompose+batch (FDB); one-table five-system summary keyed on concurrency control / clock / cross-shard atomicity / blocking window; Percolator-in-six-lines with THE-COMMIT-POINT marked), 4 reading guides (Percolator OSDI '10 + TiKV — three-column-families ASCII (data/lock/write with write-CF-as-commit-index), lifecycle sequenceDiagram with the any-reader-resolves note, TiKV walk actions/prewrite.rs:37 (pessimistic_action + secondary_keys args as post-paper hardening) → commit.rs:64 with the :57 duplicate-commit-returns-Ok idempotency arm → check_txn_status.rs:92/:241 + MissingLockAction :458 as production resolve_lock → cleanup.rs:24 Rollback records our sim skips → latch.rs/scheduler.rs local-vs-distributed conflict split, txn_status_cache; Spanner OSDI '12 + HLC OPODIS '14 paired — bound-the-ERROR-vs-bound-the-SKEW fork diagram, commit-wait derivation, HLC rules + the l≤max-pt anti-Lamport-drift bound, uncertainty-interval restarts, CRDB walk hlc.go:38/:411/:471/:517 (UpdateAndCheckMaxOffset crashes the node — maxOffset is a promise) + txn_coord_sender.go:113 interceptor stack + txn_interceptor_committer.go:128 parallel commits with STAGING-is-implicitly-committed :195-205 as Percolator's-resolve-idea-shaving-a-latency-round; Calvin SIGMOD '12 — agree-on-inputs-not-outcomes diagram, sequencer/scheduler/executor layers, deterministic-locking-kills-both-deadlock-and-2PC, OLLP reconnaissance for dependent txns with the graph-traversals-are-the-ultimate-dependent-txn M29 question; FoundationDB SIGMOD '21 — unbundled roles ASCII, ConflictSet.cpp:947 detectConflicts over the :224 SkipList as the-whole-SI-check-in-one-data-structure, CommitBatchContext :504 batch-is-the-unit, masterserver-is-barely-a-counter, ResolverBug.cpp injectable-wrong-answers as DST culture beyond crash injection, vs-Calvin and vs-Percolator design reads), experiments crate compiles: kv.rs PROVIDED (the three Percolator column families as HashMaps/BTreeMap, strictly-monotonic TSO, latest_write_before range scan, 2-shard cluster, Zipf transfer workload — 4 provided tests pass) — tpc.rs (2PC coordinator with 4-point CrashPoint injection + recovery-from-durable-state-only; blocking_window_demonstrated test names the flaw), percolator.rs (get/prewrite/commit_primary/commit_secondaries/resolve_lock with roll-forward-iff-primary-committed tests, no-lock-leak on failed prewrite, total-conserved-after-rollback), hlc.rs (send/recv rules; monotonic-under-backward-clocks, l-bounded-by-max-pt over 1000 skewed messages, concurrent-events-collide-without-node-id-tiebreak as an assert_eq teaching test) are `todo!()` stubs — 14 tests fail as todo panics; txn_bench lane 1 RUN (conflict table above), lanes 2 (abort rate vs θ + bank invariant) and 3 (20K-txn crash storm, crash every 100th cycling 4 points, blocked_aborts counts the blocking window empirically) armed behind catch_unwind; notes.md predicts lane 2/3 numbers before implementation; M29 log: shard by node id, cross-shard edge = prewrite both adjacency sides with primary on u, supernodes are the Zipf head — per-shard adjacency segments turn the WW hotspot into scatter-gather reads, every protocol step gets a kill point with no-dangling-half-edges as the invariant. + +## 2026-07-10 — topic 28 scaffolded + +topic 28 scaffolded: study guide (the latency ladder measured and priced — local NVMe p50 0.10 ms vs raw S3 p50 14.17 / p99 112.99 ms = 140× median, **940× tail**, and why everyone moved anyway: $/GB, 11-nines durability = replication-is-someone-else's-problem, scale-to-zero; the 2008→Snowflake→Aurora→Socrates→Neon→SlateDB lineage mermaid; Neon's four-box data flow with the safekeepers-commit-fast / pageserver-serves-pages split; five-system design-space table with what-crosses-the-network as the axis; WAL-rule-promoted-to-architecture rosetta linking topic 27's Kafka thesis), 5 reading guides (Aurora SIGMOD '17 — the-log-is-the-database, 6-copy 4/6 AZ+1 quorums over 10 GB protection groups, 35× network amplification killed, VDL-replaces-2PC question, commit=log-quorum-ack + recovery-without-REDO-at-compute; Socrates SIGMOD '19 — durability≠availability as THE decomposition, XLOG landing zone vs page servers vs XStore mapped onto topic 5's WAL lifecycle and onto Neon components, RBPEX = buffer pool made restart-durable; Snowflake SIGMOD '16 + Building-a-DB-on-S3 SIGMOD '08 paired — the prescient paper's three blockers (eventual consistency/no CAS/request cost) vs what fixed each (strong consistency 2020, conditional PUT 2024, immutability routed around all three), micro-partitions as CoW-clone-by-file-list, min/max pruning = topic 26's BRIN at cloud scale; Neon code walk — get_rel_page_at_lsn pgdatadir_mapping.rs:258 → Timeline::get :1227 → LayerMap::search layer_map.rs:448 as an LSM over (page, LSN), walredo.rs:173 = REDO on the read path in a sandboxed Postgres, branch_timeline_impl tenant.rs:4985 O(1) branches + the timeline.rs:4548 ancestor walk our stub reimplements, branch-aware GC retain-point question; SlateDB+Quickwit S3-first code walk — tablestore.rs:835 block-granular ranged GETs, cached_object_store part cache = our cache.rs in production form, **fence.rs:105 CAS-epoch fencing = consensus outsourced to S3 conditional PUT**, clone.rs:38 zero-copy clones, quickwit bundle hotcache footer + TimeoutAndRetryStorage :37 hedging with the AWS-recommends-it citation, pathology→countermeasure convergence table), experiments crate compiles: virtual-latency sim (charged-not-slept lognormal S3 with 2% 8× stragglers, NVMe, scripted Fixed model for exact contract tests) + block store + zipf + percentiles PROVIDED — 6 provided tests pass, tier_bench provided lanes RUN (numbers above) — LruBlockCache+TieredReader (touch-protects, 1/8-cache zipf hit-rate >50%, block-sharing hits), hedged_get (scripted 50ms-primary/1ms-backup/10ms-deadline ⇒ exactly 11ms, p99-halves-at-<10%-extra-GETs on straggler-heavy S3), BranchStore::get (parent-prefix visibility, sibling isolation, PITR historical branch points, 100-deep chains, branching-copies-nothing proven by version_count) are `todo!()` stubs — 13 tests fail as todo panics; notes.md predicts cache-fixes-the-median-hedging-fixes-the-tail and flags the O(n)-eviction-scan wall-time trap; M28 log: L0+WAL stay local (landing-zone lesson), manifest-CAS fencing not leases, branch at SST-list not page granularity, image-layer materialization deferred until ancestor walks profile hot. + +## 2026-07-10 — topic 27 scaffolded + +topic 27 scaffolded: study guide (recompute-is-the-enemy with the priced motivation table — full recompute per 100-change batch: triangles 97.2 ms / wedge self-join 894.3 ms / re-BFS 24.7 ms vs µs-scale incremental targets; the one algebraic idea as a table — LINEAR ops stream deltas statelessly, BILINEAR joins need arranged inputs via Δ(A⋈B)=ΔA⋈B+A⋈ΔB+ΔA⋈ΔB, NONLINEAR distinct/aggregates need integrals; DBSP I→Q→D mermaid; timestamps/watermarks section: timely frontiers are proofs where Flink watermarks are heuristics; four-system comparison timely/DBSP/Materialize/RisingWave), 5 reading guides (Naiad+timely — could-result-in pointstamp protocol, MutableAntichain frontier.rs:380/update_iter :533, ChangeBatch :16 progress-updates-are-Z-set-shaped, worker.rs:235 step as the topic-7 event loop one layer up, the rosetta table frontier=vacuum-watermark; differential — consolidation.rs:24 = our from_updates verbatim, arrangements as LSM-of-batches with advance=compaction, join_traces join.rs:69 with the fuel/effort loop :348-395 as operator-level cooperative yielding, iterate.rs:192 Variable + bfs.rs:101-107 as the 40 lines our stub can't do, why deletion-in-recursion needs lattice times; DBSP — Q^Δ=D∘Q∘I with the chain rule as the compositional bombshell, feldera anchors z1.rs:221/integrate.rs:85/differentiate.rs:38/join.rs:123-350/delta0.rs, nested-circuits-vs-lattice trade, the M27 mapping: delta matrix DP−DM=ΔA and wedges need NO new state because the integrals ARE the adjacency matrices; Materialize+RisingWave — dogs^3 delta_join.rs:47 + half_join :315/:402 avoiding intermediate arrangements, indexes-are-arrangements-are-memory, RisingWave Op enum stream_chunk.rs:45 as Z-set-weights-as-protocol, hash_join.rs:158 degree tables :269 as hand-rolled weight bookkeeping vs one consolidation rule, barrier checkpoints, single-writer-gets-the-hard-parts-free table; Kafka NetDB'11 — dumb-broker/smart-consumer, offset=LSN rosetta, log-compaction=arrangement-advance=LSM-GC same operation three communities, exactly-once = where-do-offsets-live, M27's raw-log-vs-result-delta subscriber decision), experiments crate compiles: ZSet with consolidation + the distinct-is-not-linear load-bearing test, churn generator with set-semantics guard, full-recompute oracles (sorted-intersect triangles, hash join with weight multiplication, BFS) PROVIDED — 6 provided tests pass, ivm_bench baselines RUN (numbers above) — delta_join+IncrementalJoin (algebra-exact vs join(A+ΔA,B+ΔB)−join(A,B), 30-batch drift-free, deletes-retract), IncrementalTriangles (oracle-tracking under churn, K4-minus-edge=−2, <4K probes for 20 changes on 40K edges), SemiNaiveReach (insert-only BY DESIGN — deletion is differential's lattice territory, documented; matches re-BFS per batch, ≤4 relaxations/edge EVER, intra-component edges free) are `todo!()` stubs — 9 tests fail as todo panics; notes.md flags the honest suspicion that IncrementalJoin's Vec-merge state integration may dominate and re-derive why arrangements exist. + +## 2026-07-10 — topic 26 scaffolded + +topic 26 scaffolded: study guide (indexes-are-bets framing with the measured motivation table — point-miss binary search 167 ns / BTreeMap 218 / HashSet 24 at 224 MB, vs blocked bloom's ~15-25 ns at 12 MB target; three-families ASCII filters/sketches/learned; bloom math block with the reproduce-it FPR derivation; bloom→blocked→cuckoo→xor→ribbon lineage mermaid; HLL sparse-30-bytes-to-dense-12KB story; PGM ε-window thesis), 6 reading guides (bloom→ribbon over RocksDB code — FastLocalBloomImpl bloom_impl.h:144 golden-ratio probe remix vs LegacyBloomImpl :364, CacheLocalFpRate :42 as the Poisson-crowding honesty function, ribbon as banded GF(2) solve with StandardBanding ribbon_impl.h:471 / num_starts_=slots−kCoeffBits+1 :504 and the build-can-fail-vs-monotone question; cuckoo+xor — partial-key involution getAltHash cuckoo.c:122, RedisBloom's LSM-of-subfilters growth CuckooFilter_InsertFP :256 vs the paper's fail-at-MAX_KICKS, delete-a-false-positive-corrupts-someone-else contract, xor peeling 1.23× vs bloom 1.44× vs ribbon 1.10×; HLL — hllPatLen :467 / Ertl tau+sigma :1016/:1033 replacing HLL++'s empirical bias tables, ZERO/XZERO/VAL sparse opcodes :380 with promote-at-3KB-or-rank>32, merge-is-a-semilattice AVX2 :1116; learned indexes — Kraska RMI's no-error-bound flaw, PGM_SUB/ADD_EPS pgm_index.hpp:32-33 + optimal hull PLA piecewise_linear_model.hpp:96/:154-190 vs our simpler shrinking cone, ALEX gapped arrays predict_position alex_nodes.h:1448 + exponential search :1462 with the degrades-in-space-vs-time-vs-write-amp scoreboard; roaring internals extending topic 23 — Store enum store/mod.rs:28-31, ARRAY_LIMIT=4096/RUN_MAX_SIZE=2048 as pure arithmetic container.rs:9-11, 3×3 pairwise kernel dispatch, three-adaptive-encodings table roaring/HLL-sparse/GIN-varbyte; postgres indexam as the classical baseline — _bt_search nbtsearch.c:100 + Lehman-Yao moveright :211, GIN varbyte ginCompressPostingList ginpostinglist.c:196 + pending-list-as-mini-LSM, BRIN bringetbitmap brin.c:301 as the one-sided filter that's 10,000× smaller than bloom when clustering holds), experiments crate compiles: splitmix64/hash2/fastrange PROVIDED (avalanche + coverage tests pass), filter_bench motivation lanes RUN — **binary search miss 167 ns ≈ 23 dependent misses, BTreeMap 218, HashSet 24 ns at 224 MB, hit≈miss 169** (the walk is the cost, not the compare) — BlockedBloom (no-FN + FPR<2.5%@10bpk + <4×-theory + halves-8→16 tests), CuckooFilter (no-FN at 90% load, FPR<1%@12-bit, delete-leaves-others-intact — the test bloom can never pass, graceful-full-failure), Hll (err<3% at 1K/100K/5M, merge registers EXACTLY equal union's), and LearnedIndex (window ≤2ε+2 always contains true pos, uniform-1M <2K segments, ε holds on hostile powers-of-2+quadratic mix) are `todo!()` stubs — 15 contract tests fail as todo panics, notes.md has predict-before-measure table and the M26 call that learned indexes are NOT in scope (node IDs are dense — a plain array is already the perfect model). + +## 2026-07-10 — topic 25 scaffolded + +topic 25 scaffolded: study guide (message-passing-is-SpMM with receipts — the message_and_aggregate table showing GCNConv = `spmm(adj_t, x)` at gcn_conv.py:273 and SAGEConv = `spmm(..., reduce=mean)` while GAT can't fuse because attention recomputes the matrix values per forward; associativity-as-query-plan — (AX)W vs A(XW) swaps which term carries the big dimension, 90× on Cora; GraphRAG closed-loop mermaid graph→embeddings→M14 vector index→hybrid Cypher), 7 reading guides (node2vec KDD '16 — second-order walk figure, p/q as BFS↔DFS knobs, the alias-table O(m·avg_deg) memory trap with rejection sampling as the fix, PyG Node2Vec.loss node2vec.py:135 as the SGNS reference; Kipf-Welling GCN — renormalization trick, gcn_norm anchors gcn_conv.py:45-71, GCN-forward-is-a-query thesis, oversmoothing as why-2-layers; GraphSAGE — sampling as a page budget (B·10·25 fan-out math), mean+lin_r≈concat, inductive-is-the-only-write-friendly-variant; GAT — SDDMM+softmax+SpMM kernel decomposition = topic 24's masked SpGEMM, a_src/a_dst per-node split as factor-out-of-join, materialized-vs-computed view line running exactly between GCN and GAT; PyG message-passing machinery — 10-stop code walk, COO gather-scatter materializes an m×d temp (145 MB on our bench) vs CSR spmm's zero temporaries = materialize-the-join vs pipeline-the-aggregate, message()-as-arbitrary-callable = Ligra's F-with-CAS tradeoff third community; TransE — relations as translations, symmetric-relation collapse, link-prediction-is-an-ANN-query so M14 serves KG completion natively; GraphRAG-SDK with systems eyes — vector_store.py:344 queryNodes / :219 SET embedding write path, relationship_expansion's ANN+k-MATCHes as the k+1-round-trip join to push down, multi_path's client-side cosine rerank, router-as-planner-with-no-cost-model, four systems smells), experiments crate compiles: CSR + SBM generator (ground-truth labels; O(m) inter-block sampling not O(n²) Bernoulli) + ring-of-cliques + dense Mat/glorot/softmax + SpMM + row-norm adjacency + uniform walks + dense GCN oracle PROVIDED and run — SBM 16,384 vertices/566K edges: uniform walks 42.8 Msteps/s, **SpMM 21.2 GFLOP/s = 81% of dense matmul's 26.2** (64-wide feature rows amortize the gather — fat RHS forgives sparsity, the number that makes M25 plausible) — node2vec_walks (4 tests: degree-stationary dist, p=q=1 ≡ uniform, q orders exploration on ring-of-cliques, p orders backtrack rate), train_skipgram (SBM intra-block cosine must beat inter by 0.2), gcn_norm+gcn_forward (dense-oracle 1e-4, sorted rows, transform-before-aggregate) are `todo!()` stubs. + +## 2026-07-10 — topic 24 scaffolded + +topic 24 scaffolded: study guide (per-source vs whole-graph algorithm map; frontier-world vs algebraic-world mermaid with the honest trade — per-vertex tricks like Afforest's edge skipping vs LAGraph's batched matrix frontiers and atomics-free bulk ops; rmat-vs-uniform baseline table making skew the headline), 6 reading guides (GAP arXiv:1508.03619 + gapbs anchors — sssp.cc:44's redundant-relaxation-beats-bookkeeping bet, bc.cc:76 succ bitmap, cc.cc:69/:106/:129 Afforest's three phases, tc.cc WorthRelabelling, and the 5-graph matrix as topic 22's change-anything-different-number; Meyer-Sanders delta-stepping as the Dijkstra↔Bellman-Ford dial with gapbs thread-local-bins vs LAGr_SSSP's MIN_PLUS tmasked vxm + three implementation traps; Brandes '01 with the dependency-recurrence derivation exercise + gapbs-vs-LAGr_Betweenness table — the batched ns×n matrix frontier amortizes what frontier code cannot; Ligra PPoPP '13 — edgeMapData ligra.h:235-272, the m/20 threshold :238, dense/sparse/denseForward, PageRank-degenerates-to-SpMV lesson, m/20 = Beamer α/β = dot-vs-saxpy; Louvain→Leiden Sci Rep '19 — the disconnected-communities bug as topic 21's greedy-destructive trap, refinement as egg's keep-both-forms, ΔQ accumulator = SPA, aggregation = S·A·Sᵀ SpGEMM, determinism-needs-seeding for CALL algo.community; LAGraph analytics — FastSV7 mngp/hooking-as-one-mxv :102 + FASTSV_SAMPLES :335, TriangleCount's six formulations with the :44 urand-flips-to-saxpy exception, PageRankGAP-vs-PageRank as benchmark-specs-fork-implementations, and FalkorDB's proc_pagerank.c:197 already calling LAGr_PageRank = M24's pattern exists, re-plumb it), experiments crate compiles: weighted CSR + RMAT/uniform generators + heap Dijkstra with pop counter + pull PageRank + degree-ordered triangle count + union-find CC + O(n³) definitional BC oracle PROVIDED and run — RMAT scale 16 vs uniform same n/m: **15,645,988 vs 5,428 triangles (2,883×)** in 376/158 ms, PR 8-vs-6 iters (hubs slow L1 decay), Dijkstra 343K pops = 1.74×n stale-entry tax, 18,844 components at avg_deg 16 (RMAT's leaf quadrant strands vertices) — delta_stepping (bucketed, extremes-must-still-be-exact test + relaxation counters), brandes (must match the O(n³) oracle exactly on n=128, then sample GAP-style), and afforest (partition-equal + <50%-of-m edges-inspected bound) are `todo!()` stubs; RMAT skew assertion needed scale-aware bounds (19.1% top-1% share at scale 12, 36.6% at 16). + +## 2026-07-10 — topic 23 scaffolded + +topic 23 scaffolded: study guide (inverted-index anatomy ASCII — analyzer → FST term dict → TermInfo{doc_freq, postings_range} → 128-doc Δ-bitpacked blocks with {last_doc, max_score} skip data; write-path mermaid making Lucene-segments-are-an-LSM explicit — tiered LogMergePolicy works for text because queries fan out anyway; the two speed tricks: compression-with-random-access + score upper bounds), 6 reading guides + 2 cross-linked (Zobel-Moffat CSUR '06 design-space map — TAAT vs DAAT, capped accumulators as 2006's WAND, merge-based construction = LSM before Lucene; Robertson-Zaragoza BM25 derivation ladder — eliteness ⇒ tf saturation at K1+1 ⇒ the static ceiling WAND needs, mapped to tantivy bm25.rs:8-59 with the 1-byte-fieldnorm quantization question; Ding-Suel SIGIR '11 block-max WAND with pivot diagram + four implementation traps (θ seeding, livelock-on-failed-refinement, k-boundary ties, docs-evaluated metric); Roaring — 4096 crossover derivation, kernel matrix, containers = GraphBLAS sparse↔bitmap lattice at 64K granularity; tantivy code walk — compression/mod.rs:3 128-blocks, skip.rs:93/:175/:186 SkipReader/block_max_score, term_info.rs:9-13, fst_termdict, block_wand_union.rs:8-24 find_pivot_doc, log_merge_policy.rs:20-24, 90-minute read order; RediSearch redisearch_rs Rust-rewrite — newly cloned — InvertedIndex core.rs:30/:75 chained varint IndexBlocks vs tantivy's immutable bitpacked segments, Encoder-as-type-parameter monomorphizing 11 codecs, gc_marker/unique_id cursor validation ↔ delta-matrix wait, new-block-on-delta-overflow, and the finding that RediSearch has NO block-max WAND — scored unions walk everything), experiments crate compiles: zipf corpus (term id = rank, df(t0)=99.9%) + tf-counting index builder with per-128-block max-BM25 metadata + saturating-BM25 + exhaustive TAAT oracle PROVIDED and run — 100K docs/7.9M postings built in 335 ms, common∧rare [t0 t12000] top-10 walks 99,964 postings in 6.34 ms at ~32 ns/posting (hash accumulate dominates — Q1's lesson again) while the rare term carries ~93% of the winning score = the WAND poster child measured, and vec two-pointer dense∧sparse AND costs O(|dense|) 52 µs for 172 hits = the roaring motivation measured — block-max `wand_topk` (recipe in doc comment, must match oracle top-k while scoring <25% of the postings) and mini-Roaring (array/bitmap containers, 3 density-crossing oracle tests) are `todo!()` stubs. + +## 2026-07-10 — topic 22 scaffolded + +topic 22 scaffolded: study guide (OLTP↔OLAP benchmark map + the number-is-four-choices mermaid workload/data/harness/metric; choke points one-liner table — Q1 tiny-group agg = expression bench, Q6 2%-scan = the GB/s headline, Q9 = optimizer punisher, TPC-C = the D_NEXT_O_ID hot counter institutionalized; benchmarking-sins checklist linking topic 0's fair-benchmarking guide), 4 new reading guides + 3 cross-linked existing ones (Boncz TPCTC '13 choke-point taxonomy with duckdb dbgen/queries dir open + hidden messages — uniform data is why JOB exists, Q1's 4-6 groups make hash-agg invisible; YCSB SoCC '10 + go-ycsb zipfian.go:92-165 anchors — zetan/eta/alpha math, the two fast paths, scrambled-fnv rationale, coordinated-omission warning; OLTP-Bench VLDB '13 + benchbase TPCC anchors — keying/think times :85-100, NURand C_LAST load-vs-run constants :94-116, why nobody runs TPC-C honestly = 12.86 tpmC/warehouse; DuckDB tpch extension — dbgen as streaming TABLE FUNCTION tpch_extension.cpp:17-99 with answers/ shipped next to queries/ = benchmark-as-oracle, run-real-TPC-H-here recipe), experiments crate compiles: dbgen-lite lineitem + row-at-a-time Q1/Q6 oracles + YCSB A-F driver over BTreeMap with ns-percentile Hist PROVIDED and run — Q1 oracle 5.6 GB/s effective (HashMap per row even at 6 groups: CP1.2 measured), Q6 branchy oracle 15.7 GB/s (2% selectivity = perfectly-predicted branch, the crater is hiding at 50%), YCSB uniform A-F 2.88/4.15/3.72/4.40/1.11/2.85 Mops/s with E's scans 4× a point read — Zipfian/Scrambled generators (the actual YCSB math with head-frequency-vs-theory statistical contract tests) and q1_flat/q6_branchless columnar lanes are `todo!()` stubs; fixed a nearest-rank percentile off-by-one in the harness itself (harness bugs are results bugs). + +## 2026-07-10 — topic 21 scaffolded + +topic 21 scaffolded: study guide (tool-per-guarantee table proptest→TLC→SMT→Lean with cost axis; e-graph = union-find + hashcons + congruence closure with egg's deferred rebuild = delta-matrix wait = LSM compaction — batch the invariant repair; equality-saturation loop mermaid; TLA+ spec-as-math with the measured state counts; Z3 DPLL(T) diagram + the euf_egraph.h:23 comment where Z3 cites egg back; Perceus RC as the Arc::make_mut compiler pass), 5 reading guides (AWS CACM '15 — 35-step S3 bug, exhaustively-testable-pseudo-code pitch, small-scope hypothesis; egg POPL '21 with full source anchors — egraph.rs:970 add / :1147 union / :1416 rebuild / :1346 process_unions fixpoint, machine.rs Bind/Scan/Compare pattern VM = topic 19's bytecode interpreter, extract.rs greedy find_best vs lp_extract, e-graph ≈ Cascades memo; Z3 TACAS '08 + src/ast/euf anchors — backtracking trail + justifications = WAL for unions, e-matching triggers = index choice; Specifying Systems + Ongaro's raft.tla — newly cloned — with the un-model-an-assumption exercise that re-derives terms; Beans + Perceus borrowed-vs-owned + reuse tokens with the proof-vs-TLC-vs-proptest calibration exercise), experiments crate compiles on egg 0.9: expr IR + hand-ordered fixpoint rewriter PROVIDED and run — ~30% cost reduction, ~2 µs/firing, and the planted trap measured: (a*2)/2 → strength-reduce fires before div-reassoc → stuck at (a<<1)/2 cost 5 where egg should reach cost 1 — egg_optimize is a `todo!()` stub with trap + never-worse-than-hand tests; **TLA+ WalReplication spec written AND model-checked** (tla2tools.jar downloaded, java 17): SyncCommit=TRUE → Durability holds over 1080 distinct states depth 14 in <1 s; SyncCommit=FALSE → TLC finds the 5-state data-loss trace Append→Commit→Crash→Failover after 123 states — the postgres synchronous_commit=off story, found exhaustively. + +## 2026-07-10 — topic 20 scaffolded + +topic 20 scaffolded: study guide (format lattice hypersparse→sparse→bitmap→full with the actual switch tests from GB_convert_sparse_to_bitmap_test.c and GB_conform; one-mxm-four-engines mermaid — dot3 iterates the MASK so work ∝ nnz(M) vs saxpy3's coarse/fine × Gustavson/hash task scheduler with its flopcount pre-pass = cudf size/retrieve five years early; push-vs-pull BFS as vxm-vs-mxv with LAGraph's shipped α=8/β1=8/β2=512; delta matrices as LSM-over-matrices — DP=memtable, DM=tombstones, wait=minor compaction, delta_mxm's `(A*(M+DP))` fold), 6 reading guides (Davis TOMS '19+'23 — zombies/pending as the library's own deltas, iso matrices, 32-bit indices; SuiteSparse internals with saxpy3.c:22-60 scheduling-essay and hash>m/16⇒Gustavson anchors; Gustavson '78 + Buluç-Gilbert — SPA design space, symbolic/numeric two-phase; Beamer SC '12 direction-optimizing with the ICPP '18 linear-algebra translation; LAGraph — BFS template switch block anchors, ANY_SECONDI parent-without-comparisons, six triangle-count formulations, PageRankGAP; FalkorDB delta_matrix with fresh eyes — header state-table as spec, transposed twin, transpose-as-masked-copy sync, over-masking question — LAGraph newly cloned), experiments crate compiles: CSR+RMAT/uniform/path generators, SpMV, hash-SpGEMM, scalar BFS, hypersparse PROVIDED and run — SpMV 16-19 GB/s single-thread (gather tax vs 30 GB/s streaming), hash SpGEMM ~60-75 Mflop/s (15 ns/flop = the accumulator cost the SPA stub should crush), **hypersparse 50× index memory and 171× full-sweep** on the 10M-nodes/100K-edges FalkorDB shape — dense-SPA Gustavson and push/pull/direction-optimizing BFS (with per-level trace + path-graph-never-pulls test) are `todo!()` stubs. + +## 2026-07-10 — topic 19 scaffolded + +topic 19 scaffolded: study guide (the spectrum tree-walker → bytecode VM → copy-and-patch → IR JIT → LLVM as a compile-latency-vs-run-speed trade with each system placed on it; produce/consume compiles the PIPELINE not the operators — push inverts control so tuples stay in registers; three JIT grains compared — postgres per-query, Umbra per-pipeline with adaptive Flying-Start→LLVM tiering, GraphBLAS per-kernel-specialization cached forever; DuckDB's deliberate no-JIT with the VLDB '18 tie as the counter-argument; M19 = expressions only, gate on measured cost), 6 reading guides (Neumann VLDB '11 pipelines/breakers + produce-consume inversion; SQLite VDBE — vdbe.c:1049 switch over 199 opcodes, register machine, OP_Yield coroutines as flattened-bytecode's free resumability; Umbra Tidy Tuples single-pass IR + copy-and-patch musttail stencils; postgres llvmjit_expr.c opblocks-per-EEOP-step + the jit_above_cost estimate-gate failure taxonomy + deform-JIT-as-the-real-win; GraphBLAS jitifyer encodify-hash → PreJIT → memory table → dlopen → invoke-cc ladder with FalkorDB cache-warming implications; cranelift-jit-demo declare→define→finalize→transmute ladder + per-node CLIF emission table for the stub), experiments crate compiles on cranelift 0.116: Expr enum + seeded generator, AST interpreter, and column-at-a-time vectorized lane PROVIDED and run — interp ~2.1 ns/node flat, vectorized 6-12× over interp across depths 2-10 (topic 11's number reproduced; both linear in nodes, no y-intercept until compile time adds one) — jit.rs compile() is a `todo!()` stub with bit-exact-vs-interpreter tests, jit_bench prints the three-way table with compile µs + e2e winner and survives the stub via catch_unwind. + +## 2026-07-10 — topic 18 scaffolded + +topic 18 scaffolded: study guide (GPU-for-DB-people translation table — SIMT = topic 17's predication in hardware, coalescing = columnar × 32, shared memory = cache blocking made explicit; the bus decides the architecture — Crystal's regime A ship-per-query vs regime B device-resident, rewritten for Apple unified memory; libcudf size/retrieve two-phase + cooperative-groups probing = SwissTable at warp scale; Gunrock advance/filter + load-balance menu; CAGRA = HNSW with SIMT-hostile parts deleted by construction), 6 reading guides (Crystal SIGMOD '20 tile model + fair-CPU-baseline lesson; wgpu compute examples ladder incl. the hello_compute doc-comment our bench proves; libcudf join size/retrieve + shared-mem-until-spill groupby with cuco/cooperative-groups anchors — newly cloned; Gunrock Essentials bfs.hxx enactor + advance.hxx thread/block/merge_path dispatch anchors — newly cloned; CAGRA ICDE '24 + single-CTA kernel/shared-mem-hashmap/bitonic-topk anchors — cuvs newly cloned; Faiss GPU billion-scale paper — WarpSelect register k-select, memory-tier table), experiments crate compiles AND RUNS on Metal: GpuCtx + workgroup-reduction sum kernel PROVIDED with per-phase timings, gpu_bench crossover sweep run on Apple M3 Pro — **CPU wins at every size up to 16M elements** (~1.5 ms fixed dispatch floor, flat 16K→1M; even amortized the GPU reads the same unified memory at ~9 GB/s effective vs CPU 30 GB/s — regime B's bandwidth ratio doesn't exist for streaming ops on this machine, which IS the lesson), filter_count (one-atomicAdd-per-workgroup, WGSL skeleton provided) and l2_batch (one-invocation-per-target + row/column-major coalescing experiment) are `todo!()` stubs with exact-match/1e-3 tests, notes.md predicts where arithmetic intensity finally flips the verdict. + +## 2026-07-10 — topic 17 scaffolded + +topic 17 scaffolded: study guide (ports×latency mental model — M-series 4 FMA ports × 3cy ⇒ ~12 independent chains, the four autovectorization failures, branchy/branchless/compress filter shapes with AVX-512 vpcompress vs NEON's missing compress, vshrn movemask idiom, FastLanes interleaved layout), 7 reading guides (simdjson VLDB '19 + arm64 nibble-LUT classification / PMULL prefix_xor quote parity / LUT-shuffle compress emulation / flatten_bits over-write-under-advance — newly cloned; polars-compute float_sum STRIPE=16 + pairwise-128 and simd_filter! with per-ISA compress + selectivity-adaptive scalar bit-iteration fallback; hashbrown Group×3 backends + memchr Vector — newly cloned — with the finding that hashbrown's NEON group is 8 BYTES so vceq output already IS the bitmask, no vshrn, while memchr keeps 16 lanes and narrows; SimSIMD under its numkong rename — per-instruction latency/port tables in headers, f64-upcast accumulation, 4-target streaming states = M14's scoring loop, and the FCMLA lesson: the specialized instruction measured 2.3× SLOWER than 4 plain FMAs; SIGMOD '15 selective-store/load primitives + vertical hash probing + gather-costs-a-load-per-lane; FastLanes VLDB '23 1024-lane transposed layout; Mojo SIMD[type,width] parametric-width ladder), experiments crate compiles: dot naive+unrolled-8 PROVIDED and run — 10.89 → 42.12 GB/s, 3.9× from accumulator count alone, zero intrinsics — wide-f32x4 and NEON vfmaq 4-accumulator rungs are `todo!()` stubs; filter branchy+branchless PROVIDED and swept — branchy craters 9× to 1.19 GB/s at 50% selectivity while branchless holds ~12.7 flat, the SIGMOD '15 curve live — NEON count (vcltq+vsubq mask-accumulate) and LUT-compress compact (simdjson trick, f32 edition, all-16-masks test) are stubs; unpack4 scalar PROVIDED at 10.20 GB/s, NEON shift/mask stub; simd_bench catches stub panics so baselines always print. + +## 2026-07-10 — topic 16 scaffolded + +topic 16 scaffolded: study guide (every technique = generator + oracle table, DST determinism boundary diagram, PQS/TLP/NoREC comparison, Jepsen/elle, Z3-as-search-engine), 6 reading guides (turso testing/simulator with clock/io/file fault-injection anchors + interaction-plan properties + doublecheck + structured fuzz targets; FDB simulation + BUGGIFY + Antithesis determinism-boundary table; SQLancer oracle base classes — newly cloned — PQS check()/rectification, TLP 3-way partition, NoREC optimized-vs-forced-scan; PQS OSDI '20 + TLP OOPSLA '20 paired; Jepsen redis-raft/Dgraph findings + elle cycle inference; Z3 — newly cloned — TACAS '08 + tactic/solver/smt_context anchors + Cosette symbolic-row rewrite verification), experiments crate compiles: sim_fs (buffered/synced/torn-tail file) + kv (WAL KV with 4 injectable bugs: LostDelete/NoSyncOnCommit/TornWriteAccepted/StaleRead) PROVIDED, dst harness + ddmin shrinker + TLP Kleene-eval checker are `todo!()` stubs with 12 contract tests (all bugs caught ≤200 seeds, zero false positives over 500, deterministic replay, 1-minimal repro, null-blind engine exposed), crash_matrix PROVIDED and run: 5000 seeds/0.02 s per bug — 0.0% false positives, bugs caught at 48.8–99.6% per-seed rates, first failing seed ≤ 3; the matrix caught a real bug in this crate's own recovery (missing WAL tail truncation made torn leftovers join the next batch — 72.7% divergence until fixed), the topic's thesis self-demonstrated. + +## 2026-07-10 — topic 15 scaffolded + +topic 15 scaffolded: study guide (topology menu as WHO-can-ack axis, Raft state-machine mermaid + election/log-matching/§5.4.2 three-way split, write-path comparison valkey/WAIT/raft, consistency ladder + ReadIndex, hash slots vs ranges), 6 reading guides (Raft ATC '14 with Fig 8 worked by hand; valkey replication.c shared repl buffer + PSYNC replid/offset + REPL_STATE_ handshake + WAIT + FAILOVER with line anchors; tikv raft-rs — newly cloned — RawNode/Ready contract + step_* dispatch + Progress tracking + maybe_commit-is-§5.4.2; qdrant consensus.rs raft-for-metadata-only split with the data path outside raft; VSR Revisited round-robin views + no-disk durability + TigerBeetle disk-can-lie; DDIA ch. 5/8/9 anomaly catalog + fencing tokens + linearizability), experiments crate compiles: sim.rs deterministic lockstep network PROVIDED (seeded delivery, partition/heal — topic 16 DST preview), raft.rs `todo!()` stub with 5 safety-pinning tests (one leader, one-per-term across 10 seeds, replicate-to-all, minority-commit-freeze, stale-leader truncation), partition_test timeline binary, repl_lag PROVIDED and run: follower fsync policy AS ack latency — every-1 = 339 entries/s at 2973 µs p50 (F_FULLFSYNC) vs never = 18568/s at 6 µs, the topic 5 ladder measured as replication lag. + +## 2026-07-10 — topic 14 scaffolded + +topic 14 scaffolded: study guide (recall-vs-QPS curve framing, HNSW-as-skip-list ASCII, quantization ladder u8/PQ/binary with oversample+rescore, IVF + DiskANN families, filtered-search menu with percolation), 6 reading guides (qdrant GraphLayers builder/serve split + visited pool + the per-query algorithm choice HNSW/ACORN/plain via estimate_cardinality + measured percolation at build; qdrant quantization crate u8 affine dot-expansion / PQ ADC LUTs / binary xor_popcnt + get_oversampled_top; usearch — newly cloned — node-tape layout + paper-default constants + striped locks (helix-db dropped: public repo no longer ships engine source); HNSW paper with skip-list lens; Jégou PQ with SDC/ADC + IVFADC residuals-as-FOR; DiskANN Vamana robust-prune α-slack + PQ-steers/f32-ranks SSD layout), experiments crate compiles: brute-force oracle PROVIDED and run (185 QPS at recall 1.0 over 100K×128-d — the floor), hnsw (Alg 1/2/4, level draw, ef knob) and quant (affine u8 + symmetric distance + rescore pipeline) are `todo!()` stubs with 10 contract tests (self-query top-1, recall@10 ≥ 0.9, sorted results, log level distribution, α/2 error bound, rescored recall ≥ 0.95), ann_bench sweeps ef 16..256 + oversampling 1/2/4. + +## 2026-07-10 — topic 13 scaffolded + +topic 13 scaffolded: study guide (adjacency representation menu with CSR ASCII, four-architecture table neo4j/memgraph/kuzu/FalkorDB across store/Expand/pattern-match/MVCC/updates, delta-overlay-as-LSM observation, pointer-chasing cost analysis, WCOJ/AGM section, LDBC referee), 6 reading guides (GraphBLAS 4 sparsity formats + dot-vs-saxpy mxm + masks-as-pushdown + FalkorDB Delta_Matrix M/DP/DM state machine — neo4j/kuzu/GraphBLAS newly cloned; neo4j 15 B node / 34 B rel fixed records + doubly-linked rel chains = one miss per edge; memgraph skip-list vertex + small_vector edges + PointerPack'd delta MVCC; kuzu columnar CSR node groups persistent+transient + Intersect WCOJ + factorization; AGM bound / Generic Join / EmptyHeaded with the `C=A²` equivalence; LDBC SNB correlated-power-law datagen + updates-during-reads), experiments crate compiles: adj_list oracle PROVIDED and run (1M-node/16M-edge preferential-attachment: 3.5 µs/query random vs 295 µs supernodes — the 85× graph-shaped tail, max degree 6565), csr (counting-sort build + slice two_hop) and matrix (masked-SpMV two_hop) are `todo!()` stubs with oracle-agreement + exact-layout + cycle-self-exclusion tests, hop_bench cross-checks via checksums. + +## 2026-07-10 — topic 12 scaffolded + +topic 12 scaffolded: study guide (row-vs-column ASCII, lightweight-encoding zoo table incl. FSST, analyze→score→compress lifecycle, zone-map pruning diagram, Arrow-vs-Parquet boundary, MergeTree/DuckDB/Pinot architecture table), 6 reading guides (DuckDB compression framework + 4-mode bitpacking + fetch_row-shapes-the-menu + CheckZonemap; ClickHouse MergeTree — newly cloned — parts/granules/sparse-index/marks two-offset trick + merge-time work; arrow-rs + parquet-rs — newly cloned — buffer recipes, RLE-hybrid, two compression layers; C-Store + SIGMOD '06 process-compressed thesis; BtrBlocks sampling cascade + FSST symbol tables; ClickHouse VLDB '24 with ClickBench-on-DuckDB exercise), experiments crate compiles: RLE/Dict/BitPacked `todo!()` stubs with exact-size + maximal-runs + FOR-width + O(1) random-access contract tests, scan_bench PROVIDED (100M values × 3 shapes, raw vs encoded scans incl. RLE sum-without-decode and dict codes-only sum — "raw-equiv GB/s > memory bandwidth" is the compression-IS-performance headline to verify). + +## 2026-07-10 — topic 11 scaffolded + +topic 11 scaffolded: study guide (Volcano→X100→HyPer mermaid, selection vectors + vector-type flags, morsel-driven parallelism diagram, vectorized hash join/agg internals), 6 reading guides (DuckDB DataChunk/2048 + pipeline executor push-pull hybrid + join-HT salt-in-pointer probe; postgres ExecProcNode self-replacing dispatch + execExprInterp computed-goto; polars-stream Morsel/MorselSeq/SourceToken + float_sum masked-SIMD multi-accumulator + DataFusion ExecutionPlan streams and intern-then-flat-arrays GroupedHashAggregateStream; X100 CIDR'05 U-curve; VLDB'18 compiled-vs-vectorized scorecard — memory-bound probes favor vectorized, the M11 architecture argument; SIGMOD'14 morsels), experiments crate compiles: one query three engines — Volcano PROVIDED and run (180.7 M rows/s; found LLVM DEVIRTUALIZING the statically-known `Box` chain, 202→180 after black_box — a compiler will silently turn your Volcano into a compiled engine), vectorized (batches + selection vectors + flat group array) and fused branchless kernel are `todo!()` stubs with oracle-agreement tests incl. partial-final-batch and mask-sign-extension traps, exec_bench sweeps selectivity 5/50/95. + +## 2026-07-10 — topic 10 scaffolded + +topic 10 scaffolded: study guide (parse→bind→logical→rewrite→join-order→physical pipeline mermaid, rewrite-rule menu, Selinger DP vs DuckDB DPccp+greedy-fallback, cardinality three-lies table, Selinger-vs-Cascades memo ASCII), 5 reading guides (DuckDB optimizer.cpp 25-pass pipeline + plan_enumerator DPccp with greedy escape hatch :234 + cost=output-cardinality-only; postgres allpaths.c standard_join_search + geqo threshold 12 + DEFAULT_EQ_SEL 0.005; sqlparser-rs Pratt parse_subexpr + DataFusion fixpoint-of-rules vs DuckDB ordered passes — sqlparser/datafusion/polars newly cloned; Selinger '79 vs Cascades with M10 architecture-choice question; Leis VLDB'15 JOB — cardinality error 10²–10⁴ dwarfs cost model 2× and search 1.2×, graph-JOB design exercise), experiments crate compiles: toy cost-based planner `todo!()` stubs (parse_and_plan naive left-deep → push_down → greedy reorder_joins → estimate with 1/NDV + independence + containment) with contract tests incl. join_order_flips_with_stats, explain binary PROVIDED for side-by-side DuckDB EXPLAIN comparison. + +## 2026-07-10 — topic 9 scaffolded + +topic 9 scaffolded: study guide (latch vs lock table, memory-ordering cheat sheet + publication idiom, latch-coupling→OLC→lock-free ladder, epoch reclamation diagram, Bw-tree cautionary arc, false sharing), 4 reading guides (postgres lwlock.c packed u32 + recheck-after-enqueue lost-wakeup dance; crossbeam-epoch pin/defer/try_advance — newly cloned; RocksDB InlineSkipList CAS+splices vs memgraph lazy-locking skiplist with accessor-id GC — memgraph newly cloned; Bw-tree ICDE'13 + SIGMOD'18 reality check + Leis OLC), experiments crate compiles: lock-free ConcurrentSet `todo!()` stub over crossbeam-epoch with 5 contract tests (same-key/remove races exactly-one-winner, reader-survives-removal-churn UAF canary), scaling shootout PROVIDED (global mutex / 16-shard / crossbeam SkipSet / yours, 1→16 threads), false_sharing PROVIDED and run — packed 63 M inc/s vs pad128 3707 M (59×), and pad64 still 2.2× slower than pad128: Apple M-series coherence granularity is 128 B, x86-style 64 B padding only half-fixes it. + +## 2026-07-10 — topic 8 scaffolded + +topic 8 scaffolded: study guide (anomaly-per-isolation-level table, doctors write-skew walkthrough, 2PL/OCC/MVCC comparison, postgres tuple-header + visibility flowchart, HOT chain, Hekaton contrast), 6 reading guides (postgres heapam.c/heapam_visibility.c HeapTupleSatisfiesMVCC + HOT + prune/vacuum with line anchors; RocksDB optimistic vs pessimistic txns over one base class — memtable-only OCC validation, point lock manager; surrealdb kvs layer — newly cloned — versioned reads + putc as portable OCC; Berenson '95 history notation + SI dethroned; SSI VLDB'12 dangerous structure + the single-writer M8 shortcut question; Hekaton + Wu/Pavlo 5-axis menu), experiments crate compiles: Mvcc `todo!()` stub with 8 contract tests including write_skew_HAPPENS_under_SI (test passes when the anomaly occurs) and Serializable-mode prevention via read-set validation, txn_bench PROVIDED (global Mutex baseline vs MVCC, 3 mixes incl. 64-key hot set, abort counts). + +## 2026-07-10 — topic 7 scaffolded + +topic 7 scaffolded: study guide (RESP wire anatomy, event-loop mermaid beforeSleep→poll→read→execute→buffer, three threading models table, backpressure: querybuf/output-buffer kills vs pgwire portals), 4 reading guides (redis ae.c + networking.c parse/reply path with line anchors; valkey 8 io_threads.c SPSC inboxes + tagged job pointers + memory_prefetch.c batch-MLP; pgwire Parse/Bind/Execute/Sync portals + qdrant dual tonic servers — both newly cloned; C10K → thread-per-core arc with the shared↔sharded plane exercise), experiments crate compiles: RESP2 parse/encode `todo!()` stub with 8 format-fixing tests (incomplete-input-keeps-bytes, binary-safe bulks, pipelining), tokio server PROVIDED (16-shard store, parse-all-then-flush-once pending-writes trick) — benches vs real redis via redis-benchmark -P 1/-P 64 + flamegraph once resp.rs is implemented. + +## 2026-07-10 — topic 6 scaffolded + +topic 6 scaffolded: study guide (translation-cost table hash/swizzle/MMU, miss-path mermaid, three shapes of approximate-LRU, swip state diagram, mmap CIDR-'22 checklist), 6 reading guides (postgres bufmgr.c packed-atomic state + CLOCK + buffer rings; DuckDB eviction queue with dead nodes + 4096-insert purge — newly cloned; LeanStore swips/cooling/hybrid latches — newly cloned; redis zmalloc per-thread padded counters + turso CLOCK page cache bonus; mmap paper with LMDB rebuttal; LeanStore+vmcache paper arc), experiments crate compiles: CLOCK BufferPool `todo!()` stub with contract tests (pinned-never-evicted, dirty-writeback, scan-pressure survival), pool_vs_mmap binary (1GiB file, 4× memory budget, Zipf, tail-latency focus), eviction bench PROVIDED and run — CLOCK 67.0% vs strict-LRU 66.3% hit rate at 20× less time per access (32ms vs 678ms per 1M trace): the "nobody ships strict LRU" lesson, measured. + +## 2026-07-10 — topic 5 scaffolded + +topic 5 scaffolded: study guide (WAL rule, four-designs axis LMDB→turso→postgres→redis-AOF, fsync ladder table, group-commit mermaid), 5 reading guides (postgres xlog.c — newly cloned — reserve-then-copy/XLogFlush-recheck/FPI with line anchors; turso WAL checksum chain + salts; redis aof.c/rdb.c with the AOF-as-LSM mapping + FalkorDB angle; ARIES three passes/CLRs; Aether four-bottleneck taxonomy), experiments crate compiles: fsync_ladder PROVIDED and run (this Mac: fsync 21µs vs F_FULLFSYNC 3.0ms — 140×, the macOS weak-fsync gap is real), Wal `todo!()` stub with format-fixing tests (torn tail, uncommitted-txn invisibility, commit_many = 1 fsync), crash_test kill-9 harness (100 rounds, acked-key + atomicity checks), commit_throughput bench (per-commit vs group 8/64/512). + +## 2026-07-10 — topic 4 scaffolded + +topic 4 scaffolded: study guide (memtable→SST lifecycle mermaid, SST block anatomy, leveled/tiered/lazy RUM table, stall triggers, Monkey intuition), 6 reading guides (lsm-tree crate — newly cloned, fjall delegates to it — + RocksDB compaction/table with line anchors; Monkey, Dostoevsky, RocksDB TODS '21, compaction design-space VLDB '21), experiments crate compiles: mini-LSM with provided Bloom (tests pass) + Memtable, SST writer/reader + Lsm engine `todo!()` stubs with correctness tests (tombstone-across-compaction, WA>1 check), write_amp binary measuring the full RUM position of leveled vs tiered. + +## 2026-07-10 — topic 3 scaffolded + +topic 3 scaffolded: study guide (slotted page anatomy, 3-sibling balance mermaid, LMDB double-meta COW commit diagram), 5 reading guides (turso btree deep + SQLite btree.c + LMDB mdb.c with line anchors from fresh clones; Graefe survey selective-read map, SQLite file-format hex-dump exercise), experiments crate compiles: slotted Page + DiskBTree `todo!()` stubs with format-fixing tests, bench vs redb (point/scan) + prefix-truncation stress case (32B keys, 24B shared prefix). + +## 2026-07-10 — topic 2 scaffolded + +topic 2 scaffolded: study guide (chaining vs open addressing cache stories, incremental-rehash mermaid, skiplist/rax ASCII, dense-filter/fat-payload pattern table), 7 reading guides (redis dict/zset/rax, hashbrown SwissTable, RocksDB InlineSkipList — line numbers from local clones; ART paper, CppCon SwissTable talk), experiments crate compiles: skiplist + incremental_map `todo!()` stubs with tests (the build work is the learning work), benches vs hashbrown/BTreeMap/crossbeam-skiplist, rehash_spike binary (HdrHistogram per-insert max/p99.9). + +## 2026-07-10 — topic 1 started + +topic 1 started: study guide (two-family write/read paths, amplification vocabulary, RUM triangle), 8 reading guides (fjall/turso/tidesdb/rocksdb code + O'Neil/Comer/RUM/Hellerstein papers, line numbers from fresh shallow clones), engine_shootout scaffold (fjall vs redb behind a common trait, db_bench workload names, durability parity) compiles + smoke-tested (space-amp binary at 20K keys shows fixed-overhead floor, not amplification — re-run at 1M+). Topic 0 plan audit: fixed phantom CMU-lecture reference in PLAN.md, added missing roofline-thinking section to topic 0 README §4. + +## 2026-07-10 — topic 0 finished + +topic 0 finished: cache_ladder (after fixing a self-caching bug: restarting the pointer chase at 0 measured an 8MB hot path — fixed by carrying the walker across iterations; true ladder 1.0 ns L1 / 5–9 ns L2 / ~110 ns DRAM+TLB), lookup_shootout (HashMap flat at ~7–9 ns thanks to MLP; binary search wins ≤1e4; linear scan never beats hashing at n≥100 — folklore busted), flamegraph captured (21% SipHash in HashMap lookups), reference baselines recorded in capstone/BASELINES.md. Topic 0 + M0 done. + +## 2026-07-10 — topic 0 started + +topic 0 started: study guide + 3 experiment benches (cache_ladder, lookup_shootout, branch_misprediction); capstone workspace scaffolded with `workload` crate (seeded Zipfian generator, ~11M ops/s). First measured result: branchy filter 8.1x slower on shuffled vs sorted data; branchless flat at 15 Gelem/s. Repo published to github.com/AviAvni/database-learning-path. + +## 2026-07-10 — repo initialized + +repo initialized: plan, capstone design, resources. + diff --git a/SUMMARY.md b/SUMMARY.md index d88a36b..fc13182 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -1,6 +1,7 @@ # Summary [Database Learning Path](README.md) +[Findings — every measured result](FINDINGS.md) [The Plan](PLAN.md) [Progress](PROGRESS.md) [Session log](SESSION-LOG.md) diff --git a/capstone/Cargo.lock b/capstone/Cargo.lock new file mode 100644 index 0000000..8f9ee1c --- /dev/null +++ b/capstone/Cargo.lock @@ -0,0 +1,684 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "workload" +version = "0.1.0" +dependencies = [ + "criterion", + "rand", + "rand_distr", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/capstone/README.md b/capstone/README.md index 91b6d2c..a30e0d5 100644 --- a/capstone/README.md +++ b/capstone/README.md @@ -40,9 +40,16 @@ flowchart TD ## Milestone map -Milestones M0–M31 map 1:1 to curriculum topics 0–31 in `PLAN.md`; each topic's +Milestones M0–M43 map 1:1 to curriculum topics 0–43 in `PLAN.md`; each topic's "Capstone milestone" line defines the scope. Status lives in `PROGRESS.md`. +M0–M31 build the engine itself, in the spine below. M32–M43 attach the later +topics' capabilities to a working engine rather than extending the spine: HTAP +routing (M32), temporal queries (M33), diagnosis surfaces (M34), admission +control (M35), sharding and distributed execution (M36/M37), and the six +graph use cases (M38–M43), each landing as procedures and index structures +over the M20 core. + Rough dependency spine: M0 → M2 → M13 (naive adjacency graph) → M10/M11 (query engine) → M20 (sparse-matrix core replaces M13). Everything else attaches to that spine — persistence (M3–M6), server (M7), MVCC/concurrency (M8/M9), indexes (M12/M14/M23), @@ -64,4 +71,7 @@ flowchart LR PF["performance
M17 / M18 / M19 / M22"] -.-> M20 ``` -Workspace is created at M0 (topic 0). Nothing lives here until then. +Workspace is created at M0 (topic 0). **M0 is the only milestone built so far** — +`crates/workload` (the seeded workload generator) plus the reference baselines in +[BASELINES.md](BASELINES.md). Everything above is the plan, not the state; check +[PROGRESS.md](../PROGRESS.md) for what actually exists before going looking for it. diff --git a/drafts/2026-07-taint-tracking.md b/drafts/2026-07-taint-tracking.md new file mode 100644 index 0000000..6b52b29 --- /dev/null +++ b/drafts/2026-07-taint-tracking.md @@ -0,0 +1,247 @@ +# 93% of Bitcoin addresses are "tainted". That's a bug in the rule, not a fact about Bitcoin. + +*Draft — rewrite in your own voice before publishing. Every number below is +attributed; the ones marked "measured" come from code you can run.* + +--- + +In 2012 someone stole 46,653 BTC from Linode's hosted wallets. Four years later, +researchers at Cambridge traced that theft forward through the blockchain using the +tainting rule the cryptocurrency-forensics industry had settled on. + +It marked **16,855,619 addresses as tainted — just over 93% of every address in +existence.** + +They then traced the same theft with a different rule and got **245,120 addresses, +or 1.35%**. + +Same blockchain, same theft, same question. A 69× difference in the answer, entirely +down to a modelling choice that almost nobody examines. And the rule that gives the +useful answer comes from an English court case decided in 1816. + +## The problem: money doesn't have identity + +Say I steal one coin from you and mix it with nine of my own in a single transaction +that pays out ten coins. Which of those outputs is your coin? + +There is no fact of the matter. A Bitcoin transaction consumes some outputs and +creates new ones; it does not carry per-satoshi provenance. So "is this coin stolen?" +is not a question the ledger answers — it's a question your *policy* answers, and +you have to pick one. + +Möser, Böhme and Breuker named the two policies the industry actually uses, and +there's a third from the law. + +``` + inputs POISON HAIRCUT FIFO + ┌──────┐ clean 3 everything each output lay the satoshis + ├──────┤ STOLEN 2 downstream is gets 2/9 end to end and cut + ├──────┤ clean 4 fully tainted stolen the outputs off the + └──────┘ front, in order + D: all 9 stolen D: 0.67 stolen D: 3 clean + E: all 9 stolen E: 0.44 stolen E: 2 STOLEN + F: all 9 stolen F: 0.89 stolen F: 4 clean +``` + +**Poison** says any tainted input contaminates every output completely. It's simple, +and it counts far more money as stolen than was ever stolen — the total grows without +bound as the chain fans out. + +**Haircut** says each output inherits the tainted *fraction* of the inputs. It +conserves the total exactly, which feels principled, and it became the default. + +**FIFO** says the first satoshi in is the first satoshi out. + +## What haircut actually does + +Haircut is the one that produced the 93%. It's worth being precise about why, because +it isn't obviously wrong. + +Haircut doesn't invent money. Trace a theft with it and the total tainted value at the +end equals exactly what was stolen — the arithmetic is conservative and correct. The +problem is *where* that value ends up. Every transaction with a tainted input gives +**every** output a nonzero share. So the tainted set grows at every hop, and after a +few hops it's most of the economy holding homeopathic quantities of your theft. + +I built a synthetic UTXO chain to watch this happen: 400 entities, 20,400 +transactions, 30,342 addresses, and one stolen coinbase worth 0.25% of all the money +on the chain. Running haircut to the end (measured): + +``` + tainted UTXOs 3657 of 3734 (97.9%) + tainted addresses 3553 of 3627 (98.0%) + tainted value 1000000 of 400000000 (the theft was 1000000) + + of those 3657 UTXOs: 658 are under 0.1% tainted + 2997 are 0.1%-5% + 2 are above 5% +``` + +Ninety-eight percent of everybody is holding a trace, and **two** UTXOs in the entire +chain hold a share worth arguing about. The Cambridge team's summary of the real-chain +version is blunter than anything I'd write: haircut tainting "smears the taint over +the actively traded bitcoin stock", and the effect of enforcing it through regulated +exchanges "might be more akin to a tax on all users." + +A rule that taints 93% of everyone is not a forensic tool. It's a tax. + +## Clayton's Case, 1816 + +In 1816 a bank called Devaynes, Dawes, Noble & Co. failed, and the court had to +work out what it owed a customer whose account had seen a long series of deposits and +withdrawals before the collapse. Which deposits had funded which withdrawals? + +The Master of the Rolls — one of the most senior judges in England — set down a rule +of stunning simplicity: **withdrawals are deemed to be drawn against the deposits +first made.** First in, first out. + +Applied to a Bitcoin transaction: lay the input satoshis end to end in input order, +then cut the outputs off the front of that queue in output order. In the diagram +above, the two stolen satoshis land entirely in output E. Outputs D and F are clean. +Not 22% clean. Clean. + +## Why FIFO wins, and it isn't the fairness + +The interesting property isn't that FIFO is more "accurate" — it's a convention, not +a discovery. It's that FIFO is **lossless**. + +A satoshi under FIFO is stolen or it is not. There's no fractional state, so nothing +accumulates rounding, and no information is destroyed when funds merge. The Cambridge +paper puts it well: "the taint does not spread or diffuse, the transaction processes +it in a lossless way. This means that we can trace a bitcoin's heritage backwards as +well as tracing taint forwards." + +Three things follow, and they're all things haircut cannot give you: + +- **Provenance survives arbitrarily many hops.** Under haircut, after two merges your + number is a fraction of a fraction of a fraction. Under FIFO it's still a satoshi + with a name on it. +- **You can run it in reverse.** "Where did *this particular* satoshi come from?" is + answerable. Under haircut it isn't, at any price. +- **It's deterministic.** Two investigators running FIFO on the same chain get the + same answer and can be cross-examined on it. That matters more in a courtroom than + in a paper. + +## The whole algorithm is fifteen lines + +Here's the core, adapted from the authors' own Rust implementation. Represent an +output's provenance as a queue of runs — contiguous stretches of same-origin value — +and the entire rule is one function that cuts `value` satoshis off the front, +splitting the run that straddles the boundary: + +```rust +struct TaintPart { name: u16, value: u64 } // name 0 = clean + +fn extract_taint(queue: &mut VecDeque, value: u64) -> VecDeque { + let mut remaining = value; + let mut taken = VecDeque::new(); + while remaining > 0 { + match queue.pop_front() { + None => { // queue dry: the rest is clean + taken.push_back(TaintPart { name: 0, value: remaining }); + remaining = 0; + } + Some(run) if remaining >= run.value => { // whole run fits + remaining -= run.value; + taken.push_back(run); + } + Some(mut run) => { // run straddles the cut: split it + run.value -= remaining; + taken.push_back(TaintPart { name: run.name, value: remaining }); + queue.push_front(run); + remaining = 0; + } + } + } + taken +} +``` + +Processing a transaction is then: concatenate the input queues in input order, and +call this once per output, in output order. That's it. Note that `name` is a `u16` and +not a `bool` — real chains have more than one crime in them, and the queue tracks +which. + +Measured on my synthetic chain: **20,400 transactions in 6.6 ms**, or 3.1 million +transactions per second. The expensive-sounding thing is a queue splice. + +## The three policies, side by side + +Same chain, same theft (measured): + +| policy | tainted UTXOs | tainted addresses | value flagged | vs actually stolen | +|---|---|---|---|---| +| poison | 3690 (98.8%) | 3585 | 394,674,821 | **394.67×** | +| haircut | 3657 (97.9%) | 3553 | 1,000,000 | 1.00× | +| **FIFO** | **32 (0.9%)** | **32** | 1,000,000 | 1.00× | + +Poison declares 394 times more money stolen than was ever taken, because it re-counts +each descendant output's full value. Haircut conserves the total and destroys it as +information. FIFO conserves the total *and* keeps it in one place — 32 UTXOs, one of +which holds 22.5% of everything flagged. + +The real-chain numbers have the same shape. Linode 2012: 93% under haircut, 1.35% +under FIFO. The 2014 Flexcoin hack: 10,421,112 addresses under haircut (over 57% of +all of them), **15,265** under FIFO. + +## Two consequences nobody expects + +**Tracing matters legally, not just academically.** *Nemo dat quod non habet* — no one +gives what they do not own — is a principle of nearly every legal system. If Alice +steals Bob's horse and sells it to Charlie, Charlie doesn't own the horse. The +exception that used to matter in Britain, *market overt*, was abolished in 1995; +exceptions remain for **money** and bills of exchange, and the USA has designated +Bitcoin a *commodity*. So a theft victim can pursue stolen coins through however many +hands they've passed. Which means a traceable coin is a coin with a claim attached. + +**And mixers make things worse, not better.** The received wisdom is that a laundry +launders: put one black coin in with nine white, get ten white out. The Cambridge +argument inverts it, and it's a legal argument rather than a technical one. Getting +good title requires acquiring in good faith. Every transaction is public. Coin +checking exists and exchanges claim to do it. So passing a coin through a mixer puts +every later holder *on notice* that something may be wrong — and therefore "the likely +outcome of feeding one black coin and nine white coins into a bitcoin laundry isn't +ten white coins, but ten black ones." + +Their conclusion, which I've thought about more than anything else in the paper: +"people designing money laundering mechanisms have been using quite the wrong metrics +of quality." + +## The part that undercuts all of it + +I'd be doing the paper a disservice if I stopped there, because its last section +quietly demolishes its own premise, and it's the most valuable page in it. + +Having built the tracing machinery, the authors went looking for theft victims to help +— and found that "with one exception, the victims we talked to were using **hosted +wallets**." The exchange holds the keys, the customer sees a balance, and increasingly +the exchange doesn't move coins on-chain at all: it settles internally against other +customers. If the transaction never reaches the chain, no amount of chain analysis +will ever see it. "In no case could we find any clear documentation of the actual +ownership of the missing cryptocurrency." + +The real problem, they conclude, is not cryptography but "the emergence of a shadow +banking system." + +Take that as a methodological warning that generalises well past blockchains: **your +analysis is only ever as good as the coverage of the log you are analysing.** You can +pick the right tainting rule, implement it perfectly, run it at three million +transactions a second — and still be answering a question about the 40% of activity +that happened to be visible. + +--- + +*The synthetic chain, the three tainting policies and the measurements above are +topic 41 of [a database-internals curriculum I'm writing](https://github.com/AviAvni/database-learning-path) +— 44 topics where every claim ships with the benchmark that produced it. +`./verify.sh 41` reproduces the tables in this post.* + +**Sources** + +- Anderson, Shumailov, Ahmed & Rietmann, *Bitcoin Redux*, WEIS 2018 — the Linode and + Flexcoin figures, Clayton's Case, `nemo dat`, and the hosted-wallet finding. + [PDF](https://www.cl.cam.ac.uk/archive/rja14/Papers/bitcoin-redux.pdf) +- Möser, Böhme & Breuker (2013, 2014) — the poison and haircut policies. +- *Devaynes v Noble* (1816), commonly Clayton's Case. +- [TaintChain/RustyTaintChain](https://github.com/TaintChain/RustyTaintChain) — the + authors' FIFO implementation, which `extract_taint` above is adapted from. diff --git a/drafts/README.md b/drafts/README.md new file mode 100644 index 0000000..de4bceb --- /dev/null +++ b/drafts/README.md @@ -0,0 +1,19 @@ +# Drafts + +Prose spun out of the topics and aimed at a different audience than the study +guides — blog-post shaped rather than curriculum shaped. Kept in the repo +because the numbers in these pieces come from the same benchmarks as everything +else, so they should live next to the code that produces them and move when it +moves. + +Not part of the book: nothing here is in `SUMMARY.md`, and none of it is +finished. Treat every file as a draft in the ordinary sense — the voice needs a +pass, the framing may be wrong, and a piece may never be published at all. + +The one rule that still applies: **every number must be attributed**, either to +a paper section or to a lane in `./verify.sh`. A draft is allowed to be badly +written. It is not allowed to invent figures. + +| draft | source topic | status | +|---|---|---| +| [2026-07-taint-tracking.md](2026-07-taint-tracking.md) | [41 — On-Chain & Crypto Analytics](../topics/41-onchain-analytics/README.md) | needs a voice pass before publishing | diff --git a/resources/codebases.md b/resources/codebases.md index c1adf49..463f786 100644 --- a/resources/codebases.md +++ b/resources/codebases.md @@ -65,3 +65,111 @@ Clone the ones in active use to `~/repos/`. | [automerge/automerge](https://github.com/automerge/automerge) | Rust | CRDT engine — state/op-based, columnar op storage (topic 31) | | [loro-dev/loro](https://github.com/loro-dev/loro) | Rust | fast modern CRDT engine, great perf blog posts (topic 31) | | [graphistry/pygraphistry](https://github.com/graphistry/pygraphistry) | Python | GPU graph ETL/analytics on RAPIDS cuDF/cuGraph, GFQL query layer — production analog to Gunrock's research code (topics 18, 24) | + +## What the `file:line` anchors were read against + +The reading guides quote a few thousand `file:line` anchors into the codebases +above. A line number only means something next to the commit it was read at, so +every clone the guides reference is pinned here — once, rather than in each of +the 230 guides, so the record stays consistent instead of drifting per file. + +Regenerate with `python3 tools/pin-table.py`, or `--check` to see whether your +clones have moved since the table was written. Both read `~/repos` (override +with `DLP_CLONES`), so this is a local command — CI has no clones and does not +check it. + +Clone to `~/repos/` and `git checkout` the listed commit if you want the +anchors to land exactly; on a newer commit expect the structure to match and the +line numbers to have moved. The mention count is there to tell you which clones +are worth fetching first. + + + +| clone | read at | dated | mentions | origin | +|---|---|---|---|---| +| `FalkorDB` | `ccb449a9a` | 2026-07-15 | 282 | [https://github.com/FalkorDB/FalkorDB](https://github.com/FalkorDB/FalkorDB) | +| `redis` | `a176d1225` | 2026-03-24 | 242 | [https://github.com/redis/redis](https://github.com/redis/redis) | +| `postgres` | `701f021` | 2026-07-10 | 226 | [https://github.com/postgres/postgres](https://github.com/postgres/postgres) | +| `duckdb` | `6c0c1a68` | 2026-07-10 | 191 | [https://github.com/duckdb/duckdb](https://github.com/duckdb/duckdb) | +| `rocksdb` | `7c80a5a` | 2026-07-09 | 146 | [https://github.com/facebook/rocksdb](https://github.com/facebook/rocksdb) | +| `neon` | `8f60b04` | 2026-05-25 | 118 | [https://github.com/neondatabase/neon](https://github.com/neondatabase/neon) | +| `GraphBLAS` | `1fd5475` | 2026-02-05 | 104 | [https://github.com/DrTimothyAldenDavis/GraphBLAS](https://github.com/DrTimothyAldenDavis/GraphBLAS) | +| `sqlite` | `951de30` | 2026-07-09 | 83 | [https://github.com/sqlite/sqlite](https://github.com/sqlite/sqlite) | +| `turso` | `dd775bc` | 2026-07-10 | 76 | [https://github.com/tursodatabase/turso](https://github.com/tursodatabase/turso) | +| `datafusion` | `1e77af8` | 2026-07-10 | 75 | [https://github.com/apache/datafusion](https://github.com/apache/datafusion) | +| `hashbrown` | `d69025b` | 2026-07-06 | 69 | [https://github.com/rust-lang/hashbrown](https://github.com/rust-lang/hashbrown) | +| `qdrant` | `44ad62f` | 2026-06-03 | 68 | [https://github.com/qdrant/qdrant](https://github.com/qdrant/qdrant) | +| `valkey` | `8891441ab` | 2026-05-03 | 64 | [https://github.com/valkey-io/valkey](https://github.com/valkey-io/valkey) | +| `memgraph` | `8f87f6a` | 2026-07-09 | 61 | [https://github.com/memgraph/memgraph](https://github.com/memgraph/memgraph) | +| `fjall` | `80cf6bc` | 2026-07-05 | 54 | [https://github.com/fjall-rs/fjall](https://github.com/fjall-rs/fjall) | +| `egg` | `f94c346` | 2026-04-14 | 50 | [https://github.com/egraphs-good/egg](https://github.com/egraphs-good/egg) | +| `LAGraph` | `e2539e2` | 2025-09-08 | 49 | [https://github.com/GraphBLAS/LAGraph](https://github.com/GraphBLAS/LAGraph) | +| `tiflash` | `b5093dd` | 2026-07-09 | 49 | [https://github.com/pingcap/tiflash](https://github.com/pingcap/tiflash) | +| `z3` | `1d425e5` | 2026-07-09 | 49 | [https://github.com/Z3Prover/z3](https://github.com/Z3Prover/z3) | +| `materialize` | `b06b3d6` | 2026-07-10 | 47 | [https://github.com/MaterializeInc/materialize](https://github.com/MaterializeInc/materialize) | +| `lmdb` | `704dc70` | 2026-06-24 | 45 | [https://github.com/LMDB/lmdb](https://github.com/LMDB/lmdb) | +| `clickhouse` | `4d598fb2c` | 2026-07-10 | 42 | [https://github.com/ClickHouse/ClickHouse](https://github.com/ClickHouse/ClickHouse) | +| `leanstore` | `90fcf18` | 2025-09-11 | 42 | [https://github.com/leanstore/leanstore](https://github.com/leanstore/leanstore) | +| `prometheus` | `f282b5c` | 2026-07-10 | 41 | [https://github.com/prometheus/prometheus](https://github.com/prometheus/prometheus) | +| `BlockSci` | `14ccc93` | 2020-11-13 | 40 | [https://github.com/citp/BlockSci](https://github.com/citp/BlockSci) | +| `kuzu` | `89f0263` | 2025-10-10 | 39 | [https://github.com/kuzudb/kuzu](https://github.com/kuzudb/kuzu) | +| `tidesdb` | `810507a` | 2026-07-10 | 39 | [https://github.com/tidesdb/tidesdb](https://github.com/tidesdb/tidesdb) | +| `polars` | `f8bcc3d` | 2026-07-10 | 37 | [https://github.com/pola-rs/polars](https://github.com/pola-rs/polars) | +| `lsm-tree` | `8526dd3` | 2026-07-05 | 35 | [https://github.com/fjall-rs/lsm-tree](https://github.com/fjall-rs/lsm-tree) | +| `neo4j` | `eccd584a` | 2026-07-02 | 35 | [https://github.com/neo4j/neo4j](https://github.com/neo4j/neo4j) | +| `ligra` | `8763202` | 2024-02-18 | 34 | [https://github.com/jshun/ligra](https://github.com/jshun/ligra) | +| `cockroach` | `a7e11788` | 2026-07-06 | 33 | [https://github.com/cockroachdb/cockroach](https://github.com/cockroachdb/cockroach) | +| `slatedb` | `323ed1b` | 2026-07-10 | 32 | [https://github.com/slatedb/slatedb](https://github.com/slatedb/slatedb) | +| `tantivy` | `7152d53` | 2026-07-10 | 32 | [https://github.com/quickwit-oss/tantivy](https://github.com/quickwit-oss/tantivy) | +| `tikv` | `eb8dd65` | 2026-07-09 | 31 | [https://github.com/tikv/tikv](https://github.com/tikv/tikv) | +| `gunrock` | `748f79e` | 2026-02-09 | 27 | [https://github.com/gunrock/gunrock](https://github.com/gunrock/gunrock) | +| `raphtory` | `5d0d286` | 2026-07-21 | 26 | [https://github.com/Pometry/Raphtory](https://github.com/Pometry/Raphtory) | +| `rayon` | `6d9e94b` | 2026-06-27 | 26 | [https://github.com/rayon-rs/rayon](https://github.com/rayon-rs/rayon) | +| `ALEX` | `4370da6` | 2024-03-12 | 24 | [https://github.com/microsoft/ALEX](https://github.com/microsoft/ALEX) | +| `raft-rs` | `ad13f3d` | 2026-05-13 | 24 | [https://github.com/tikv/raft-rs](https://github.com/tikv/raft-rs) | +| `simdjson` | `c783809` | 2026-07-10 | 24 | [https://github.com/simdjson/simdjson](https://github.com/simdjson/simdjson) | +| `splink` | `04189f5` | 2026-07-23 | 24 | [https://github.com/moj-analytical-services/splink](https://github.com/moj-analytical-services/splink) | +| `cudf` | `2f082a7` | 2026-07-10 | 23 | [https://github.com/rapidsai/cudf](https://github.com/rapidsai/cudf) | +| `RediSearch` | `87276ca` | 2026-07-09 | 23 | [https://github.com/RediSearch/RediSearch](https://github.com/RediSearch/RediSearch) | +| `risingwave` | `119de0a` | 2026-07-10 | 23 | [https://github.com/risingwavelabs/risingwave](https://github.com/risingwavelabs/risingwave) | +| `memchr` | `5fdb40c` | 2026-07-07 | 21 | [https://github.com/BurntSushi/memchr](https://github.com/BurntSushi/memchr) | +| `pgwire` | `6bb6299` | 2026-06-29 | 21 | [https://github.com/sunng87/pgwire](https://github.com/sunng87/pgwire) | +| `quickwit` | `a5ad540` | 2026-07-08 | 21 | [https://github.com/quickwit-oss/quickwit](https://github.com/quickwit-oss/quickwit) | +| `foundationdb` | `4c775a9` | 2026-07-10 | 19 | [https://github.com/apple/foundationdb](https://github.com/apple/foundationdb) | +| `gapbs` | `b5e3e19` | 2024-05-11 | 19 | [https://github.com/sbeamer/gapbs](https://github.com/sbeamer/gapbs) | +| `loro` | `b81abfc` | 2026-07-07 | 19 | [https://github.com/loro-dev/loro](https://github.com/loro-dev/loro) | +| `tidb` | `b94006d` | 2026-07-10 | 19 | [https://github.com/pingcap/tidb](https://github.com/pingcap/tidb) | +| `usearch` | `9fd6b01` | 2026-05-24 | 19 | [https://github.com/unum-cloud/usearch](https://github.com/unum-cloud/usearch) | +| `cr-sqlite` | `891fe9e` | 2024-10-25 | 18 | [https://github.com/vlcn-io/cr-sqlite](https://github.com/vlcn-io/cr-sqlite) | +| `SimSIMD` | `63a254f` | 2026-05-23 | 16 | [https://github.com/ashvardanian/SimSIMD](https://github.com/ashvardanian/SimSIMD) | +| `sqlancer` | `af6ae85` | 2026-06-21 | 16 | [https://github.com/sqlancer/sqlancer](https://github.com/sqlancer/sqlancer) | +| `wgpu` | `f945c78` | 2026-07-10 | 15 | [https://github.com/gfx-rs/wgpu](https://github.com/gfx-rs/wgpu) | +| `automerge` | `c39339d` | 2026-07-10 | 14 | [https://github.com/automerge/automerge](https://github.com/automerge/automerge) | +| `diamond-types` | `ad48b9c` | 2026-05-29 | 14 | [https://github.com/josephg/diamond-types](https://github.com/josephg/diamond-types) | +| `spicedb` | `8422483` | 2026-07-24 | 14 | [https://github.com/authzed/spicedb](https://github.com/authzed/spicedb) | +| `bloodhound` | `1968388` | 2026-07-24 | 13 | [https://github.com/SpecterOps/BloodHound](https://github.com/SpecterOps/BloodHound) | +| `crossbeam` | `6b7458d` | 2026-07-10 | 13 | [https://github.com/crossbeam-rs/crossbeam](https://github.com/crossbeam-rs/crossbeam) | +| `GraphRAG-SDK` | `f42ab3d` | 2026-04-12 | 13 | [https://github.com/FalkorDB/GraphRAG-SDK](https://github.com/FalkorDB/GraphRAG-SDK) | +| `influxdb` | `d783411` | 2026-06-17 | 13 | [https://github.com/influxdata/influxdb](https://github.com/influxdata/influxdb) | +| `VictoriaMetrics` | `c1e39b2` | 2026-07-10 | 12 | [https://github.com/VictoriaMetrics/VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) | +| `falkordb-rs-next-gen` | `67c71b81` | 2026-07-27 | 10 | [https://github.com/FalkorDB/falkordb-rs-next-gen](https://github.com/FalkorDB/falkordb-rs-next-gen) | +| `raft.tla` | `6ecbdbc` | 2025-02-18 | 9 | [https://github.com/ongardie/raft.tla](https://github.com/ongardie/raft.tla) | +| `surrealdb` | `9d9a5b0` | 2026-07-02 | 9 | [https://github.com/surrealdb/surrealdb](https://github.com/surrealdb/surrealdb) | +| `feldera` | `bb49055` | 2026-07-10 | 7 | [https://github.com/feldera/feldera](https://github.com/feldera/feldera) | +| `RedisBloom` | `ab734fa` | 2026-07-05 | 7 | [https://github.com/RedisBloom/RedisBloom](https://github.com/RedisBloom/RedisBloom) | +| `sqlparser-rs` | `aeb616f` | 2026-07-03 | 6 | [https://github.com/apache/datafusion-sqlparser-rs](https://github.com/apache/datafusion-sqlparser-rs) | +| `falkordb-py` | `ac68e59` | 2026-04-28 | 5 | [https://github.com/FalkorDB/falkordb-py](https://github.com/FalkorDB/falkordb-py) | +| `pytorch_geometric` | `1f0661c` | 2026-06-19 | 5 | [https://github.com/pyg-team/pytorch_geometric](https://github.com/pyg-team/pytorch_geometric) | +| `roaring-rs` | `83caaca` | 2026-04-24 | 5 | [https://github.com/RoaringBitmap/roaring-rs](https://github.com/RoaringBitmap/roaring-rs) | +| `arrow-rs` | `fed7862` | 2026-07-10 | 4 | [https://github.com/apache/arrow-rs](https://github.com/apache/arrow-rs) | +| `benchbase` | `33c0047` | 2025-12-13 | 4 | [https://github.com/cmu-db/benchbase](https://github.com/cmu-db/benchbase) | +| `differential-dataflow` | `3f279da` | 2026-05-29 | 4 | [https://github.com/TimelyDataflow/differential-dataflow](https://github.com/TimelyDataflow/differential-dataflow) | +| `RustyTaintChain` | `4e12fd0` | 2021-03-05 | 4 | [https://github.com/TaintChain/RustyTaintChain](https://github.com/TaintChain/RustyTaintChain) | +| `cuvs` | `8b97b61` | 2026-07-10 | 3 | [https://github.com/rapidsai/cuvs](https://github.com/rapidsai/cuvs) | +| `go-ycsb` | `f030f99` | 2025-12-31 | 3 | [https://github.com/pingcap/go-ycsb](https://github.com/pingcap/go-ycsb) | +| `cranelift-jit-demo` | `3e5e9b6` | 2025-11-07 | 2 | [https://github.com/bytecodealliance/cranelift-jit-demo](https://github.com/bytecodealliance/cranelift-jit-demo) | +| `helix-db` | `47191c6` | 2026-07-05 | 2 | [https://github.com/HelixDB/helix-db](https://github.com/HelixDB/helix-db) | +| `PGM-index` | `c6fcf3d` | 2024-11-28 | 2 | [https://github.com/gvinciguerra/PGM-index](https://github.com/gvinciguerra/PGM-index) | +| `timely-dataflow` | `15fc7c9` | 2026-06-12 | 2 | [https://github.com/TimelyDataflow/timely-dataflow](https://github.com/TimelyDataflow/timely-dataflow) | + + diff --git a/tools/pin-table.py b/tools/pin-table.py new file mode 100644 index 0000000..15749c0 --- /dev/null +++ b/tools/pin-table.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Regenerate the "read against" pin table in resources/codebases.md. + +Why this exists +--------------- +The reading guides quote `file:line` anchors into real codebases — a few +thousand of them. An anchor is only meaningful next to the commit it was read +at, because line numbers move. CONTRIBUTING asks for that commit to be +recorded; recording it in each of the 230 guides would mean thousands of SHAs +drifting independently, so it is recorded once, here, for every clone the +guides reference. + +Usage +----- + python3 tools/pin-table.py # rewrite the table in place + python3 tools/pin-table.py --check # exit 1 if the table is stale + +It reads the clones under ~/repos (override with DLP_CLONES) and only emits a +row for a repo it can actually verify: a real git checkout, with a resolvable +HEAD and an origin URL. Repos it cannot verify are reported and left out +rather than guessed at. +""" + +import os +import re +import subprocess +import sys +from pathlib import Path + +CLONES = Path(os.environ.get("DLP_CLONES", Path.home() / "repos")) +REPO = Path(__file__).resolve().parent.parent +TARGET = REPO / "resources" / "codebases.md" +START = "" +END = "" + +# Words that collide with ordinary prose or with directories that are not +# reference clones, so a mention count means nothing for them. +IGNORE = {"database-learning-path", "tensor", "benchmark", "llm-model"} +MIN_MENTIONS = 2 + + +def git(repo: Path, *args: str) -> str: + r = subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, text=True + ) + return r.stdout.strip() if r.returncode == 0 else "" + + +def corpus() -> str: + """Everything the mention count is computed over. + + The generated table is stripped out first: it names every repo it lists, so + counting it would make each run's numbers depend on the previous run's + output and `--check` would never settle. + """ + text = [] + for pat in ("topics/*/*.md", "resources/*.md", "capstone/*.md"): + for f in REPO.glob(pat): + s = f.read_text(encoding="utf8") + s = re.sub( + re.escape(START) + r".*?" + re.escape(END), "", s, flags=re.S + ) + text.append(s) + return "\n".join(text) + + +def main() -> int: + check = "--check" in sys.argv + if not CLONES.is_dir(): + print(f"no clone directory at {CLONES} — set DLP_CLONES", file=sys.stderr) + return 2 + + blob = corpus() + rows, unverified = [], [] + for d in sorted(os.listdir(CLONES)): + if d in IGNORE or not (CLONES / d / ".git").exists(): + continue + mentions = len(re.findall(rf"(? WAL, buffer pool, MVCC, compaction, columnar layout — is a refinement of the choice > made here: **update in place, or write out of place?** +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release` in `experiments/` — 1.08 M records of 100 B, random key +order, batches of 1000, synced: + +``` +fjall logical 108000000 B on-disk 48429915 B space-amp 0.45x +redb logical 108000000 B on-disk 6833917952 B space-amp 63.28x +``` + +Same data, same durability, **a 140× difference in bytes on disk** — and the +LSM is *below* 1.0 while the B-tree is at 63×. Neither number is a bug. fjall +compresses its sorted runs, so it spends read cost to buy space; redb is a +copy-on-write B-tree being fed random keys in 1080 separate commits, which is +precisely its worst case, because every commit copies the whole root-to-leaf +path and cannot free the old pages yet. + +That is the RUM conjecture with a price tag attached, and it is why this topic +comes before the other thirty: you do not get to optimize read, update and +memory at once, and the choice you make here propagates into every later topic. +The caveat is as important as the number — change the key order to sequential +or compact at the end and redb's 63× collapses. One measurement is one point in +the design space, never a verdict. + ## Outcomes By the end you can: diff --git a/topics/01-storage-engine-landscape/experiments/Cargo.lock b/topics/01-storage-engine-landscape/experiments/Cargo.lock new file mode 100644 index 0000000..3d8a8ce --- /dev/null +++ b/topics/01-storage-engine-landscape/experiments/Cargo.lock @@ -0,0 +1,1031 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "byteview" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "compare" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "double-ended-peekable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "engine-shootout" +version = "0.1.0" +dependencies = [ + "criterion", + "fjall", + "rand", + "rand_distr", + "redb", + "tempfile", +] + +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fjall" +version = "2.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc" +dependencies = [ + "byteorder", + "byteview", + "dashmap", + "log", + "lsm-tree", + "path-absolutize", + "std-semaphore", + "tempfile", + "xxhash-rust", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "guardian" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "interval-heap" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6" +dependencies = [ + "compare", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lsm-tree" +version = "2.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799399117a2bfb37660e08be33f470958babb98386b04185288d829df362ea15" +dependencies = [ + "byteorder", + "crossbeam-skiplist", + "double-ended-peekable", + "enum_dispatch", + "guardian", + "interval-heap", + "log", + "lz4_flex", + "path-absolutize", + "quick_cache", + "rustc-hash", + "self_cell", + "tempfile", + "value-log", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick_cache" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "self_cell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "std-semaphore" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "value-log" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" +dependencies = [ + "byteorder", + "byteview", + "interval-heap", + "log", + "path-absolutize", + "rustc-hash", + "tempfile", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "varint-rs" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa6c38708f6257f1ec2ca7e5a11f9bbf58a27d7060078b6b333624968183d96" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/topics/01-storage-engine-landscape/notes.md b/topics/01-storage-engine-landscape/notes.md index dd90ae2..42778d6 100644 --- a/topics/01-storage-engine-landscape/notes.md +++ b/topics/01-storage-engine-landscape/notes.md @@ -2,6 +2,37 @@ Numbers from this machine (Apple Silicon, macOS). Record *why*, not just what. +## Baseline (provided lane, Apple M3 Pro, measured 2026-07-28) + +`cargo run --release` — 1.08 M records of 100 B, random key order, batches of +1000, `sync()` at the end. Logical bytes vs bytes actually on disk: + +| engine | family | logical | on disk | space amp | +|---|---|---|---|---| +| fjall | LSM | 108.0 MB | 48.4 MB | **0.45×** | +| redb | B-tree (CoW) | 108.0 MB | 6833.9 MB | **63.28×** | + +**A 140× spread on the same data, and the LSM's number is below 1.0.** Both +halves are worth sitting with: + +- fjall lands at 0.45× because an LSM writes *compressed sorted runs*: the + value bytes are LZ4'd on the way into the SST, so "amplification" below 1 is + not a paradox, it is the third axis of the RUM triangle being spent — read + cost — to buy space. +- redb's 63× is **not** a defect, it is this workload hitting a copy-on-write + B-tree at its worst point. Random-order inserts touch a new leaf almost every + time; each of the 1080 batch commits copies every page on the path to the + root and cannot reuse the old ones until a later commit frees them. Random + keys plus per-batch durability plus no compaction is the adversarial case, + and it is exactly the case the RUM conjecture says you cannot escape — you + can only choose which axis pays. + +The honest caveat, and the reason this is a *starting* number rather than a +verdict: this measures ONE point in the design space (random keys, small +batches, no compaction pass afterwards). Change the key order to sequential, or +compact at the end, and redb's figure collapses. The exercise lanes are where +you find out how far. + ## Predictions (write BEFORE running the shootout) Per README §7 — predict the winner and the mechanism, then let the data grade you: diff --git a/topics/01-storage-engine-landscape/reading-architecture-of-a-dbms.md b/topics/01-storage-engine-landscape/reading-architecture-of-a-dbms.md index eb43d05..e020e1d 100644 --- a/topics/01-storage-engine-landscape/reading-architecture-of-a-dbms.md +++ b/topics/01-storage-engine-landscape/reading-architecture-of-a-dbms.md @@ -185,6 +185,14 @@ Skim NOW, return LATER: A database is five cooperating managers, and a storage engine is just one of them — this paper is the org chart for everything the capstone will build. +## Done when + +- [ ] You can draw the five boxes from memory and say which one owns the bytes on disk. +- [ ] You can trace one query through all five, naming the four stages inside the query processor. +- [ ] You can state both arguments §6 gives for bypassing OS caching — and connect them to the mmap tail you will measure in topic 6. +- [ ] You can say which of the five boxes fjall and redb implement, and which they deliberately do not. +- [ ] You wrote answers to both questions in notes.md. + ## References **Papers** diff --git a/topics/01-storage-engine-landscape/reading-comer-btree.md b/topics/01-storage-engine-landscape/reading-comer-btree.md index 541065d..26cd742 100644 --- a/topics/01-storage-engine-landscape/reading-comer-btree.md +++ b/topics/01-storage-engine-landscape/reading-comer-btree.md @@ -185,6 +185,14 @@ Read in this order: The B-tree is the memory hierarchy turned into a data structure: node size = transfer unit, fanout = whatever fits, height = the IO budget. +## Done when + +- [ ] You can state the disk access model in one line — cost is blocks touched, not comparisons made — and use it to explain why a binary search tree is the wrong shape. +- [ ] You can list the B-tree invariants and say which one forces the >=50% occupancy guarantee. +- [ ] You can narrate a split and say where the separator key ends up in a B-tree versus a B+-tree. +- [ ] You can compute fanout and height for a given page size and key width, and check yourself against topic 3's measured table (185 leaf cells and fanout 255 for 8 B keys). +- [ ] You wrote answers to both questions in notes.md, including what turso actually implements. + ## References **Papers** diff --git a/topics/01-storage-engine-landscape/reading-lsm-paper.md b/topics/01-storage-engine-landscape/reading-lsm-paper.md index 94e7bb9..19f9f33 100644 --- a/topics/01-storage-engine-landscape/reading-lsm-paper.md +++ b/topics/01-storage-engine-landscape/reading-lsm-paper.md @@ -173,6 +173,14 @@ Read in this order: LSM is not a data structure, it's an *IO scheduling policy*: convert random writes into sequential ones by deferring and batching — and pay for it at read time. +## Done when + +- [ ] You can explain why random in-place writes are the problem the paper is solving, in terms of what the disk is asked to do. +- [ ] You can define read, write and space amplification precisely enough to compute each one. +- [ ] You can work the §3 cost model far enough to say where the insert speedup comes from and what pays for it. +- [ ] You can explain the title claim — an IO scheduling policy, not a data structure — and defend it against the obvious objection that C0/C1 are clearly data structures. +- [ ] You wrote answers to all questions in notes.md, and can connect the paper's rolling merge to the leveled/tiered choice topic 4 asks you to implement. + ## References **Papers** diff --git a/topics/01-storage-engine-landscape/reading-rum-conjecture.md b/topics/01-storage-engine-landscape/reading-rum-conjecture.md index 79278f9..2dfbffd 100644 --- a/topics/01-storage-engine-landscape/reading-rum-conjecture.md +++ b/topics/01-storage-engine-landscape/reading-rum-conjecture.md @@ -143,6 +143,14 @@ elsewhere — find where the cost went before believing the benchmark. There is no best index, only a workload-shaped position on a three-way frontier — "which engine is better" is an ill-posed question until the workload is named. +## Done when + +- [ ] You can define RO, UO and MO as ratios and say what the denominator is in each. +- [ ] You can name one real structure per corner of the triangle. +- [ ] You can state what the conjecture does and does not claim (it is a conjecture and a compass, not a proven bound). +- [ ] You have placed this topic's own measured result on the triangle: fjall at 0.45x and redb at 63.28x space amp, and can say which axis each engine is spending. +- [ ] You wrote answers to both questions in notes.md, including where FalkorDB's matrix adjacency sits. + ## References **Papers** diff --git a/topics/02-in-memory-structures/README.md b/topics/02-in-memory-structures/README.md index e6fff1b..e696b7c 100644 --- a/topics/02-in-memory-structures/README.md +++ b/topics/02-in-memory-structures/README.md @@ -4,6 +4,31 @@ > in-memory database. This topic is where topic 0's cache lessons become design > rules: every structure here is a different answer to "how do I avoid DRAM misses?" +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin rehash_spike` — 10 M keys into `hashbrown`, one at a +time, every single insert timed: + +``` +hashbrown p50=42ns p99=291ns p99.9=1292ns p99.99=13423ns max=58392575ns + +per-decile max (ns): +[8110084, 13203125, 36417, 28320250, 85792, 46625, 51125, 58385375, 470917, 62083] + 8.1ms 13.2ms 36µs 28.3ms 86µs 47µs 51µs 58.4ms 471µs 62µs +``` + +**A 42-nanosecond median and a 58-millisecond maximum: the same operation, +1.4 million times apart.** Four of the ten deciles carry a multi-millisecond +spike and the rest sit in the tens of microseconds, because a doubling rehash +is not amortized over anything a *latency* measurement cares about — one +unlucky insert copies the entire table while every other insert waits. + +Read the decile row again: the spikes land where the table crossed a power of +two, so they are perfectly predictable and perfectly invisible to a throughput +number. This is why redis rehashes incrementally, and it is the concrete reason +topic 0 insisted on percentiles: an average over this data is 5.9 µs, which +describes no insert that actually happened. + ## Outcomes By the end you can: diff --git a/topics/02-in-memory-structures/experiments/Cargo.lock b/topics/02-in-memory-structures/experiments/Cargo.lock new file mode 100644 index 0000000..96ad088 --- /dev/null +++ b/topics/02-in-memory-structures/experiments/Cargo.lock @@ -0,0 +1,811 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hdrhistogram" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d1053f4708f0af3cf9fc5bffc7e68a914a3c45becb231c80068c9c3f78bea" +dependencies = [ + "base64", + "byteorder", + "crossbeam-channel", + "flate2", + "nom", + "num-traits", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "topic02-experiments" +version = "0.1.0" +dependencies = [ + "criterion", + "crossbeam-skiplist", + "hashbrown", + "hdrhistogram", + "rand", + "rand_distr", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/topics/02-in-memory-structures/experiments/src/bin/rehash_spike.rs b/topics/02-in-memory-structures/experiments/src/bin/rehash_spike.rs index 8161fb7..d24a48b 100644 --- a/topics/02-in-memory-structures/experiments/src/bin/rehash_spike.rs +++ b/topics/02-in-memory-structures/experiments/src/bin/rehash_spike.rs @@ -31,39 +31,67 @@ fn percentiles(name: &str, h: &Histogram) { ); } -fn main() { - // deterministic "random" keys without RNG overhead in the timed region - let key = |i: u64| i.wrapping_mul(0x9E3779B97F4A7C15); - - println!("inserting {N} keys one by one, timing each insert\n"); +/// deterministic "random" keys without RNG overhead in the timed region +fn key(i: u64) -> u64 { + i.wrapping_mul(0x9E3779B97F4A7C15) +} - let mut h_hb = Histogram::::new(3).unwrap(); - let mut hb = hashbrown::HashMap::new(); +/// Time N individual inserts into `map`, reporting percentiles and the +/// per-decile max so the spikes can be lined up with the doubling points. +fn measure(name: &str, mut insert: impl FnMut(u64, u64)) -> Histogram { + let mut h = Histogram::::new(3).unwrap(); let mut decile_max = vec![0u64; 10]; for i in 0..N { let t = Instant::now(); - hb.insert(key(i), i); + insert(key(i), i); let ns = t.elapsed().as_nanos() as u64; - h_hb.record(ns).unwrap(); + h.record(ns).unwrap(); let d = (i * 10 / N) as usize; decile_max[d] = decile_max[d].max(ns); } - percentiles("hashbrown", &h_hb); + percentiles(name, &h); println!(" per-decile max (ns): {decile_max:?}\n"); + h +} - let mut h_inc = Histogram::::new(3).unwrap(); - let mut inc = IncrementalMap::new(); - let mut decile_max = vec![0u64; 10]; - for i in 0..N { - let t = Instant::now(); - inc.insert(key(i), i); - let ns = t.elapsed().as_nanos() as u64; - h_inc.record(ns).unwrap(); - let d = (i * 10 / N) as usize; - decile_max[d] = decile_max[d].max(ns); +/// Run an exercise lane, reporting unimplemented `todo!()`s as a note +/// instead of a crash, so the provided lanes above always print. +fn stub_lane(name: &str, f: impl FnOnce() -> T) -> Option { + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { + println!("[stub — implement the todo!()s to unlock {name}]\n"); } - percentiles("incremental", &h_inc); - println!(" per-decile max (ns): {decile_max:?}"); + r.ok() +} + +fn main() { + println!("inserting {N} keys one by one, timing each insert\n"); + + // lane 1 (PROVIDED): hashbrown's doubling rehash — the spike we are here for + let mut hb = hashbrown::HashMap::new(); + let h_hb = measure("hashbrown", |k, v| { + hb.insert(k, v); + }); - println!("\nheadline: max ratio hashbrown/incremental = {:.1}x", h_hb.max() as f64 / h_inc.max() as f64); + // lane 2 (EXERCISE): your incremental rehash — same work, no spike + let h_inc = stub_lane("incremental rehash (src/incremental_map.rs)", || { + let mut inc = IncrementalMap::new(); + measure("incremental", |k, v| inc.insert(k, v)) + }); + + match h_inc { + Some(h_inc) => println!( + "headline: max ratio hashbrown/incremental = {:.1}x", + h_hb.max() as f64 / h_inc.max() as f64 + ), + None => println!( + "headline: hashbrown max = {} ns ({:.1} ms). The point of the exercise is\n\ + to get the second row's max down to microseconds without moving p50 much.", + h_hb.max(), + h_hb.max() as f64 / 1e6 + ), + } } diff --git a/topics/02-in-memory-structures/notes.md b/topics/02-in-memory-structures/notes.md index 9a6b248..c33741f 100644 --- a/topics/02-in-memory-structures/notes.md +++ b/topics/02-in-memory-structures/notes.md @@ -1,5 +1,36 @@ # Topic 2 — notes +## Baseline (provided lane, Apple M3 Pro, measured 2026-07-28) + +`cargo run --release --bin rehash_spike` — 10 M keys inserted one at a time, +every individual insert timed into an HdrHistogram (not criterion: the whole +point is the max, which averaging destroys). + +| impl | p50 | p99 | p99.9 | p99.99 | max | +|---|---|---|---|---|---| +| hashbrown (doubling) | 42 ns | 291 ns | 1292 ns | 13.4 µs | **58.4 ms** | +| incremental (yours) | | | | | stub | + +Per-decile max, in ns — the doubling sweeps are visible in the data: + +``` +[8110084, 13203125, 36417, 28320250, 85792, 46625, 51125, 58385375, 470917, 62083] + 8.1ms 13.2ms 36µs 28.3ms 86µs 47µs 51µs 58.4ms 471µs 62µs +``` + +**p50 is 42 ns and the max is 58.4 ms — a 1.4-millionfold spread inside one +operation type.** Four deciles carry a multi-millisecond spike and the rest are +in the tens of microseconds, because a rehash is not spread over anything: one +unlucky insert copies the whole table. That is the entire argument for redis's +incremental rehash, and it is why a p50 (or a mean, or a throughput figure) +cannot see this class of problem at all. + +Note the spikes are NOT evenly spaced across deciles: they land where the table +crossed a power of two, and 10 M keys crosses 2²³ near the eighth decile — the +58.4 ms max. Your incremental map has to move that max to microseconds while +keeping p50 near 42 ns; the trade you are making is that *every* insert now +does a little migration work. + ## Predictions (fill BEFORE running benches) | Bench | hashbrown | BTreeMap | crossbeam SkipMap | my skiplist | my inc. map | diff --git a/topics/03-btree-internals/README.md b/topics/03-btree-internals/README.md index 7d667a9..8b6f13f 100644 --- a/topics/03-btree-internals/README.md +++ b/topics/03-btree-internals/README.md @@ -4,6 +4,35 @@ > and LMDB's copy-on-write variant are three answers to the same question: > how do you keep a sorted map in fixed-size blocks that survive power loss? +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin btree_baseline`. The page arithmetic comes from the +format in `src/page.rs`; the timings are redb, warm, so the file is in the page +cache and this is in-page search plus pointer chasing, not disk: + +``` +key shape leaf cells fanout height @ 1e6 height @ 1e9 +8 B key, 8 B value 185 255 3 4 +32 B key, 8 B value 88 102 4 5 + +keys ns/lookup file MB height (our fmt) +10000 367 1.6 2 +100000 423 9.0 3 +1000000 862 67.9 3 +4000000 1101 270.0 3 +``` + +**The tidy version of this topic is wrong, and the ladder shows exactly how.** +"Height is the metric" predicts a step function — flat while height holds, +jumping when it grows. Instead cost climbs 862 → 1101 ns from 1e6 to 4e6 keys +with height pinned at 3. + +Height sets how many pages a lookup *touches*. What a touch *costs* is set by +whether that page is in CPU cache, and at 270 MB it is not. So there are two +levers here, not one: fanout (which you control through the page format, and +which suffix truncation exists to protect) and residency (which you do not +control at all until topic 6 gives you a buffer pool). Keep both columns. + ## Outcomes By the end you can: diff --git a/topics/03-btree-internals/experiments/Cargo.lock b/topics/03-btree-internals/experiments/Cargo.lock new file mode 100644 index 0000000..8d67609 --- /dev/null +++ b/topics/03-btree-internals/experiments/Cargo.lock @@ -0,0 +1,748 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "topic03-experiments" +version = "0.1.0" +dependencies = [ + "criterion", + "rand", + "redb", + "tempfile", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/topics/03-btree-internals/experiments/benches/disk_btree.rs b/topics/03-btree-internals/experiments/benches/disk_btree.rs index e27df0f..52f39d3 100644 --- a/topics/03-btree-internals/experiments/benches/disk_btree.rs +++ b/topics/03-btree-internals/experiments/benches/disk_btree.rs @@ -1,6 +1,12 @@ //! disk_btree — your slotted-page B+tree vs redb, plus the prefix-truncation //! experiment. //! +//! This one needs your `src/btree.rs`: every group puts your tree and redb in +//! the same table, so it panics until `DiskBTree` works. For the redb-only +//! baseline that runs on a fresh clone — the page arithmetic, the height +//! ladder and the long-key case — run `cargo run --release --bin +//! btree_baseline` first and record those numbers as your targets. +//! //! Honesty rules (topic 0/1): both engines warm (OS page cache holds everything //! at 1M keys — you are NOT benching the disk, note it); fixed seed; predict //! before running (notes.md table). diff --git a/topics/03-btree-internals/experiments/src/bin/btree_baseline.rs b/topics/03-btree-internals/experiments/src/bin/btree_baseline.rs new file mode 100644 index 0000000..2583bed --- /dev/null +++ b/topics/03-btree-internals/experiments/src/bin/btree_baseline.rs @@ -0,0 +1,185 @@ +//! Lane 1 (PROVIDED): the numbers your B+tree is aiming at, measured on a +//! production one — plus the page arithmetic that predicts them. +//! +//! cargo run --release --bin btree_baseline +//! +//! Nothing here touches your `src/page.rs` or `src/btree.rs`, so it runs on a +//! fresh clone. It exists because the topic's claim — *height is the metric, +//! fanout is the lever* — is checkable before you write a line of B-tree code: +//! +//! 1. the fanout arithmetic, derived from the fixed 4 KiB page format +//! 2. the height ladder: lookup cost vs key count in redb, warm +//! 3. the long-key case: 32-byte keys sharing a 24-byte prefix, which is +//! what suffix truncation exists to fix — priced on a real B-tree +//! +//! Predict in notes.md BEFORE running: the fanout for each key shape, the +//! height at 1e6 keys, and how much of the lookup ladder you expect to see +//! (all of this is warm — the OS page cache holds the whole file, so you are +//! measuring pointer chasing and in-page search, not the disk). + +use std::time::Instant; + +use rand::prelude::*; + +const TABLE: redb::TableDefinition<&[u8], &[u8]> = redb::TableDefinition::new("t"); +const PAGE_SIZE: usize = 4096; +const HEADER: usize = 8; +const PROBES: usize = 200_000; + +/// Cells per leaf and interior fanout for the page format documented in +/// src/page.rs. Arithmetic, not measurement — labelled as such in the output. +fn geometry(key_len: usize, val_len: usize) -> (usize, usize) { + // leaf cell: key_len u16 ∥ val_len u16 ∥ key ∥ val (+ 2 for its ptr) + let leaf_cell = 2 + 2 + key_len + val_len + 2; + // interior cell: child u32 ∥ key_len u16 ∥ key (+ 2 for its ptr) + let interior_cell = 4 + 2 + key_len + 2; + let usable = PAGE_SIZE - HEADER; + (usable / leaf_cell, usable / interior_cell) +} + +/// Height of a B+tree holding `n` keys given leaf capacity and fanout — +/// counting levels of page reads a point lookup must do. +fn height(n: u64, leaf_cells: usize, fanout: usize) -> u32 { + let mut pages = (n as f64 / leaf_cells as f64).ceil().max(1.0); + let mut h = 1; + while pages > 1.0 { + pages = (pages / fanout as f64).ceil(); + h += 1; + } + h +} + +fn short_key(i: u64) -> [u8; 8] { + i.to_be_bytes() +} + +/// 32 bytes, 24-byte shared prefix — the case that collapses fanout when +/// separators keep the whole key. +fn long_key(i: u64) -> [u8; 32] { + let mut k = [b'p'; 32]; + k[24..].copy_from_slice(&i.to_be_bytes()); + k +} + +fn dir_size(path: &std::path::Path) -> u64 { + std::fs::metadata(path).map(|m| m.len()).unwrap_or(0) +} + +/// Load `n` keys into a fresh redb file, then time random point lookups. +/// Returns (ns per lookup, file bytes). +fn measure_redb(n: u64, long: bool) -> (f64, u64) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("redb.db"); + let db = redb::Database::create(&path).unwrap(); + + // sorted inserts in one transaction: the cheap path, so the load time + // here is not what we are measuring + let tx = db.begin_write().unwrap(); + { + let mut t = tx.open_table(TABLE).unwrap(); + for i in 0..n { + let v = i.to_le_bytes(); + if long { + t.insert(&long_key(i)[..], &v[..]).unwrap(); + } else { + t.insert(&short_key(i)[..], &v[..]).unwrap(); + } + } + } + tx.commit().unwrap(); + + let mut rng = StdRng::seed_from_u64(11); + let ps: Vec = (0..PROBES).map(|_| rng.gen_range(0..n)).collect(); + + let rtx = db.begin_read().unwrap(); + let t = rtx.open_table(TABLE).unwrap(); + // warm every level of the tree before timing + for &i in ps.iter().take(1000) { + let _ = t.get(&short_key(i)[..]).unwrap(); + } + + let start = Instant::now(); + let mut found = 0u64; + for &i in &ps { + let hit = if long { + t.get(&long_key(i)[..]).unwrap() + } else { + t.get(&short_key(i)[..]).unwrap() + }; + found += hit.is_some() as u64; + } + let ns = start.elapsed().as_secs_f64() * 1e9 / PROBES as f64; + assert_eq!(found, PROBES as u64, "every probe key was inserted"); + drop(t); + drop(rtx); + (ns, dir_size(&path)) +} + +fn main() { + println!("== page arithmetic (from the format in src/page.rs, not measured) =="); + println!( + " {:<26} {:>10} {:>9} {:>16} {:>16}", + "key shape", "leaf cells", "fanout", "height @ 1e6", "height @ 1e9" + ); + for (name, kl, vl) in [ + ("8 B key, 8 B value", 8, 8), + ("32 B key, 8 B value", 32, 8), + ("8 B key, 100 B value", 8, 100), + ] { + let (leaf, fanout) = geometry(kl, vl); + println!( + " {name:<26} {leaf:>10} {fanout:>9} {:>16} {:>16}", + height(1_000_000, leaf, fanout), + height(1_000_000_000, leaf, fanout) + ); + } + println!(" a 32 B key costs {}x the interior slots of an 8 B key — that ratio,", { + let (_, f8) = geometry(8, 8); + let (_, f32b) = geometry(32, 8); + format!("{:.1}", f8 as f64 / f32b as f64) + }); + println!(" not the byte count, is what suffix truncation is buying back.\n"); + + println!("== the height ladder: redb point lookup vs key count, warm =="); + println!( + " {:<12} {:>12} {:>14} {:>16}", + "keys", "ns/lookup", "file MB", "height (our fmt)" + ); + let (leaf, fanout) = geometry(8, 8); + for n in [10_000u64, 100_000, 1_000_000, 4_000_000] { + let (ns, bytes) = measure_redb(n, false); + println!( + " {n:<12} {ns:>12.0} {:>14.1} {:>16}", + bytes as f64 / 1e6, + height(n, leaf, fanout) + ); + } + + println!("\n== the long-key case: 32 B keys, 24 B shared prefix, 1e6 keys =="); + let (ns_short, b_short) = measure_redb(1_000_000, false); + let (ns_long, b_long) = measure_redb(1_000_000, true); + println!(" 8 B keys : {ns_short:>8.0} ns/lookup {:>8.1} MB", b_short as f64 / 1e6); + println!(" 32 B keys: {ns_long:>8.0} ns/lookup {:>8.1} MB", b_long as f64 / 1e6); + println!( + " ratio : {:>8.2}x slower, {:>7.2}x bigger", + ns_long / ns_short, + b_long as f64 / b_short as f64 + ); + + println!("\nnotes:"); + println!("- everything above is WARM: the file fits in the page cache, so this is"); + println!(" in-page binary search plus pointer chasing, not disk I/O. Say so when"); + println!(" you record it, or the numbers mean nothing (topic 0's rule)."); + println!("- the last column is the height in OUR page format, so it is a cross-check"); + println!(" on the arithmetic, not a prediction of redb's own layout."); + println!("- READ THE LADDER CAREFULLY. The tidy version of this topic says cost is a"); + println!(" step function of height: flat while height is constant, jumping when it"); + println!(" grows. That is not what the middle column does — it keeps climbing from"); + println!(" 1e6 to 4e6 keys while the height stays put. Height sets how many pages a"); + println!(" lookup TOUCHES; what those touches COST is set by whether the pages are"); + println!(" in CPU cache, and at 270 MB they are not. Two levers, not one — and the"); + println!(" second is the reason topic 6 exists. Write down both numbers."); + println!("- these are the targets for your own DiskBTree. Record them in notes.md,"); + println!(" then run `cargo bench --bench disk_btree` once src/btree.rs works to"); + println!(" put your tree in the same table."); +} diff --git a/topics/03-btree-internals/experiments/src/btree.rs b/topics/03-btree-internals/experiments/src/btree.rs index 1dcc9b1..f45782a 100644 --- a/topics/03-btree-internals/experiments/src/btree.rs +++ b/topics/03-btree-internals/experiments/src/btree.rs @@ -17,6 +17,9 @@ use std::fs::File; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::Path; +// The fields and page helpers below are the scaffolding your create/open/ +// insert/get will use; rustc cannot see that while those are still todo!(). +#[allow(dead_code, reason = "read by the methods you are about to implement")] pub struct DiskBTree { file: File, root: u32, @@ -34,6 +37,7 @@ impl DiskBTree { todo!() } + #[allow(dead_code, reason = "called by the methods you are about to implement")] fn read_page(&mut self, no: u32) -> std::io::Result { let mut buf = [0u8; PAGE_SIZE]; self.file.seek(SeekFrom::Start(no as u64 * PAGE_SIZE as u64))?; @@ -41,11 +45,13 @@ impl DiskBTree { Ok(Page { buf }) } + #[allow(dead_code, reason = "called by the methods you are about to implement")] fn write_page(&mut self, no: u32, p: &Page) -> std::io::Result<()> { self.file.seek(SeekFrom::Start(no as u64 * PAGE_SIZE as u64))?; self.file.write_all(&p.buf) } + #[allow(dead_code, reason = "called by the methods you are about to implement")] fn alloc_page(&mut self) -> u32 { let n = self.npages; self.npages += 1; diff --git a/topics/03-btree-internals/notes.md b/topics/03-btree-internals/notes.md index 5ef9579..49eb52e 100644 --- a/topics/03-btree-internals/notes.md +++ b/topics/03-btree-internals/notes.md @@ -1,5 +1,52 @@ # Topic 3 — notes +## Baseline (provided lane, Apple M3 Pro, measured 2026-07-28) + +`cargo run --release --bin btree_baseline`. Two provided things: the page +arithmetic from the format documented in `src/page.rs`, and redb measured on +the workloads this topic's table asks about. Everything is **warm** — the file +sits in the page cache — so this is in-page search plus pointer chasing, not +disk I/O. + +### Fanout arithmetic (computed, not measured) + +| key shape | leaf cells | fanout | height @ 1e6 | height @ 1e9 | +|---|---|---|---|---| +| 8 B key, 8 B value | 185 | 255 | 3 | 4 | +| 32 B key, 8 B value | 88 | 102 | 4 | 5 | +| 8 B key, 100 B value | 35 | 255 | 3 | 5 | + +A 32 B key costs **2.5×** the interior slots of an 8 B key. That ratio — not +the byte count — is what suffix truncation buys back. + +### The height ladder (redb, warm) + +| keys | ns/lookup | file MB | height (our fmt) | +|---|---|---|---| +| 10 000 | 367 | 1.6 | 2 | +| 100 000 | 423 | 9.0 | 3 | +| 1 000 000 | 862 | 67.9 | 3 | +| 4 000 000 | 1101 | 270.0 | 3 | + +**This is the topic's tidy story failing, and it is the most useful number +here.** "Height is the metric" predicts a step function: flat while height is +constant, jumping when it grows. Instead cost climbs 862 → 1101 ns from 1e6 to +4e6 keys with height pinned at 3. Height sets how many pages a lookup *touches*; +what a touch *costs* is set by whether that page is in CPU cache, and at 270 MB +it is not. Two levers, and the second one is why topic 6 exists. + +### The long-key case (1e6 keys) + +| keys | ns/lookup | file MB | +|---|---|---| +| 8 B | 733 | 67.9 | +| 32 B, 24 B shared prefix | 882 | 135.3 | +| ratio | 1.20× | 1.99× | + +The file doubles and lookups slow 20%. The arithmetic above predicted a 2.5× +fanout loss; redb absorbs most of it, which is itself the finding — a +production B-tree already does some of what you are about to implement by hand. + ## Predictions (fill BEFORE running) | Bench | my btree | redb | diff --git a/topics/04-lsm-deep-dive/README.md b/topics/04-lsm-deep-dive/README.md index e266752..9e7823b 100644 --- a/topics/04-lsm-deep-dive/README.md +++ b/topics/04-lsm-deep-dive/README.md @@ -4,6 +4,31 @@ > topic is the rest of the LSM machine: SST anatomy, bloom filters, and > compaction — a scheduling problem wearing a storage-engine costume. +## The problem, predicted before it is measured + +This is one of two topics whose benchmark measures **only your own code** — +`write_amp` runs your LSM under leveled and tiered compaction, so on a fresh +clone it prints two stub notices and exits, and it has no lane in `./verify.sh`. +That makes the arithmetic below the thing to commit to first: + +``` + write amp read amp space amp +leveled ~ T/2 x L (~20x) ~ L runs (~4) low: L1+ disjoint +tiered ~ L (~4x) ~ K x L (~16) high: shadowed + versions survive + T = size ratio (10), L = levels (4), K = runs per level (4) +``` + +**A 5× swing in write amplification and a 4× swing in read amplification, from +one policy decision, in opposite directions.** That is the whole topic: leveled +and tiered are not better and worse, they are two positions on the RUM triangle +that topic 1 priced in bytes and this one prices in rewrites. + +Predict both numbers before you implement, then check `describe()`. If your +leveled figure lands near 4× or your tiered figure near 20×, you have wired the +strategies backwards — and the fact that you can tell from the amplification +number alone is why these are the metrics compaction papers argue about. + ## Outcomes By the end you can: diff --git a/topics/04-lsm-deep-dive/experiments/Cargo.lock b/topics/04-lsm-deep-dive/experiments/Cargo.lock new file mode 100644 index 0000000..a5e9308 --- /dev/null +++ b/topics/04-lsm-deep-dive/experiments/Cargo.lock @@ -0,0 +1,249 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "topic04-experiments" +version = "0.1.0" +dependencies = [ + "lz4_flex", + "rand", + "tempfile", + "xxhash-rust", +] + +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/04-lsm-deep-dive/experiments/src/bin/write_amp.rs b/topics/04-lsm-deep-dive/experiments/src/bin/write_amp.rs index 0f7a93b..43fee1e 100644 --- a/topics/04-lsm-deep-dive/experiments/src/bin/write_amp.rs +++ b/topics/04-lsm-deep-dive/experiments/src/bin/write_amp.rs @@ -63,7 +63,36 @@ fn run(name: &str, strategy: CompactionStrategy) { ); } +/// Run an exercise lane, reporting unimplemented `todo!()`s as a note +/// instead of a crash. +fn stub_lane(name: &str, f: impl FnOnce()) -> bool { + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { + println!("[stub — implement the todo!()s to unlock {name}]\n"); + } + r.is_ok() +} + fn main() { - run("leveled (ratio 10)", CompactionStrategy::Leveled { ratio: 10 }); - run("tiered (K=4)", CompactionStrategy::Tiered { k: 4 }); + // Both lanes are the exercise: this whole benchmark measures YOUR LSM + // (src/lsm.rs + src/sst.rs). There is no provided baseline to compare + // against here — the comparison is leveled vs tiered, both yours. + let a = stub_lane("leveled compaction", || { + run("leveled (ratio 10)", CompactionStrategy::Leveled { ratio: 10 }) + }); + let b = stub_lane("tiered compaction", || { + run("tiered (K=4)", CompactionStrategy::Tiered { k: 4 }) + }); + + if !(a && b) { + println!( + "This binary is the topic's acceptance test: it measures the RUM position\n\ + of your own LSM, so it has nothing to print until src/lsm.rs and\n\ + src/sst.rs are implemented. `cargo test` is the specification; the\n\ + reference write/read/space-amp figures to aim at are in notes.md." + ); + } } diff --git a/topics/04-lsm-deep-dive/experiments/src/lsm.rs b/topics/04-lsm-deep-dive/experiments/src/lsm.rs index 2add534..a8397b6 100644 --- a/topics/04-lsm-deep-dive/experiments/src/lsm.rs +++ b/topics/04-lsm-deep-dive/experiments/src/lsm.rs @@ -11,7 +11,8 @@ //! at the next level. L0 in both: every flush is one overlapping run. use crate::memtable::Memtable; -use crate::sst::{SstReader, SstWriter}; +// SstWriter is what your flush() writes through — see the todo!() below. +use crate::sst::SstReader; use std::path::PathBuf; #[derive(Clone, Copy, PartialEq)] @@ -38,6 +39,9 @@ impl Stats { } } +// Everything but `stats` is read only by the methods you are about to +// implement, so rustc reports them as dead while those are still todo!(). +#[allow(dead_code, reason = "read by the methods you are about to implement")] pub struct Lsm { pub stats: Stats, memtable: Memtable, @@ -68,10 +72,12 @@ impl Lsm { todo!("memtable → L0 newest-first → deeper levels; L1+ disjoint ⇒ pick by key range; count stats") } + #[allow(dead_code, reason = "called from put() once you implement it")] fn flush(&mut self) -> std::io::Result<()> { todo!("write memtable to a new L0 SST via SstWriter; add finish() bytes to stats") } + #[allow(dead_code, reason = "called from put() once you implement it")] fn maybe_compact(&mut self) -> std::io::Result<()> { todo!("per strategy: pick level, k-way merge runs (drop shadowed versions; drop tombstones ONLY into last level), replace inputs with output") } diff --git a/topics/04-lsm-deep-dive/notes.md b/topics/04-lsm-deep-dive/notes.md index c9c73c9..96c7bb1 100644 --- a/topics/04-lsm-deep-dive/notes.md +++ b/topics/04-lsm-deep-dive/notes.md @@ -1,5 +1,26 @@ # Topic 4 — notes +## No provided baseline in this topic — and why + +`write_amp` is the only binary here and it measures **your** LSM +(`src/lsm.rs` + `src/sst.rs`) in both of its lanes: leveled vs tiered +compaction. There is no third implementation to stand next to, so on a fresh +clone it prints two stub notices and exits — that is the intended state, not a +broken build, and it is why this topic has no lane in `./verify.sh`. + +The numbers to aim at are arithmetic you can do before writing any code, and +they are the point of the exercise: + +- **leveled**, ratio T, L levels: write amp ≈ `T/2 × L` (every byte is rewritten + about T/2 times per level it descends). At T=10 over 4 levels that is ~20×. +- **tiered**, K runs per level: write amp ≈ `L` (a byte is written once per + level), so ~4× — but read amp becomes ~`K × L` runs to probe, and space amp + rises because shadowed versions survive longer. + +Predict both, then check the `describe()` output against them. If your leveled +number comes out near 4× or your tiered number near 20×, you have the strategies +backwards. For the *provided* measurements in the LSM family, topic 1's lane +prices fjall (an LSM) against redb (a B-tree) end to end. ## Predictions (fill BEFORE running write_amp) At 10M ops, 3.3M distinct keys, 100B values, 1MB memtable, ratio 10 / K=4: diff --git a/topics/05-durability-wal/README.md b/topics/05-durability-wal/README.md index 32c11a2..2c838a3 100644 --- a/topics/05-durability-wal/README.md +++ b/topics/05-durability-wal/README.md @@ -5,6 +5,30 @@ > style redo), turso/SQLite (WAL with checksum chain), LMDB (topic 3: no log at > all), redis (AOF command log + fork snapshots). +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin fsync_ladder` on APFS. macOS has no `fdatasync`, so +the rungs are `write()`, `fsync`, and `F_FULLFSYNC`: + +``` +rung p50 p99 p99.9 implied max commits/s +write() only 1.17 µs 4.54 µs 14.46 µs 856898 +fsync 22.67 µs 56.73 µs 157.06 µs 44109 +F_FULLFSYNC 2.97 ms 3.61 ms 9.89 ms 337 +``` + +**856 898, then 44 109, then 337 commits per second — and only the last row is +actually durable on this hardware.** The middle rung is the one that ruins +people: `fsync` on macOS returns when the data reaches the *drive*, not when the +drive has committed it to stable media, so the write can still be lost in the +disk's volatile cache. `F_FULLFSYNC` forces the cache flush and costs 131× more +than the call most code makes while believing it is safe. + +337 commits/s is the number to carry forward. It is a hard ceiling on any design +that syncs once per transaction, no matter how fast everything above it is — +which is why group commit is structural rather than an optimization, and why +topic 15's follower-fsync table bottoms out at 341 entries/s on the same box. + ## Outcomes By the end you can: diff --git a/topics/05-durability-wal/experiments/Cargo.lock b/topics/05-durability-wal/experiments/Cargo.lock new file mode 100644 index 0000000..bcb0218 --- /dev/null +++ b/topics/05-durability-wal/experiments/Cargo.lock @@ -0,0 +1,826 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "durability-experiments" +version = "0.1.0" +dependencies = [ + "crc32fast", + "criterion", + "hdrhistogram", + "libc", + "rand", + "tempfile", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hdrhistogram" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d1053f4708f0af3cf9fc5bffc7e68a914a3c45becb231c80068c9c3f78bea" +dependencies = [ + "base64", + "byteorder", + "crossbeam-channel", + "flate2", + "nom", + "num-traits", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/topics/05-durability-wal/experiments/src/bin/crash_test.rs b/topics/05-durability-wal/experiments/src/bin/crash_test.rs index 7a79e50..3c5faa7 100644 --- a/topics/05-durability-wal/experiments/src/bin/crash_test.rs +++ b/topics/05-durability-wal/experiments/src/bin/crash_test.rs @@ -47,17 +47,24 @@ fn child(wal_path: &Path, ack_path: &Path) -> ! { } fn round(n: usize) -> bool { + round_inner(n, false) +} + +/// `quiet` silences the child's stderr — used only for the one probe round +/// that detects an unimplemented `Wal`, so a reader who has started the +/// exercise still sees their own child's panics during the real rounds. +fn round_inner(n: usize, quiet: bool) -> bool { let dir = tempfile::tempdir().unwrap(); let wal_path = dir.path().join("wal"); let ack_path = dir.path().join("ack"); let exe = std::env::current_exe().unwrap(); - let mut kid = Command::new(exe) - .arg("child") - .arg(&wal_path) - .arg(&ack_path) - .spawn() - .expect("spawn child"); + let mut cmd = Command::new(exe); + cmd.arg("child").arg(&wal_path).arg(&ack_path); + if quiet { + cmd.stderr(std::process::Stdio::null()); + } + let mut kid = cmd.spawn().expect("spawn child"); let ms = rand::thread_rng().gen_range(5..80); std::thread::sleep(std::time::Duration::from_millis(ms)); @@ -103,6 +110,24 @@ fn main() { child(Path::new(&args[2]), Path::new(&args[3])); } + // This whole binary is the topic's acceptance test for YOUR src/wal.rs: + // there is no provided lane to fall back on, so probe once and explain + // the state rather than dumping a panic trace 100 times. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let implemented = std::panic::catch_unwind(|| round_inner(0, true)).is_ok(); + std::panic::set_hook(prev); + if !implemented { + println!( + "[stub — implement src/wal.rs to unlock the crash matrix]\n\n\ + This binary kill -9's a child mid-commit {ROUNDS} times and checks two\n\ + things after each replay: no acknowledged write is lost, and no\n\ + transaction is half-applied. `cargo test` is the smaller specification;\n\ + \"Done when: {ROUNDS}/{ROUNDS} crash rounds pass\" is this binary." + ); + return; + } + let mut passed = 0; for n in 1..=ROUNDS { if round(n) { diff --git a/topics/05-durability-wal/notes.md b/topics/05-durability-wal/notes.md index 140671b..7f91490 100644 --- a/topics/05-durability-wal/notes.md +++ b/topics/05-durability-wal/notes.md @@ -1,5 +1,29 @@ # Topic 5 notes — durability, WAL, crash recovery +## Baseline (provided lane, Apple M3 Pro / APFS, measured 2026-07-28) + +`cargo run --release --bin fsync_ladder`. macOS, so the rungs are `write()`, +`fsync`, and `F_FULLFSYNC` (there is no `fdatasync`; the bench compiles the +Linux rung out). + +| rung | p50 | p99 | p99.9 | implied max commits/s | +|---|---|---|---|---| +| `write()` only | 1.17 µs | 4.54 µs | 14.46 µs | 856 898 | +| `fsync` | 22.67 µs | 56.73 µs | 157.06 µs | 44 109 | +| `F_FULLFSYNC` | **2.97 ms** | 3.61 ms | 9.89 ms | **337** | + +**Three rungs, a 2540× spread in the last column, and only the bottom one is +actually durable on this hardware.** The middle rung is the trap: `fsync` on +macOS returns once the data reaches the *drive*, not once the drive has +committed it to stable media — the write can still be sitting in the disk's +volatile cache. `F_FULLFSYNC` is what forces a cache flush, and it costs 131× +more than the `fsync` that most code calls and believes. + +337 commits/s is the number to keep. Any single-threaded design that fsyncs per +transaction is capped there regardless of how fast the rest of the engine is, +which is why group commit is not an optimization but a structural requirement — +and why topic 15's follower-fsync table looks the way it does. + ## Predictions (fill BEFORE running fsync_ladder) | Rung | Predicted p50 | Measured p50 | Measured p99 | diff --git a/topics/06-buffer-pool/README.md b/topics/06-buffer-pool/README.md index 0d30358..6e98efe 100644 --- a/topics/06-buffer-pool/README.md +++ b/topics/06-buffer-pool/README.md @@ -7,6 +7,29 @@ > Plus redis's answer to a different question: not page caching but > *allocator accounting* (zmalloc + jemalloc + active defrag). +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin pool_vs_mmap` — 1 GiB file, 2 M Zipf(0.99) page +reads, 8 bytes touched per page so the access dominates the copy: + +``` +mmap p50 42 ns p99 1500 ns p99.9 4459 ns max 181887 ns +``` + +**42 nanoseconds at the median, 182 microseconds at the max — a 4300× spread, +and all of it lives in the tail.** The median is a hit on a page the kernel +already had resident, which is essentially free and exactly why mmap is so +tempting for a storage engine. The p99.9 and the max are minor page faults: a +trap, a read, a TLB shootdown — and, crucially, an event the database cannot +see, schedule around, or prefetch ahead of. + +That asymmetry is the entire argument of *"Are You Sure You Want to Use MMAP in +Your DBMS?"*, and it is why every serious engine reimplements paging it could +have had for free. Note the handicap when you compare your own pool to this +row: mmap gets the whole machine's page cache here, while your pool will be +held to 256 MiB against a 1 GiB file. If your pool still wins the tail under +that disadvantage, the result is conclusive. + ## Outcomes By the end you can: diff --git a/topics/06-buffer-pool/experiments/Cargo.lock b/topics/06-buffer-pool/experiments/Cargo.lock new file mode 100644 index 0000000..cb63f61 --- /dev/null +++ b/topics/06-buffer-pool/experiments/Cargo.lock @@ -0,0 +1,852 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "buffer-pool-experiments" +version = "0.1.0" +dependencies = [ + "criterion", + "hdrhistogram", + "memmap2", + "rand", + "rand_distr", + "tempfile", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hdrhistogram" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d1053f4708f0af3cf9fc5bffc7e68a914a3c45becb231c80068c9c3f78bea" +dependencies = [ + "base64", + "byteorder", + "crossbeam-channel", + "flate2", + "nom", + "num-traits", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/topics/06-buffer-pool/experiments/src/bin/pool_vs_mmap.rs b/topics/06-buffer-pool/experiments/src/bin/pool_vs_mmap.rs index 4fc8867..37e9a96 100644 --- a/topics/06-buffer-pool/experiments/src/bin/pool_vs_mmap.rs +++ b/topics/06-buffer-pool/experiments/src/bin/pool_vs_mmap.rs @@ -52,6 +52,18 @@ fn report(name: &str, hist: &Histogram) { ); } +/// Run an exercise lane, reporting unimplemented `todo!()`s as a note +/// instead of a crash, so the provided lane above always prints. +fn stub_lane(name: &str, f: impl FnOnce()) { + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { + println!("[stub — implement the todo!()s to unlock {name}]\n"); + } +} + fn main() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("big.dat"); @@ -59,7 +71,9 @@ fn main() { make_file(&path); let pages = zipf_pages(OPS); - // -- mmap --------------------------------------------------------------- + // -- lane 1 (PROVIDED): mmap --------------------------------------------- + // The kernel's page cache doing the paging for you — and, per CIDR '22, + // doing it with no idea which pages matter to you. let file = std::fs::File::open(&path).unwrap(); let map = unsafe { memmap2::Mmap::map(&file).unwrap() }; let mut hist = Histogram::::new_with_bounds(1, 60_000_000_000, 3).unwrap(); @@ -72,24 +86,26 @@ fn main() { } report("mmap", &hist); - // -- buffer pool ---------------------------------------------------------- - let mut pool = BufferPool::open(&path, POOL_PAGES).unwrap(); - let mut hist = Histogram::::new_with_bounds(1, 60_000_000_000, 3).unwrap(); - for &p in &pages { - let t = Instant::now(); - let v = pool - .with_page(p, |b| u64::from_le_bytes(b[..8].try_into().unwrap())) - .unwrap(); - hist.record(t.elapsed().as_nanos() as u64).unwrap(); - sink = sink.wrapping_add(v); - } - report("pool(CLOCK)", &hist); - let s = pool.stats(); - println!( - "pool hit rate: {:.2}% ({} hits / {} misses)", - 100.0 * s.hits as f64 / (s.hits + s.misses) as f64, - s.hits, - s.misses - ); + // -- lane 2 (EXERCISE): your buffer pool --------------------------------- + stub_lane("the CLOCK buffer pool (src/buffer_pool.rs)", || { + let mut pool = BufferPool::open(&path, POOL_PAGES).unwrap(); + let mut hist = Histogram::::new_with_bounds(1, 60_000_000_000, 3).unwrap(); + for &p in &pages { + let t = Instant::now(); + let v = pool + .with_page(p, |b| u64::from_le_bytes(b[..8].try_into().unwrap())) + .unwrap(); + hist.record(t.elapsed().as_nanos() as u64).unwrap(); + sink = sink.wrapping_add(v); + } + report("pool(CLOCK)", &hist); + let s = pool.stats(); + println!( + "pool hit rate: {:.2}% ({} hits / {} misses)", + 100.0 * s.hits as f64 / (s.hits + s.misses) as f64, + s.hits, + s.misses + ); + }); std::hint::black_box(sink); } diff --git a/topics/06-buffer-pool/notes.md b/topics/06-buffer-pool/notes.md index c33e7aa..6f6bb91 100644 --- a/topics/06-buffer-pool/notes.md +++ b/topics/06-buffer-pool/notes.md @@ -1,5 +1,29 @@ # Topic 6 notes — buffer pool & memory management +## Baseline (provided lane, Apple M3 Pro, measured 2026-07-28) + +`cargo run --release --bin pool_vs_mmap` — 1 GiB file, 2 M Zipf(0.99) page +reads, 8 bytes touched per page so the access rather than the copy dominates. + +| impl | p50 | p99 | p99.9 | max | +|---|---|---|---|---| +| mmap | 42 ns | 1500 ns | 4459 ns | **181 887 ns** | +| pool (CLOCK, yours) | | | | stub | + +**p50 42 ns, max 182 µs: a 4300× spread, and every bit of it is the tail.** The +median is a hit on a page the kernel already had resident — essentially free, +which is exactly why mmap is so tempting. The p99.9 and the max are minor page +faults: a trap into the kernel, a read, a TLB shootdown, and no way for the +database to know it happened, schedule around it, or prefetch ahead of it. That +is CIDR '22's argument in one row. + +Read the handicap honestly before you compare your pool to this: the mmap side +gets the *whole* machine's page cache (there is no per-process cap on macOS), +while your pool will be held to a 256 MiB budget against a 1 GiB file. mmap is +playing with an advantage. If your pool still wins the tail, that is conclusive; +if mmap wins the median, that is expected and the interesting question is why +the tail behaves differently from the median at all. + ## Predictions (fill BEFORE running) - pool_vs_mmap p50: mmap ___ ns vs pool ___ ns (who wins the median and why?) diff --git a/topics/07-networking-protocols/README.md b/topics/07-networking-protocols/README.md index 7effa55..725da42 100644 --- a/topics/07-networking-protocols/README.md +++ b/topics/07-networking-protocols/README.md @@ -5,6 +5,35 @@ > side: one loop, many sockets, and a protocol designed to be parsed with > `memchr`. +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin loopback_bench` — 200 000 ops per depth, 32 B in / +8 B out, one connection, `TCP_NODELAY`, and deliberately **no protocol parsing +and no store**. Pipeline depth P is the only variable: + +``` + P ops/s µs per request syscalls per op vs P=1 + 1 44088 22.68 2.000 1.0x + 2 88262 11.33 1.000 2.0x + 8 353067 2.83 0.250 8.0x + 32 1517490 0.66 0.062 34.4x + 64 2919728 0.34 0.031 66.2x + 256 12321414 0.08 0.008 279.5x +``` + +**Identical work, none of it useful, and throughput spans 279×.** Nothing here +parses, stores or computes; the whole curve is syscalls, wakeups and round +trips. So when a benchmark result arrives without its pipeline depth, it is not +a measurement of a system — a `-P 1` number and a `-P 64` number describe two +different bottlenecks in the same process. + +Two details worth keeping. The scaling is slightly *super*-linear against the +2/P syscall floor (279× at P=256, not 256×), because bigger writes amortize +per-byte costs too. And per-request latency *improves* with depth, 22.68 → 0.08 +µs: client-side batching is not the usual throughput-for-latency trade, because +what it removes was pure round-trip overhead. Server-side batching — group +commit, topic 5 — is the trade. This is not. + ## Outcomes By the end you can: diff --git a/topics/07-networking-protocols/experiments/Cargo.lock b/topics/07-networking-protocols/experiments/Cargo.lock new file mode 100644 index 0000000..28be260 --- /dev/null +++ b/topics/07-networking-protocols/experiments/Cargo.lock @@ -0,0 +1,219 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "networking-experiments" +version = "0.1.0" +dependencies = [ + "bytes", + "tokio", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/topics/07-networking-protocols/experiments/src/bin/loopback_bench.rs b/topics/07-networking-protocols/experiments/src/bin/loopback_bench.rs new file mode 100644 index 0000000..940acd0 --- /dev/null +++ b/topics/07-networking-protocols/experiments/src/bin/loopback_bench.rs @@ -0,0 +1,130 @@ +//! Lane 1 (PROVIDED): what a request actually costs when the work is zero. +//! +//! cargo run --release --bin loopback_bench +//! +//! This is the measurement the whole topic rests on, and it deliberately +//! contains no protocol parsing at all — the frame is a fixed 32 bytes in, +//! 8 bytes out, so nothing here depends on your `src/resp.rs`. What is left +//! when you remove the parser and the store is the part nobody budgets for: +//! the syscalls, the wakeups, and the round trip. +//! +//! The knob is pipeline depth P — how many requests the client puts on the +//! wire before it reads any reply. At P=1 every request pays a full +//! write→wake→read→write→wake→read round trip. At P=64 that same cost is +//! amortized over 64 requests, which is why `redis-benchmark -P 64` prints +//! numbers that look like a different database from `-P 1`. +//! +//! Predict in notes.md BEFORE running: +//! - the P=1 → P=64 throughput ratio (redis's own docs claim ~10x) +//! - whether per-request latency gets better or worse as P grows, and why +//! - where the curve stops improving, and what has become the bottleneck +//! +//! Then run `server.rs` under `redis-benchmark -P 1` and `-P 64` and check +//! that the shape of this curve survives a real protocol on top of it. + +use std::time::Instant; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +const REQ: usize = 32; +const REP: usize = 8; +const OPS: usize = 200_000; +const DEPTHS: [usize; 6] = [1, 2, 8, 32, 64, 256]; + +/// The server side: read whatever arrived, reply once per complete frame, +/// flush once per read. This is the "flush only when drained" trick from +/// redis's handleClientsWithPendingWrites, which is the entire reason +/// pipelining pays off — see the design notes in server.rs. +async fn serve(mut stream: TcpStream) { + stream.set_nodelay(true).unwrap(); + let mut inbuf = vec![0u8; 1 << 16]; + let mut filled = 0usize; + let mut outbuf = Vec::with_capacity(1 << 16); + loop { + let n = match stream.read(&mut inbuf[filled..]).await { + Ok(0) | Err(_) => return, + Ok(n) => n, + }; + filled += n; + let frames = filled / REQ; + if frames > 0 { + outbuf.clear(); + for _ in 0..frames { + outbuf.extend_from_slice(&[b'+'; REP]); + } + if stream.write_all(&outbuf).await.is_err() { + return; + } + let consumed = frames * REQ; + inbuf.copy_within(consumed..filled, 0); + filled -= consumed; + } + } +} + +/// The client side: keep `depth` requests in flight, then drain their +/// replies. Returns (ops/sec, mean per-request latency in µs). +async fn client(addr: std::net::SocketAddr, depth: usize) -> (f64, f64) { + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream.set_nodelay(true).unwrap(); + let req = vec![b'x'; REQ * depth]; + let mut rep = vec![0u8; REP * depth]; + let batches = OPS / depth; + + // warm the connection so the first batch's page faults are not in the number + stream.write_all(&req[..REQ]).await.unwrap(); + stream.read_exact(&mut rep[..REP]).await.unwrap(); + + let start = Instant::now(); + for _ in 0..batches { + stream.write_all(&req).await.unwrap(); + stream.read_exact(&mut rep).await.unwrap(); + } + let secs = start.elapsed().as_secs_f64(); + let ops = (batches * depth) as f64; + // per-request latency as the client experiences it: a batch's round trip + // divided over the requests that shared it + (ops / secs, secs / ops * 1e6) +} + +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] +async fn main() -> std::io::Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(serve(stream)); + } + }); + + println!("{OPS} ops per depth, {REQ} B request / {REP} B reply, loopback TCP,"); + println!("TCP_NODELAY on, one connection, no protocol parsing and no store.\n"); + println!( + "{:>5} {:>14} {:>16} {:>18} {:>12}", + "P", "ops/s", "µs per request", "syscalls per op", "vs P=1" + ); + + let mut base = 0.0; + for depth in DEPTHS { + let (ops, us) = client(addr, depth).await; + if base == 0.0 { + base = ops; + } + println!( + "{depth:>5} {ops:>14.0} {us:>16.2} {:>18.3} {:>11.1}x", + 2.0 / depth as f64, + ops / base + ); + } + + println!("\nnotes:"); + println!("- 'syscalls per op' is the floor 2/P (one write + one read per batch);"); + println!(" the throughput column should track it until something else binds"); + println!("- the µs column is per REQUEST, not per batch: pipelining improves"); + println!(" throughput and per-request latency at the same time, which is why"); + println!(" a client-side batch is not the same trade as a server-side one"); + println!("- record the P=1 and P=64 rows in notes.md, then compare them to"); + println!(" redis-benchmark against real redis and against server.rs"); + Ok(()) +} diff --git a/topics/07-networking-protocols/experiments/src/bin/server.rs b/topics/07-networking-protocols/experiments/src/bin/server.rs index f860c73..d1cd196 100644 --- a/topics/07-networking-protocols/experiments/src/bin/server.rs +++ b/topics/07-networking-protocols/experiments/src/bin/server.rs @@ -111,6 +111,32 @@ async fn handle(stream: TcpStream, store: Store) -> std::io::Result<()> { #[tokio::main] async fn main() -> std::io::Result<()> { + // Every byte in and out of this server goes through YOUR src/resp.rs, so + // there is nothing to serve until it is implemented. Probe once rather + // than binding a port and panicking on the first client. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let implemented = std::panic::catch_unwind(|| { + let mut buf = BytesMut::from(&b"*1\r\n$4\r\nPING\r\n"[..]); + let _ = parse(&mut buf); + }) + .is_ok(); + std::panic::set_hook(prev); + if !implemented { + println!( + "[stub — implement src/resp.rs to unlock the server]\n\n\ + `cargo test` is the specification: RESP framing, partial reads that must\n\ + not consume the buffer, and the inline-command case. Once it passes:\n\n\ + \x20 cargo run --release --bin server\n\ + \x20 redis-benchmark -p 7379 -t get,set -n 1000000 -P 1\n\ + \x20 redis-benchmark -p 7379 -t get,set -n 1000000 -P 64\n\n\ + For the measurement this topic actually turns on — what a request costs\n\ + when the protocol and the store are removed — run the provided lane:\n\n\ + \x20 cargo run --release --bin loopback_bench" + ); + return Ok(()); + } + let store: Store = Arc::new((0..SHARDS).map(|_| RwLock::new(HashMap::new())).collect()); let listener = TcpListener::bind("127.0.0.1:7379").await?; println!("listening on 127.0.0.1:7379 — try: redis-cli -p 7379 ping"); diff --git a/topics/07-networking-protocols/notes.md b/topics/07-networking-protocols/notes.md index d9547c3..58cdb2f 100644 --- a/topics/07-networking-protocols/notes.md +++ b/topics/07-networking-protocols/notes.md @@ -2,6 +2,38 @@ Predict FIRST, then measure. Numbers without predictions are just trivia. +## Baseline (provided lane, Apple M3 Pro, loopback, measured 2026-07-28) + +`cargo run --release --bin loopback_bench` — 200 000 ops per depth, 32 B +request / 8 B reply, one connection, `TCP_NODELAY` on, **no protocol parsing +and no store**. Pipeline depth P is the only knob. + +| P | ops/s | µs per request | syscalls per op | vs P=1 | +|---|---|---|---|---| +| 1 | 44 088 | 22.68 | 2.000 | 1.0× | +| 2 | 88 262 | 11.33 | 1.000 | 2.0× | +| 8 | 353 067 | 2.83 | 0.250 | 8.0× | +| 32 | 1 517 490 | 0.66 | 0.062 | 34.4× | +| 64 | 2 919 728 | 0.34 | 0.031 | 66.2× | +| 256 | 12 321 414 | 0.08 | 0.008 | **279.5×** | + +**Identical work, zero of it useful, and the throughput spans 279×.** Nothing +in this benchmark parses, stores, or computes anything — the entire curve is +syscalls, wakeups and round trips. That is the number to hold in your head the +next time a benchmark result is quoted without its pipeline depth: a +`redis-benchmark -P 64` figure and a `-P 1` figure are not the same measurement +of the same system, they are measurements of different bottlenecks. + +Two things worth noticing: + +- The scaling is slightly **super**-linear against the 2/P syscall floor (279× + at P=256, not 256×). Larger writes amortize per-byte costs too, so the + syscall count is a floor on the improvement, not a ceiling. +- Per-request latency *improves* with depth, 22.68 µs → 0.08 µs. Client-side + batching is not the usual throughput-for-latency trade, because the queueing + it removes was pure round-trip overhead. Server-side batching (group commit, + topic 5) is the trade; this is not. + ## Predictions (fill in BEFORE running anything) | Measurement | Prediction | Actual | Surprised? | diff --git a/topics/07-networking-protocols/reading-bolt-packstream.md b/topics/07-networking-protocols/reading-bolt-packstream.md index 57cfde4..5f47495 100644 --- a/topics/07-networking-protocols/reading-bolt-packstream.md +++ b/topics/07-networking-protocols/reading-bolt-packstream.md @@ -194,6 +194,14 @@ Step 5 diagram's order. result set) and which need a Bolt twin? Sketch the `bolt_reply_*`-equivalent trait your result set must implement. +## Done when + +- [ ] You can write a PackStream marker byte from memory and decode type and size from its nibbles. +- [ ] You can explain how one structure mechanism serves both protocol messages and graph types, and why that is more than an aesthetic choice. +- [ ] You can say what RUN/PULL buys by splitting execute from fetch, and what the server must therefore hold between them. +- [ ] You can explain how chunking substitutes for a message length prefix, and what that costs a parser. +- [ ] You wrote answers to all questions in notes.md, including the honest cost list for why FalkorDB removed Bolt. + ## References **Papers** diff --git a/topics/08-transactions-mvcc/README.md b/topics/08-transactions-mvcc/README.md index 879402f..8a8bf6b 100644 --- a/topics/08-transactions-mvcc/README.md +++ b/topics/08-transactions-mvcc/README.md @@ -7,6 +7,32 @@ must die?* Budget: ~12 h. Order: §1 anomalies → §2 three concurrency schools → §3 postgres on-disk MVCC → §4 in-memory MVCC → experiments → M8. +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin txn_bench` — 4 threads × 50 000 transactions × 4 +ops, against one global `Mutex`: + +``` +mix global-lock/s mvcc txn/s aborts +read-heavy 95/5, 10K keys 623454 — — +write-heavy 50/50, 10K keys 594264 — — +write-heavy 50/50, 64 keys (HOT) 676691 — — +``` + +**The baseline barely moves across three workloads that should be wildly +different, and the flatness is the finding.** A single mutex cannot benefit from +the read-heavy row (95% of those operations could have run concurrently, and +none of them did) and cannot be hurt by the HOT row (everything was already +contending on one lock, so shrinking the keyspace to 64 changes nothing). The +hot row is even the *fastest*, because a 64-key working set fits in cache. + +That gives you a sharp pair of predictions to write down before you implement +MVCC: it should crush row 1, since readers never block writers. It may well +*lose* row 3, where first-committer-wins converts key contention into aborted +work the mutex never had to redo. Find the keyspace size where the crossover +happens — that number, not the read-heavy speedup, is what decides whether MVCC +is the right answer for a given workload. + ## 1. Isolation levels are defined by their bugs Read the levels bottom-up, as "which anomalies are permitted": diff --git a/topics/08-transactions-mvcc/experiments/Cargo.lock b/topics/08-transactions-mvcc/experiments/Cargo.lock new file mode 100644 index 0000000..c1493f5 --- /dev/null +++ b/topics/08-transactions-mvcc/experiments/Cargo.lock @@ -0,0 +1,133 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "mvcc-experiments" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/08-transactions-mvcc/experiments/src/bin/txn_bench.rs b/topics/08-transactions-mvcc/experiments/src/bin/txn_bench.rs index 0ee6395..40fa151 100644 --- a/topics/08-transactions-mvcc/experiments/src/bin/txn_bench.rs +++ b/topics/08-transactions-mvcc/experiments/src/bin/txn_bench.rs @@ -1,7 +1,7 @@ //! Txn throughput: your MVCC vs one big lock. //! -//! Runs the global-lock baseline immediately; the MVCC half panics until -//! src/mvcc.rs is implemented. Then: cargo run --release --bin txn_bench +//! Runs the global-lock baseline immediately; the MVCC half reports as a stub +//! until src/mvcc.rs is implemented. Then: cargo run --release --bin txn_bench //! //! Predict in notes.md BEFORE running: //! - read-heavy (95/5): who wins, by how much? (MVCC readers never block; @@ -118,13 +118,35 @@ fn main() { THREADS, TXNS_PER_THREAD, OPS_PER_TXN ); println!("{:<36} {:>14} {:>14} {:>9}", "mix", "global-lock/s", "mvcc txn/s", "aborts"); + + // lane 1 (PROVIDED) is the global-lock column; the MVCC column is the + // exercise, so it reports as a stub and leaves the baseline standing. + let mut mvcc_ok = true; for &mix in MIXES { let lock_tps = run_global_lock(mix); - let (mvcc_tps, aborts) = run_mvcc(mix); - println!( - "{:<36} {:>14.0} {:>14.0} {:>9}", - mix.name, lock_tps, mvcc_tps, aborts - ); + let mvcc = if mvcc_ok { + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = std::panic::catch_unwind(|| run_mvcc(mix)); + std::panic::set_hook(prev); + mvcc_ok = r.is_ok(); + r.ok() + } else { + None + }; + match mvcc { + Some((mvcc_tps, aborts)) => println!( + "{:<36} {:>14.0} {:>14.0} {:>9}", + mix.name, lock_tps, mvcc_tps, aborts + ), + None => println!( + "{:<36} {:>14.0} {:>14} {:>9}", + mix.name, lock_tps, "—", "—" + ), + } + } + if !mvcc_ok { + println!("\n[stub — implement src/mvcc.rs to unlock the mvcc column]"); } println!("\nRecord all three rows + the abort counts in notes.md."); } diff --git a/topics/08-transactions-mvcc/notes.md b/topics/08-transactions-mvcc/notes.md index 0659ea2..d2dc17b 100644 --- a/topics/08-transactions-mvcc/notes.md +++ b/topics/08-transactions-mvcc/notes.md @@ -2,6 +2,30 @@ Predict FIRST, then measure. +## Baseline (provided lane, Apple M3 Pro, measured 2026-07-28) + +`cargo run --release --bin txn_bench` — 4 threads × 50 000 txns × 4 ops, one +global `Mutex` as the baseline. + +| mix | global-lock txn/s | mvcc txn/s | aborts | +|---|---|---|---| +| read-heavy 95/5, 10K keys | 623 454 | stub | stub | +| write-heavy 50/50, 10K keys | 594 264 | stub | stub | +| write-heavy 50/50, 64 keys (HOT) | 676 691 | stub | stub | + +**The baseline is flat across all three mixes — and that flatness is the +finding.** A single mutex does not care what the transactions do or how much +they collide, because it has already serialized them: the read-heavy row cannot +exploit the fact that 95% of operations are reads that could have run +concurrently, and the HOT row is not penalised for contending on 64 keys +because everything was contending on one lock anyway. It even comes out +*fastest*, since a 64-key working set is cache-resident. + +That gives you a sharp prediction to write down before implementing MVCC: your +version should crush the baseline on row 1 (readers never block) and may well +*lose* on row 3, where first-committer-wins turns key contention into aborted +work that the mutex never has to redo. Find the crossover keyspace size. + ## Predictions (fill in BEFORE running txn_bench) | Measurement | Prediction | Actual | Surprised? | diff --git a/topics/09-concurrency/README.md b/topics/09-concurrency/README.md index e7b35c8..5506b9a 100644 --- a/topics/09-concurrency/README.md +++ b/topics/09-concurrency/README.md @@ -7,6 +7,40 @@ question for NANOSECONDS: two threads, one cache line, who wins? Budget: ~12 h. Order: §1 vocabulary → §2 memory ordering → §3 latch protocols → §4 reclamation → §5 code → experiments → M9. +## The problem, measured (bench lanes 1 and 2, provided — run today) + +`cargo run --release --bin false_sharing` and `--bin scaling`: + +``` +8 threads, 5M increments each, each on its OWN counter +packed 202.7 ms 197.4 M inc/s +pad64 20.4 ms 1957.6 M inc/s +pad128 11.4 ms 3502.9 M inc/s + +Mops/s total, 90/10 read/write, keyspace 100000 +impl 1t 2t 4t 8t 16t +global 8.65 5.32 2.84 2.86 2.96 +sharded 11.63 8.40 8.66 11.22 12.65 +crossbeam 4.21 9.07 14.39 14.82 19.28 +``` + +**The global mutex gets 2.9× slower as you add cores.** Not "fails to scale" — +negative: 8.65 Mops/s on one thread, 2.96 on sixteen. The cores spend their time +moving the lock's cache line between caches and parking each other rather than +doing work, and that line shape (peak at one thread, decay after) is the single +most useful signature to recognise in production, because it means the fix is +never "add threads". + +The counter table is the same physics one level down: three layouts where every +thread owns its own counter and touches nobody else's, spanning 17.8×. `pad64` +— the x86-default `CachePadded` — is still 1.8× off `pad128`, because M-series +coherence granularity is 128 B. Check that assumption on your own hardware +before trusting any padding. + +And note which structure is *slowest* single-threaded: crossbeam's lock-free +set, at 4.21 vs the mutex's 8.65. Atomics and epoch bookkeeping cost real +sequential performance to buy a slope. That trade is the topic. + ## 1. Latches vs locks (say it right) | | lock (topic 8) | latch (this topic) | diff --git a/topics/09-concurrency/experiments/Cargo.lock b/topics/09-concurrency/experiments/Cargo.lock new file mode 100644 index 0000000..c082c16 --- /dev/null +++ b/topics/09-concurrency/experiments/Cargo.lock @@ -0,0 +1,160 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "concurrency-experiments" +version = "0.1.0" +dependencies = [ + "crossbeam-epoch", + "crossbeam-skiplist", + "rand", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/09-concurrency/experiments/src/bin/scaling.rs b/topics/09-concurrency/experiments/src/bin/scaling.rs index 4590c74..a6aa53c 100644 --- a/topics/09-concurrency/experiments/src/bin/scaling.rs +++ b/topics/09-concurrency/experiments/src/bin/scaling.rs @@ -119,12 +119,32 @@ fn main() { ("crossbeam", Box::new(|| Arc::new(SkipSet::new()))), ("mine", Box::new(|| Arc::new(ConcurrentSet::new()))), ]; + // "global", "sharded" and "crossbeam" are provided — they are the three + // reference points. "mine" is the exercise (src/concurrent_set.rs) and + // reports as a stub until it is implemented, so the other three rows + // always land. for (name, mk) in contestants { - print!("{name:<10}"); - for &t in &thread_counts { - print!(" {:>8.2}", run(mk(), t)); + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let row = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + thread_counts + .iter() + .map(|&t| run(mk(), t)) + .collect::>() + })); + std::panic::set_hook(prev); + match row { + Ok(vals) => { + print!("{name:<10}"); + for v in vals { + print!(" {v:>8.2}"); + } + println!(); + } + Err(_) => println!( + "{name:<10} [stub — implement src/concurrent_set.rs to unlock this row]" + ), } - println!(); } println!("\nRecord the table + line shapes in notes.md."); } diff --git a/topics/09-concurrency/notes.md b/topics/09-concurrency/notes.md index 3240495..22333ea 100644 --- a/topics/09-concurrency/notes.md +++ b/topics/09-concurrency/notes.md @@ -2,19 +2,50 @@ Predict FIRST, then measure. -## Measured already (false_sharing, provided binary — this Mac, 8 threads) +## Baseline (provided lanes, Apple M3 Pro, measured 2026-07-28) -| layout | time | rate | -|---|---|---| -| packed | 636 ms | 63 M inc/s | -| pad64 | 24 ms | 1697 M inc/s | -| pad128 | 11 ms | 3707 M inc/s | +### false_sharing — 8 threads, 5 M increments each, own counter per thread -- **59×** packed → pad128. "Independent" counters in one line are not +| layout | time | rate | vs pad128 | +|---|---|---|---| +| packed | 202.7 ms | 197.4 M inc/s | 17.8× slower | +| pad64 | 20.4 ms | 1957.6 M inc/s | 1.8× slower | +| pad128 | 11.4 ms | 3502.9 M inc/s | — | + +- **17.8× packed → pad128.** "Independent" counters sharing a line are not independent — this is the whole reason redis pads `used_memory`. -- **pad64 is still 2.2× slower than pad128**: Apple M-series coherence - granularity is 128 B. `#[repr(align(64))]`, the x86 default, HALF-fixes - false sharing on this machine. Check every CachePadded assumption. +- **pad64 is still 1.8× slower than pad128**: Apple M-series coherence + granularity is 128 B. `#[repr(align(64))]`, the x86 default, only HALF-fixes + false sharing on this machine. Check every `CachePadded` assumption against + the hardware you are actually on. +- **Run-to-run variance is large on the packed row** and worth knowing about: + an earlier run of this same binary recorded 636 ms / 63 M inc/s, i.e. a 59× + ratio rather than 17.8×. Contended-line throughput depends on how the threads + happen to interleave, so treat the *order of magnitude* as the finding and + quote a range, not a point, if you cite it. + +### scaling — 90/10 read/write mix, keyspace 100 000, Mops/s total + +| impl | 1t | 2t | 4t | 8t | 16t | +|---|---|---|---|---|---| +| global mutex | 8.65 | 5.32 | 2.84 | 2.86 | 2.96 | +| sharded ×16 | 11.63 | 8.40 | 8.66 | 11.22 | 12.65 | +| crossbeam SkipSet | 4.21 | 9.07 | 14.39 | 14.82 | 19.28 | +| mine | | | | | stub | + +**The global mutex gets 2.9× SLOWER going from 1 thread to 16.** Not "stops +scaling" — actually negative: 8.65 → 2.96 Mops/s. Adding cores to a +single-lock structure removes throughput, because the cores spend their time +transferring the lock's cache line and parking/unparking instead of working. +That line shape (peak at 1 thread, decay after) is the signature to recognise in +production: it means the fix is never "more threads". + +Two more shapes worth naming: sharding recovers most of it but is *non-monotonic* +(dips at 2t, recovers by 8t) because 16 shards with few threads is mostly +uncontended luck; and the lock-free skip set is the only one that is slowest at +1 thread (4.21, vs the mutex's 8.65) and fastest at 16 — atomics and epoch +bookkeeping cost real single-threaded performance to buy a slope. That trade is +the whole topic. ## Predictions (fill in BEFORE running scaling.rs) diff --git a/topics/10-query-planning/README.md b/topics/10-query-planning/README.md index d5b8719..6115341 100644 --- a/topics/10-query-planning/README.md +++ b/topics/10-query-planning/README.md @@ -8,6 +8,32 @@ Budget: ~12 h. Order: §1 pipeline → §2 rewrites → §3 join ordering → §4 cardinality (where it all goes wrong) → §5 architectures → code → experiments → M10. +## The problem, and why this topic has no measured opener + +This is the second of two topics whose only binary measures **your** code: +every plan `explain` prints comes out of your `src/planner.rs`, so a fresh clone +prints one stub notice and exits, and there is no lane in `./verify.sh`. + +That is not a gap to apologise for — it is what a planner is. The failure mode +of a query optimizer is not being slow, it is being *confidently wrong about row +counts*, and you cannot see that in a timing number. So the oracle here is +external and deliberately not Rust: + +``` + load the same 3-table schema into DuckDB + EXPLAIN the same 3 queries + diff the join orders against yours + every disagreement = one cardinality estimate to go find +``` + +The one number worth committing to before you start is on the third query +(`items ⋈ orders ⋈ users`, two selective filters on `users`): once those filters +are pushed down, does `{users, orders}` become a cheaper first join than +`{orders, items}`? Write the estimate down, then let `estimate()` grade you. +Getting the *direction* right matters more than getting the number right, and a +planner that gets the direction wrong will pick a plan that is orders of +magnitude slow — which is how this topic's mistakes actually show up. + ## 1. The pipeline ```mermaid diff --git a/topics/10-query-planning/experiments/Cargo.lock b/topics/10-query-planning/experiments/Cargo.lock new file mode 100644 index 0000000..5daafca --- /dev/null +++ b/topics/10-query-planning/experiments/Cargo.lock @@ -0,0 +1,25 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "planner-experiments" +version = "0.1.0" +dependencies = [ + "sqlparser", +] + +[[package]] +name = "sqlparser" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a875d8cd437cc8a97e9aeaeea352ec9a19aea99c23e9effb17757291de80b08" +dependencies = [ + "log", +] diff --git a/topics/10-query-planning/experiments/src/bin/explain.rs b/topics/10-query-planning/experiments/src/bin/explain.rs index c570802..f5627c0 100644 --- a/topics/10-query-planning/experiments/src/bin/explain.rs +++ b/topics/10-query-planning/experiments/src/bin/explain.rs @@ -91,6 +91,26 @@ fn main() { table(5_000_000, &[("order_id", 1_000_000), ("sku", 20_000)]), ); + // Every plan below comes out of YOUR src/planner.rs, so there is no + // provided lane here — probe once and explain the state instead of + // dumping a panic trace. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let implemented = + std::panic::catch_unwind(|| parse_and_plan("SELECT users.id FROM users").is_ok()).is_ok(); + std::panic::set_hook(prev); + if !implemented { + println!( + "[stub — implement src/planner.rs to unlock EXPLAIN]\n\n\ + This binary prints naive / pushed-down / reordered plans with cardinality\n\ + estimates for three queries, so you can see a predicate move and a join\n\ + order flip. `cargo test` is the specification; the third query is the\n\ + interesting one — predict in notes.md whether the selective users filter\n\ + makes {{users, orders}} cheaper than {{orders, items}} before you run it." + ); + return; + } + explain( "SELECT users.city FROM users, orders \ WHERE users.city = 7 AND users.id = orders.user_id", diff --git a/topics/10-query-planning/notes.md b/topics/10-query-planning/notes.md index 89e7bff..0127326 100644 --- a/topics/10-query-planning/notes.md +++ b/topics/10-query-planning/notes.md @@ -1,5 +1,22 @@ # Topic 10 notes — parsing, planning, optimization +## No provided baseline in this topic — and why + +`explain` is the only binary here and every plan it prints comes out of **your** +`src/planner.rs`, so on a fresh clone it prints one stub notice and exits. That +is the intended state, and it is why this topic has no lane in `./verify.sh`. + +The external baseline is deliberately not Rust: load the same three-table schema +into DuckDB, run `EXPLAIN` on the same three queries, and diff the join orders +against yours. Every disagreement is a lead — find which cardinality estimate +produced it. That comparison is worth more than a timing number here, because a +planner's failure mode is not being slow, it is being confidently wrong about +row counts. + +The one figure worth predicting before you start: for the third query +(`items ⋈ orders ⋈ users` with two selective filters on `users`), does pushing +those filters down make `{users, orders}` the cheaper first join than +`{orders, items}`? Write the estimate, then let `estimate()` grade you. ## Predictions (fill BEFORE running / reading) ### explain.rs query 3 (items ⋈ orders ⋈ users, users filtered to city=7 AND age=30) diff --git a/topics/11-execution-models/README.md b/topics/11-execution-models/README.md index d07e0ab..4dee382 100644 --- a/topics/11-execution-models/README.md +++ b/topics/11-execution-models/README.md @@ -16,6 +16,30 @@ flowchart LR H --> C ``` +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin exec_bench` — 50 M rows, +`SELECT k, SUM(v) WHERE f < t GROUP BY k`, best of 3: + +``` +selectivity volcano + 5% 0.386 s 129.4 M rows/s + 50% 0.484 s 103.3 M rows/s + 95% 0.669 s 74.7 M rows/s +``` + +**103 M rows/s is the tuple-at-a-time ceiling on this machine — and notice which +way it moves.** Cost rises monotonically with selectivity, because in a Volcano +pipeline every row that *survives* the filter pays the full per-tuple +interpretation bill on its way to the aggregate: a virtual call per operator per +row. The filter is not the expensive part. Passing it is. + +Put that next to topic 12's measurement of the same machine — ~50 GB/s of +sequential u64 fold, about 6.2 G values/s — and the gap is 60×. That gap is what +vectorized execution and compiled kernels are competing to close, and the +interesting question is not whether they close it but *what the remainder +consists of* once they do. Predict the split before you implement either lane. + ## 1. The Volcano (iterator) model Every operator implements `open() / next() / close()`; `next()` returns diff --git a/topics/11-execution-models/experiments/Cargo.lock b/topics/11-execution-models/experiments/Cargo.lock new file mode 100644 index 0000000..b22a9a4 --- /dev/null +++ b/topics/11-execution-models/experiments/Cargo.lock @@ -0,0 +1,133 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "exec-experiments" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/11-execution-models/experiments/src/bin/exec_bench.rs b/topics/11-execution-models/experiments/src/bin/exec_bench.rs index 4e747ae..ff8997e 100644 --- a/topics/11-execution-models/experiments/src/bin/exec_bench.rs +++ b/topics/11-execution-models/experiments/src/bin/exec_bench.rs @@ -34,15 +34,40 @@ fn bench(name: &str, table: &Table, threshold: u32, f: impl Fn(&Table, u32) -> V ); } +/// Run an exercise lane, reporting unimplemented `todo!()`s as a note +/// instead of a crash, so the volcano baseline always prints. +fn stub_lane(name: &str, f: impl FnOnce()) -> bool { + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { + println!(" [stub — implement the todo!()s to unlock {name}]"); + } + r.is_ok() +} + fn main() { println!("generating {} M rows...", ROWS / 1_000_000); let table = Table::generate(ROWS, 42); + let mut vec_ok = true; + let mut kern_ok = true; for threshold in [50, 5, 95] { println!("\nSELECT k, SUM(v) WHERE f < {threshold} GROUP BY k (selectivity ~{threshold}%)"); + // lane 1 (PROVIDED): tuple-at-a-time volcano — the baseline to beat bench("volcano", &table, threshold, volcano::run); - bench("vectorized", &table, threshold, vectorized::run); - bench("kernel", &table, threshold, kernels::run); + // lanes 2-3 (EXERCISE): batch-at-a-time, then typed kernels + if vec_ok { + vec_ok = stub_lane("vectorized (src/vectorized.rs)", || { + bench("vectorized", &table, threshold, vectorized::run) + }); + } + if kern_ok { + kern_ok = stub_lane("kernels (src/kernels.rs)", || { + bench("kernel", &table, threshold, kernels::run) + }); + } } println!("\nnotes:"); diff --git a/topics/11-execution-models/notes.md b/topics/11-execution-models/notes.md index 27fa327..82a2a63 100644 --- a/topics/11-execution-models/notes.md +++ b/topics/11-execution-models/notes.md @@ -1,5 +1,27 @@ # Topic 11 notes — execution models +## Baseline (provided lane, Apple M3 Pro, measured 2026-07-28) + +`cargo run --release --bin exec_bench` — 50 M rows, +`SELECT k, SUM(v) WHERE f < t GROUP BY k`, best of 3. + +| selectivity | volcano | vectorized | kernels | +|---|---|---|---| +| 5% | 0.386 s / 129.4 M rows/s | stub | stub | +| 50% | 0.484 s / 103.3 M rows/s | stub | stub | +| 95% | 0.669 s / 74.7 M rows/s | stub | stub | + +**103 M rows/s at 50% selectivity is the tuple-at-a-time ceiling on this +machine, and note which direction it moves.** Cost rises monotonically with +selectivity (74.7 M rows/s at 95%) because in a Volcano pipeline every surviving +row pays the full per-tuple interpretation cost — a virtual call per operator +per row — on its way to the aggregate. The filter is not the expensive part; +*passing the filter* is. + +Predict before implementing: at 50% selectivity, how much of the gap between +103 M rows/s and memory bandwidth (topic 12 measures ~50 GB/s = 6.2 G u64/s on +this box) does batching close, and what does the remainder consist of? + ## Predictions (fill BEFORE implementing vectorized.rs / kernels.rs) Measured baseline (provided volcano, release, 50M rows, sel 50%): diff --git a/topics/12-columnar-analytics/README.md b/topics/12-columnar-analytics/README.md index 87e0d00..6080e2b 100644 --- a/topics/12-columnar-analytics/README.md +++ b/topics/12-columnar-analytics/README.md @@ -21,6 +21,38 @@ Columns compress because a column is SELF-SIMILAR: same type, similar values, sorted or clustered. Rows interleave types and kill every trick below. +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin scan_bench` — 100 M u64 (800 MB) per shape, +single-threaded fold, best of 3: + +``` +shape raw sum raw-equiv GB/s +sorted low-cardinality 0.033 s 24.4 +shuffled low-cardinality 0.016 s 50.0 +small-range random 0.014 s 57.0 +``` + +**That is the floor every encoding in this topic has to beat: 24–57 GB/s on a +machine whose peak memory bandwidth is 150 GB/s.** One core gets roughly a third +of the bus, with LLVM already vectorizing the fold across several accumulators +to manage it. Beating this number is not a matter of shaving instructions; it +requires moving *fewer bytes*, which is the entire thesis — compression IS +performance. + +Two honesty notes that matter more than the figures. First, the spread across +the three shapes is noise, not signal: all three do byte-identical work, and +repeat runs put this lane anywhere from 24 to 76 GB/s depending on machine +state. A bandwidth-bound single number wants an error bar; take the high end as +the target. + +Second, **this lane used to print 19 047 619 GB/s** — roughly 20 000× the +machine's bandwidth. The timing loop let LLVM hoist the pure fold out of its own +repetition loop, so two of three reps timed nothing and best-of-3 reported +0.000 s. A `black_box` on the input fixed it. Topic 0's first failure mode, +found in this repo's own code, which is the best argument going for why the +numbers here are printed rather than asserted. + ## 1. The lightweight encoding zoo Not gzip. These are encodings the SCAN can execute over directly: diff --git a/topics/12-columnar-analytics/experiments/Cargo.lock b/topics/12-columnar-analytics/experiments/Cargo.lock new file mode 100644 index 0000000..009d2d6 --- /dev/null +++ b/topics/12-columnar-analytics/experiments/Cargo.lock @@ -0,0 +1,133 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "columnar-experiments" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/12-columnar-analytics/experiments/src/bin/scan_bench.rs b/topics/12-columnar-analytics/experiments/src/bin/scan_bench.rs index c33f415..7867a8c 100644 --- a/topics/12-columnar-analytics/experiments/src/bin/scan_bench.rs +++ b/topics/12-columnar-analytics/experiments/src/bin/scan_bench.rs @@ -2,11 +2,23 @@ //! //! cargo run --release --bin scan_bench //! -//! Panics on the stubs until encodings.rs is implemented. The raw -//! baseline runs first regardless. Predict in notes.md before running: -//! for each (shape, encoding), is the encoded scan faster or slower -//! than raw, and why (bytes moved vs decode work vs shortcuts)? +//! Lane 1 (the raw baseline) runs today. The encoded lanes are the +//! exercise: they print a `[stub — ...]` note until `encodings.rs` is +//! implemented, so the baseline above them always survives. +//! +//! Predict in notes.md before running: for each (shape, encoding), is the +//! encoded scan faster or slower than raw, and why (bytes moved vs decode +//! work vs shortcuts)? +//! +//! Note on the timing loop (topic 0's lesson, learned the hard way here): +//! every timed closure runs its input through `black_box`. Without it LLVM +//! hoists these pure folds *out* of the repetition loop — it computes the +//! sum once, reuses it for the remaining reps, and the fastest rep clocks +//! in at 0.000 s. This benchmark used to print 19,047,619 GB/s for the raw +//! lane, which is roughly 20,000× the machine's memory bandwidth and was +//! entirely a measurement artifact. +use std::hint::black_box; use std::time::Instant; use columnar_experiments::data; @@ -15,12 +27,16 @@ use columnar_experiments::encodings::{BitPacked, Dict, Rle}; const N: usize = 100_000_000; const REPS: usize = 3; -fn time(f: impl Fn() -> T) -> (f64, T) { +/// Below this, the reported rate is not a measurement — it is timer noise +/// or an elided loop, and we say so rather than print a figure. +const MIN_CREDIBLE_SECS: f64 = 1e-4; + +fn time(mut f: impl FnMut() -> T) -> (f64, T) { let mut best = f64::MAX; let mut out = None; for _ in 0..REPS { let start = Instant::now(); - let r = f(); + let r = black_box(f()); best = best.min(start.elapsed().as_secs_f64()); out = Some(r); } @@ -28,29 +44,55 @@ fn time(f: impl Fn() -> T) -> (f64, T) { } fn report(name: &str, raw_bytes: usize, enc_bytes: usize, secs: f64, sum: u64) { + let mb = enc_bytes as f64 / 1e6; + if secs < MIN_CREDIBLE_SECS { + println!( + " {name:<22} {mb:>7.1} MB {secs:>7.3} s {:>6} GB/s(raw-equiv) sum={sum}", + "n/a" + ); + println!(" ^ below timer resolution — treat as unmeasured, not as fast"); + return; + } let gbps = raw_bytes as f64 / secs / 1e9; - println!( - " {name:<22} {:>7.1} MB {secs:>7.3} s {gbps:>6.1} GB/s(raw-equiv) sum={sum}", - enc_bytes as f64 / 1e6 - ); + println!(" {name:<22} {mb:>7.1} MB {secs:>7.3} s {gbps:>6.1} GB/s(raw-equiv) sum={sum}"); } -fn bench_shape(name: &str, values: &[u64]) { - println!("\n== {name} ({} M values, {} MB raw)", N / 1_000_000, N * 8 / 1_000_000); +/// Lane 1 (PROVIDED): the raw baseline — 800 MB of u64 through a fold. +/// This is the memory-bandwidth floor every encoded scan is measured against. +fn lane1_raw(values: &[u64]) { let raw_bytes = values.len() * 8; - - let (t, s) = time(|| values.iter().copied().fold(0u64, u64::wrapping_add)); + let (t, s) = time(|| { + black_box(values) + .iter() + .copied() + .fold(0u64, u64::wrapping_add) + }); report("raw sum", raw_bytes, raw_bytes, t, s); +} + +/// Lanes 2-3 (EXERCISE): scans over the encoded forms. Returns false the +/// first time it hits an unimplemented encoding, so the caller can stop +/// re-announcing the same stub for every shape. +fn encoded_lanes(values: &[u64]) { + let raw_bytes = values.len() * 8; + // lane 2a: RLE — sum on the encoding itself, one multiply-add per run let rle = Rle::encode(values); - let (t, s) = time(|| rle.sum()); + let (t, s) = time(|| black_box(&rle).sum()); report("rle sum (no decode)", raw_bytes, rle.size_bytes(), t, s); - let (t, s) = time(|| rle.decode().iter().copied().fold(0u64, u64::wrapping_add)); + let (t, s) = time(|| { + black_box(&rle) + .decode() + .iter() + .copied() + .fold(0u64, u64::wrapping_add) + }); report("rle decode+sum", raw_bytes, rle.size_bytes(), t, s); + // lane 2b: dictionary — process-compressed, sum via per-code counts let dict = Dict::encode(values); let (t, s) = time(|| { - // process-compressed: sum via per-code counts, decode never + let dict = black_box(&dict); let mut counts = vec![0u64; dict.dict.len()]; for &c in &dict.codes { counts[c as usize] += 1; @@ -62,8 +104,15 @@ fn bench_shape(name: &str, values: &[u64]) { }); report("dict sum (codes only)", raw_bytes, dict.size_bytes(), t, s); + // lane 3: frame-of-reference bit-packing — decode then sum let bp = BitPacked::encode(values); - let (t, s) = time(|| bp.decode().iter().copied().fold(0u64, u64::wrapping_add)); + let (t, s) = time(|| { + black_box(&bp) + .decode() + .iter() + .copied() + .fold(0u64, u64::wrapping_add) + }); report("bitpack decode+sum", raw_bytes, bp.size_bytes(), t, s); println!( @@ -75,13 +124,51 @@ fn bench_shape(name: &str, values: &[u64]) { ); } +/// Run an exercise lane, reporting unimplemented `todo!()`s as a note +/// instead of a crash, so the provided lanes always print. Returns false +/// if the lane is still stubbed. +fn try_lane(f: impl FnOnce()) -> bool { + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + std::panic::set_hook(prev); + r.is_ok() +} + fn main() { - bench_shape("sorted low-cardinality", &data::sorted_low_cardinality(N, 42)); - bench_shape("shuffled low-cardinality", &data::shuffled_low_cardinality(N, 42)); - bench_shape("small-range random", &data::small_range_random(N, 42)); + let shapes: [(&str, fn(usize, u64) -> Vec); 3] = [ + ("sorted low-cardinality", data::sorted_low_cardinality), + ("shuffled low-cardinality", data::shuffled_low_cardinality), + ("small-range random", data::small_range_random), + ]; + + let mut encoded_ok = true; + for (name, gen) in shapes { + // one shape live at a time: 100 M u64 is 800 MB + let values = gen(N, 42); + println!( + "\n== {name} ({} M values, {} MB raw)", + N / 1_000_000, + N * 8 / 1_000_000 + ); + lane1_raw(&values); + if encoded_ok { + encoded_ok = try_lane(|| encoded_lanes(&values)); + } + } + + if !encoded_ok { + println!( + "\n[stub — implement src/encodings.rs to unlock the encoded scan lanes]\n\ + \x20 `cargo test` shows the contract: round-trips, exact sizes, width-0\n\ + \x20 bit-packing, and Rle::sum operating on runs without decoding." + ); + } println!("\nnotes:"); println!("- 'raw-equiv GB/s' = raw bytes / time: >memory-bandwidth means the"); println!(" encoding beat the memory bus — compression IS performance"); - println!("- record the full table + your Mac's ~bandwidth in notes.md"); + println!("- the raw lane IS your machine's scan bandwidth; compare it to the"); + println!(" DRAM figure from topic 0's cache_ladder before trusting either"); + println!("- record the full table in notes.md"); } diff --git a/topics/12-columnar-analytics/notes.md b/topics/12-columnar-analytics/notes.md index 42d1553..bf52f11 100644 --- a/topics/12-columnar-analytics/notes.md +++ b/topics/12-columnar-analytics/notes.md @@ -1,5 +1,35 @@ # Topic 12 notes — columnar storage & analytics +## Baseline (provided lane, Apple M3 Pro, measured 2026-07-28) + +`cargo run --release --bin scan_bench` — 100 M u64 (800 MB raw) per shape, +best of 3, single-threaded fold. + +| shape | raw sum | raw-equiv GB/s | +|---|---|---| +| sorted low-cardinality | 0.033 s | 24.4 | +| shuffled low-cardinality | 0.016 s | 50.0 | +| small-range random | 0.014 s | 57.0 | + +**This is the floor every encoded scan has to beat: 24–57 GB/s of sequential +u64 fold, on a machine whose peak memory bandwidth is 150 GB/s.** One core gets +roughly a third of the bus, and LLVM is already vectorizing the +`wrapping_add` fold across several accumulators to do it. + +Two honesty notes, both of which matter more than the numbers: + +- **The spread across the three shapes is measurement noise, not a property of + the data.** All three do byte-identical work — the same 800 MB fold — and the + shapes differ only in what the *encoders* will later be able to exploit. + Repeat runs put this lane anywhere from 24 to 76 GB/s depending on machine + state, which is a useful reminder that a bandwidth-bound single number wants + an error bar. Take the high end as the target to beat. +- **This lane used to print 19 047 619 GB/s.** The timing loop hoisted the pure + fold out of its own repetition loop, so reps 2 and 3 timed nothing and + best-of-3 reported ~0.000 s. `black_box` on the input fixed it. If you write + your own lane here, that is the failure mode to expect (topic 0's lesson, + learned the hard way in this file). + ## Predictions (fill BEFORE running scan_bench) Raw baseline context: 100M u64 = 800 MB; this Mac's bandwidth ≈ ? GB/s diff --git a/topics/13-graph-engines/README.md b/topics/13-graph-engines/README.md index 91e8cb9..bcbbc2c 100644 --- a/topics/13-graph-engines/README.md +++ b/topics/13-graph-engines/README.md @@ -5,6 +5,33 @@ competes with — with line numbers and benchmarks, not marketing. Four architectures, one question: what does an Expand (get neighbors) cost, and what does pattern matching (multi-way Expand) cost? +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin hop_bench` — preferential-attachment graph, 1 M nodes +/ 16.0 M directed edges, two-hop `COUNT(DISTINCT)` from 1000 sources. Max degree +6565, p50 degree 11: + +``` +impl source set ns/query distinct reached +adj_list (oracle) random 4914 10220457 +adj_list (oracle) supernodes 495378 7890665 +``` + +**The same query, on the same graph, through the same code: 101× slower +depending on which node you start from.** Nothing before this topic behaves like +that. A B-tree point lookup costs what it costs; a two-hop traversal costs +whatever the degree distribution under your start node says, and on a scale-free +graph that distribution is a power law with no useful mean. The median node has +11 neighbours. The top one has 6565. + +Now look at the last column: the slow case reaches *fewer* distinct nodes — 7.9 M +against 10.2 M — while taking 101× longer. High-degree neighbourhoods overlap +heavily, so the extra work is redundant rather than productive. That redundancy +is the opening the CSR and masked-SpMV lanes attack, and it is why graph engines +are built around set operations on sorted adjacency rather than around pointer +chasing. It is also why "supernode" is a word in this field and not in the +others. + ## 1. The adjacency representation menu ``` diff --git a/topics/13-graph-engines/experiments/Cargo.lock b/topics/13-graph-engines/experiments/Cargo.lock new file mode 100644 index 0000000..f1a1744 --- /dev/null +++ b/topics/13-graph-engines/experiments/Cargo.lock @@ -0,0 +1,133 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "graph-experiments" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/13-graph-engines/experiments/src/bin/hop_bench.rs b/topics/13-graph-engines/experiments/src/bin/hop_bench.rs index 1fd51ed..62ed7c7 100644 --- a/topics/13-graph-engines/experiments/src/bin/hop_bench.rs +++ b/topics/13-graph-engines/experiments/src/bin/hop_bench.rs @@ -26,6 +26,19 @@ fn report(name: &str, label: &str, secs: f64, n: usize, checksum: u64) { ); } +/// Run an exercise lane, reporting unimplemented `todo!()`s as a note +/// instead of a crash, so the adj_list oracle above always prints. +fn stub_lane(name: &str, f: impl FnOnce() -> T) -> Option { + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { + println!(" [stub — implement the todo!()s to unlock {name}]"); + } + r.ok() +} + fn main() { let t = Instant::now(); let g = data::preferential_attachment(N, M, 42); @@ -67,29 +80,38 @@ fn main() { } println!("\n== csr"); - let t = Instant::now(); - let csr = Csr::build(g.num_nodes, &g.edges); - println!(" build: {:.2} s", t.elapsed().as_secs_f64()); - for (label, srcs) in [("random", &random), ("supernodes", &supernodes)] { - let mut checksum = 0u64; + let csr = stub_lane("the CSR build + two_hop (src/csr.rs)", || { let t = Instant::now(); - for &s in srcs { - stamp += 1; - checksum += csr.two_hop(s, &mut seen, stamp); + let csr = Csr::build(g.num_nodes, &g.edges); + println!(" build: {:.2} s", t.elapsed().as_secs_f64()); + for (label, srcs) in [("random", &random), ("supernodes", &supernodes)] { + let mut checksum = 0u64; + let t = Instant::now(); + for &s in srcs { + stamp += 1; + checksum += csr.two_hop(s, &mut seen, stamp); + } + report("csr", label, t.elapsed().as_secs_f64(), srcs.len(), checksum); } - report("csr", label, t.elapsed().as_secs_f64(), srcs.len(), checksum); - } + csr + }); - println!("\n== matrix (masked SpMV over the same CSR)"); - let (mut f1, mut f2) = (Vec::new(), Vec::new()); - for (label, srcs) in [("random", &random), ("supernodes", &supernodes)] { - let mut checksum = 0u64; - let t = Instant::now(); - for &s in srcs { - stamp += 1; - checksum += matrix::two_hop(&csr, s, &mut seen, stamp, &mut f1, &mut f2); - } - report("matrix", label, t.elapsed().as_secs_f64(), srcs.len(), checksum); + // the matrix lane runs over the CSR the previous lane built, so it can + // only run once that one does + if let Some(csr) = csr.as_ref() { + println!("\n== matrix (masked SpMV over the same CSR)"); + stub_lane("masked SpMV (src/matrix.rs)", || { + let (mut f1, mut f2) = (Vec::new(), Vec::new()); + for (label, srcs) in [("random", &random), ("supernodes", &supernodes)] { + let mut checksum = 0u64; + let t = Instant::now(); + for &s in srcs { + stamp += 1; + checksum += matrix::two_hop(csr, s, &mut seen, stamp, &mut f1, &mut f2); + } + report("matrix", label, t.elapsed().as_secs_f64(), srcs.len(), checksum); + } + }); } println!("\nnotes:"); diff --git a/topics/13-graph-engines/notes.md b/topics/13-graph-engines/notes.md index 90d4484..7807e82 100644 --- a/topics/13-graph-engines/notes.md +++ b/topics/13-graph-engines/notes.md @@ -1,5 +1,34 @@ # Topic 13 notes — graph engines +## Baseline (provided lane, Apple M3 Pro, measured 2026-07-28) + +`cargo run --release --bin hop_bench` — preferential-attachment graph, 1 M +nodes / 16.0 M directed edges, two-hop `COUNT(DISTINCT)` from 1000 sources. +Max degree 6565, p50 degree 11. + +| impl | source set | ns/query | checksum | +|---|---|---|---| +| adj_list (oracle) | random | 4 914 | 10 220 457 | +| adj_list (oracle) | supernodes (top-100 degree) | **495 378** | 7 890 665 | +| CSR (yours) | | | stub | +| masked SpMV (yours) | | | stub | + +**Same query, same graph, same code path: 101× slower depending on where you +start.** That ratio is the single most important fact about graph workloads and +it has no analogue in the topics before this one. A B-tree point lookup costs +what it costs; a two-hop traversal costs whatever the *degree distribution* +under your start node says it costs, and on a scale-free graph that is a +power law with no useful average. The p50 node has 11 neighbours; the top node +has 6565. + +Note the supernode checksum is *smaller* (7.9 M vs 10.2 M distinct nodes +reached) while taking 101× longer — high-degree neighbourhoods overlap heavily, +so the work is redundant, not productive. That redundancy is what the CSR and +SpMV lanes are able to attack; the adjacency-list oracle cannot. + +Checksums must match across all three implementations per source set — that is +the correctness gate before any timing comparison means anything. + ## Predictions (fill BEFORE implementing csr.rs / matrix.rs) Baseline (provided, measured): adj_list 3484 ns/query random, diff --git a/topics/13-graph-engines/reading-graphblas-internals.md b/topics/13-graph-engines/reading-graphblas-internals.md index 092675b..1c9c57c 100644 --- a/topics/13-graph-engines/reading-graphblas-internals.md +++ b/topics/13-graph-engines/reading-graphblas-internals.md @@ -241,6 +241,14 @@ GraphBLAS's format/algorithm anchors as the layer below. 5. Map Delta_Matrix states to LSM vocabulary: what's the memtable, the SST, the tombstone, the compaction? +## Done when + +- [ ] You can write the `read = (M ∪ DP) ∖ DM` identity and explain what each of the three matrices holds. +- [ ] You can explain why CSR is hostile to single-edge inserts, and why that fact alone forces something like Delta_Matrix. +- [ ] You can say when dot beats saxpy for a BFS step, in terms of frontier size against matrix dimension. +- [ ] You can explain what a mask pushes into the kernel and what it saves — connect it to the masked-SpMV lane in this topic's bench. +- [ ] You wrote answers to all five questions in notes.md, including the Delta_Matrix-to-LSM vocabulary mapping. + ## References **Papers** diff --git a/topics/13-graph-engines/reading-kuzu.md b/topics/13-graph-engines/reading-kuzu.md index a07aad4..5ba701b 100644 --- a/topics/13-graph-engines/reading-kuzu.md +++ b/topics/13-graph-engines/reading-kuzu.md @@ -174,6 +174,14 @@ carry the chapter: encoding wins for CSR targets sorted by src, and why? (Think about what's monotonic within a run and what isn't.) +## Done when + +- [ ] You can explain how a CSR header turns sorted rows into an O(1) expand. +- [ ] You can describe the persistent-CSR-plus-transient-overlay scheme per node group, and state its worst-case update cost. +- [ ] You can say why Intersect requires sorted adjacency lists, and what breaks without that. +- [ ] You can estimate intermediate sizes for a triangle count under a binary plan against a WCOJ plan on this topic's 16 M edge graph. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/13-graph-engines/reading-ldbc-snb.md b/topics/13-graph-engines/reading-ldbc-snb.md index 5b833f2..a57598e 100644 --- a/topics/13-graph-engines/reading-ldbc-snb.md +++ b/topics/13-graph-engines/reading-ldbc-snb.md @@ -146,6 +146,14 @@ reporting), not its full query set. 5. Which SNB scale factor fits in this Mac's RAM as (a) memgraph objects, (b) CSR, (c) Delta_Matrix? Rough per-edge byte estimates. +## Done when + +- [ ] You can name the three workloads and the different question each one asks. +- [ ] You can explain why correlated power-law data is the point rather than a realism garnish — and connect it to the 101x supernode gap this topic measures. +- [ ] You can say what running updates during reads prevents a vendor from doing. +- [ ] You can state what a pinned scale factor and an audit rule are for. +- [ ] You wrote answers to all questions in notes.md, including what you intend to steal for M22. + ## References **Papers** diff --git a/topics/13-graph-engines/reading-memgraph-storage.md b/topics/13-graph-engines/reading-memgraph-storage.md index 2f765fb..5249523 100644 --- a/topics/13-graph-engines/reading-memgraph-storage.md +++ b/topics/13-graph-engines/reading-memgraph-storage.md @@ -178,6 +178,14 @@ row against a field. 5. Sketch what an analytics query (PageRank) costs on this layout vs a matrix. Where does the memory bus time go? +## Done when + +- [ ] You can draw the Vertex struct and say what per-node state lives in one place. +- [ ] You can explain why every edge is stored twice and which query shape breaks if it is not. +- [ ] You can state the difference between per-object delta chains here and per-version rows in postgres, and what each makes cheap. +- [ ] You can compare the cost of expanding one vertex here against kuzu's CSR slice, and say which workload each layout is built for. +- [ ] You wrote answers to all five questions in notes.md, including the PageRank cost sketch. + ## References **Papers** diff --git a/topics/13-graph-engines/reading-neo4j-record-store.md b/topics/13-graph-engines/reading-neo4j-record-store.md index 0fdc894..1c81817 100644 --- a/topics/13-graph-engines/reading-neo4j-record-store.md +++ b/topics/13-graph-engines/reading-neo4j-record-store.md @@ -189,6 +189,14 @@ fields, then trace Step 4's walk mentally against them. version of the argument that still holds, and the part that died with DRAM. +## Done when + +- [ ] You can explain the index-free adjacency bet and name the hardware assumption that aged out from under it. +- [ ] You can compute expand cost for a 1000-edge node as dependent loads, and compare it to a CSR slice. +- [ ] You can say what the doubly-linked relationship chain buys and what it costs on insert. +- [ ] You can state the modern version of the index-free adjacency argument — the one that survives the disk era ending. +- [ ] You wrote answers to all five questions in notes.md, including the 15 B versus 34 B field accounting. + ## References **Code** diff --git a/topics/13-graph-engines/reading-query-languages.md b/topics/13-graph-engines/reading-query-languages.md index cb9b0e9..0c4ab8f 100644 --- a/topics/13-graph-engines/reading-query-languages.md +++ b/topics/13-graph-engines/reading-query-languages.md @@ -182,6 +182,14 @@ the query says what, not how. path pattern that can represent Cypher's `[*1..5]` AND GQL's `ALL ACYCLIC (a)(-[:R]->){1,5}(b)` without a parser rewrite. +## Done when + +- [ ] You can state the three matching semantics and count the 2-paths in a triangle under each. +- [ ] You can explain what GQL's restrictors and selectors make explicit that Cypher left implicit. +- [ ] You can say what property graphs and RDF actually disagree about, beyond syntax. +- [ ] You can name, for each language, one thing its semantics lets the planner do that another's forbids. +- [ ] You wrote answers to all questions in notes.md, including this topic's 2-hop query written in more than one language. + ## References **Papers** diff --git a/topics/13-graph-engines/reading-wcoj.md b/topics/13-graph-engines/reading-wcoj.md index e3599d8..235f8ce 100644 --- a/topics/13-graph-engines/reading-wcoj.md +++ b/topics/13-graph-engines/reading-wcoj.md @@ -189,6 +189,15 @@ optimality without ever naming it. vs intersect for a pattern — what's the detectable trigger? (Cyclicity of the pattern graph.) +## Done when + +- [ ] You can explain why every pairwise plan loses on the triangle query, using intermediate sizes rather than intuition. +- [ ] You can state the AGM bound and compute the fractional edge cover for the triangle. +- [ ] You can narrate Generic Join as one variable at a time, and say where the intersections happen. +- [ ] You can say when galloping beats a merge intersection, in terms of the two list lengths. +- [ ] You can explain why `C = A²` is the same algorithm in matrix spelling — and connect it to the masked-SpMV lane here. +- [ ] You wrote answers to all questions in notes.md. + ## References **Papers** diff --git a/topics/14-vector-search/README.md b/topics/14-vector-search/README.md index 5a1be12..44aa945 100644 --- a/topics/14-vector-search/README.md +++ b/topics/14-vector-search/README.md @@ -5,6 +5,29 @@ the k nearest vectors WITHOUT scanning everything, trading exactness for speed. The whole field is one curve — **recall@k vs QPS** — and every algorithm is a point-generator on it. +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin ann_bench` — 100 000 × 128-dim f32 (51 MB) in 200 +clusters, 500 queries, k=10: + +``` +brute force: 4.28 s total, 117 QPS — recall 1.000 by definition +``` + +**117 queries per second, with perfect recall for free.** That single point is +what the entire approximate-nearest-neighbour field exists to beat, and stating +it first changes how you read every ANN result afterwards: a QPS figure quoted +without its recall is not a measurement, because the trivial way to get more QPS +is to return worse answers. + +It is worth being precise about *why* it is slow, because it decides which lane +should win. 51 MB of vectors is small — this is a compute problem, not a memory +one: 500 queries × 100 000 candidates × 128 dimensions is 6.4 G multiply-adds. +So brute force here is really a SIMD exercise (topic 17), which means the two +exercise lanes attack different walls. HNSW cuts the *number* of candidates; +scalar quantization shrinks each candidate 4× and rescopes the same scan. Predict +where the two land relative to each other, and to this row, before writing either. + ## 1. The problem shape Exact k-NN over n vectors of dimension d = n·d multiply-adds per diff --git a/topics/14-vector-search/experiments/Cargo.lock b/topics/14-vector-search/experiments/Cargo.lock new file mode 100644 index 0000000..c683ce8 --- /dev/null +++ b/topics/14-vector-search/experiments/Cargo.lock @@ -0,0 +1,133 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "vector-experiments" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/14-vector-search/experiments/src/bin/ann_bench.rs b/topics/14-vector-search/experiments/src/bin/ann_bench.rs index 951b174..f24baa2 100644 --- a/topics/14-vector-search/experiments/src/bin/ann_bench.rs +++ b/topics/14-vector-search/experiments/src/bin/ann_bench.rs @@ -2,10 +2,10 @@ //! //! cargo run --release --bin ann_bench //! -//! Brute-force baseline runs first (the QPS floor + ground truth); -//! panics on the stubs after that. Predict in notes.md: recall and -//! QPS at each ef, and where quantized+rescore lands relative to the -//! HNSW curve. +//! Brute-force baseline runs first (the QPS floor + ground truth); the +//! index lanes report as stubs until you implement them. Predict in +//! notes.md: recall and QPS at each ef, and where quantized+rescore lands +//! relative to the HNSW curve. use std::time::Instant; @@ -19,6 +19,18 @@ const CLUSTERS: usize = 200; const NUM_QUERIES: u32 = 500; const K: usize = 10; +/// Run an exercise lane, reporting unimplemented `todo!()`s as a note +/// instead of a crash, so the brute-force baseline always prints. +fn stub_lane(name: &str, f: impl FnOnce()) { + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { + println!("[stub — implement the todo!()s to unlock {name}]"); + } +} + fn main() { let t = Instant::now(); let d = data::clustered(N, DIM, CLUSTERS, 42); @@ -41,43 +53,49 @@ fn main() { q.len() as f64 / brute_secs ); - let t = Instant::now(); - let h = Hnsw::build(&d, HnswConfig::default()); - println!("hnsw build: {:.1} s (m=16, ef_c=128), max_level={}", t.elapsed().as_secs_f64(), h.max_level); - - println!("\n {:<10} {:>10} {:>12}", "ef", "recall@10", "QPS"); - for ef in [16, 32, 64, 128, 256] { - let mut total_recall = 0.0; + // lane 2 (EXERCISE): HNSW — the recall/QPS curve against the oracle above + stub_lane("HNSW (src/hnsw.rs)", || { let t = Instant::now(); - for qi in 0..q.len() { - let found = h.search(&d, q.get(qi), K, ef); - total_recall += recall(&found, &truth[qi as usize]); + let h = Hnsw::build(&d, HnswConfig::default()); + println!("hnsw build: {:.1} s (m=16, ef_c=128), max_level={}", t.elapsed().as_secs_f64(), h.max_level); + + println!("\n {:<10} {:>10} {:>12}", "ef", "recall@10", "QPS"); + for ef in [16, 32, 64, 128, 256] { + let mut total_recall = 0.0; + let t = Instant::now(); + for qi in 0..q.len() { + let found = h.search(&d, q.get(qi), K, ef); + total_recall += recall(&found, &truth[qi as usize]); + } + let secs = t.elapsed().as_secs_f64(); + println!( + " {ef:<10} {:>10.3} {:>12.0}", + total_recall / q.len() as f64, + q.len() as f64 / secs + ); } - let secs = t.elapsed().as_secs_f64(); - println!( - " {ef:<10} {:>10.3} {:>12.0}", - total_recall / q.len() as f64, - q.len() as f64 / secs - ); - } + }); - let t = Instant::now(); - let sq = ScalarQuant::encode(&d); - println!("\nscalar u8 encode: {:.1} s ({} MB codes)", t.elapsed().as_secs_f64(), sq.codes.len() / 1_000_000); - for oversample in [1, 2, 4] { - let mut total_recall = 0.0; + // lane 3 (EXERCISE): scalar quantization — 4x smaller codes, then rescore + stub_lane("scalar quantization (src/quant.rs)", || { let t = Instant::now(); - for qi in 0..q.len() { - let found = quant::search_rescore(&d, &sq, q.get(qi), K, oversample); - total_recall += recall(&found, &truth[qi as usize]); + let sq = ScalarQuant::encode(&d); + println!("\nscalar u8 encode: {:.1} s ({} MB codes)", t.elapsed().as_secs_f64(), sq.codes.len() / 1_000_000); + for oversample in [1, 2, 4] { + let mut total_recall = 0.0; + let t = Instant::now(); + for qi in 0..q.len() { + let found = quant::search_rescore(&d, &sq, q.get(qi), K, oversample); + total_recall += recall(&found, &truth[qi as usize]); + } + let secs = t.elapsed().as_secs_f64(); + println!( + " u8 scan+rescore x{oversample}: recall {:.3}, {:.0} QPS", + total_recall / q.len() as f64, + q.len() as f64 / secs + ); } - let secs = t.elapsed().as_secs_f64(); - println!( - " u8 scan+rescore x{oversample}: recall {:.3}, {:.0} QPS", - total_recall / q.len() as f64, - q.len() as f64 / secs - ); - } + }); println!("\nnotes:"); println!("- record the full curve in notes.md; optional: same data via"); diff --git a/topics/14-vector-search/notes.md b/topics/14-vector-search/notes.md index becb7c4..dc46ce7 100644 --- a/topics/14-vector-search/notes.md +++ b/topics/14-vector-search/notes.md @@ -1,5 +1,30 @@ # Topic 14 notes — vector search +## Baseline (provided lane, Apple M3 Pro, measured 2026-07-28) + +`cargo run --release --bin ann_bench` — 100 000 × 128-dim f32 (51 MB) in 200 +clusters, 500 queries, k=10. + +| method | recall@10 | QPS | total | +|---|---|---|---| +| brute force | 1.000 (by definition) | **117** | 4.28 s | +| HNSW (yours) | | | stub | +| u8 quantized + rescore (yours) | | | stub | + +**117 QPS is the number the whole field exists to beat, and it comes with +perfect recall for free.** Every approximate index in this topic is a bet that +you can trade a measurable amount of that 1.000 for orders of magnitude of +throughput — so the honest way to read any ANN benchmark is as a *curve* +against this point, never as a single QPS figure. + +Worth being precise about what makes it slow: 51 MB of vectors is small enough +to be a pure compute problem, not a memory-bound one — 500 queries × 100 000 +candidates × 128 dimensions is 6.4 G multiply-adds. So brute force here is a +SIMD/bandwidth exercise (topic 17), and a scalar-quantized scan that shrinks +each vector 4× is attacking the same wall from the data side rather than the +algorithm side. Predict where those two lanes land relative to each other +before you write either. + ## Predictions (fill BEFORE implementing hnsw.rs / quant.rs) Baseline (provided, measured): brute force 185 QPS at recall 1.0 diff --git a/topics/14-vector-search/reading-diskann.md b/topics/14-vector-search/reading-diskann.md index 5a39e1c..85f277f 100644 --- a/topics/14-vector-search/reading-diskann.md +++ b/topics/14-vector-search/reading-diskann.md @@ -166,6 +166,14 @@ survive. when a "read" is 50 ms S3 GET instead of 100 µs NVMe? Which knob moves? +## Done when + +- [ ] You can say why HNSW does not survive being put on disk, in reads per hop rather than in generalities. +- [ ] You can explain what `α > 1` does to greedy walk length, and why that is the property Vamana is buying. +- [ ] You can describe the block layout and count the SSD reads per hop it achieves. +- [ ] You can explain the division of labour in the search loop: PQ steers, f32 ranks, W reads in flight — and what recall failure each part is responsible for. +- [ ] You wrote answers to all five questions in notes.md, including the M28 object-storage preview. + ## References **Papers** diff --git a/topics/14-vector-search/reading-hnsw-paper.md b/topics/14-vector-search/reading-hnsw-paper.md index a01a9ff..fd181bb 100644 --- a/topics/14-vector-search/reading-hnsw-paper.md +++ b/topics/14-vector-search/reading-hnsw-paper.md @@ -228,6 +228,15 @@ lens: 5. The paper claims robustness to dimensionality vs NSW. What's the skip-list analogue of "the entry point is always the same node"? +## Done when + +- [ ] You can explain what makes "approximate" the product rather than a compromise, using this topic's measured 117 QPS brute-force floor. +- [ ] You can derive why `mL = 1/ln(M)` gives an expected max level of `ln(n)/ln(M)`. +- [ ] You can state what Algorithm 4's neighbour selection does differently from taking the M nearest, and what breaks if you take the nearest. +- [ ] You can say why `ef >= k` is required and what happens at exactly `ef = k`. +- [ ] You can account for HNSW's memory at n=1M, d=128, M=16, splitting vectors from links. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/14-vector-search/reading-pq.md b/topics/14-vector-search/reading-pq.md index ec67384..354c652 100644 --- a/topics/14-vector-search/reading-pq.md +++ b/topics/14-vector-search/reading-pq.md @@ -172,6 +172,15 @@ Four pieces of this 2011 paper are load-bearing in 2026 systems: 5. SDC would let you precompute ALL tables once (no per-query work). Why does nobody care? +## Done when + +- [ ] You can explain the product move: why quantizing subspaces independently gives 2^128 effective centroids from 16 bytes. +- [ ] You can state the difference between SDC and ADC and say where each eats its approximation. +- [ ] You can explain why chunks must be roughly statistically independent, and what correlated dimensions do to the code. +- [ ] You can compute the per-query ADC table build cost and say at what candidate count it stops mattering. +- [ ] You can say why IVFADC encodes residuals rather than raw vectors, in terms of the quantizer's dynamic range. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/14-vector-search/reading-qdrant-hnsw.md b/topics/14-vector-search/reading-qdrant-hnsw.md index be11a24..5a4940f 100644 --- a/topics/14-vector-search/reading-qdrant-hnsw.md +++ b/topics/14-vector-search/reading-qdrant-hnsw.md @@ -186,6 +186,14 @@ thesis), then chase each branch to its implementation. map it onto topic 13's transient/persistent kuzu split and Delta_Matrix. What's the graph-index "flush"? +## Done when + +- [ ] You can explain why the build structure and the serve structure differ, and what freezing buys. +- [ ] You can explain percolation: why a selective filter shatters a proximity graph rather than just shrinking it. +- [ ] You can describe the per-query decision qdrant makes and name the inputs it uses (cardinality estimate, thresholds). +- [ ] You can say what ACORN's 2-hop expansion costs in scoring work and what it buys in connectivity. +- [ ] You wrote answers to all five questions in notes.md, including why `full_scan_threshold` is expressed in bytes. + ## References **Papers** diff --git a/topics/14-vector-search/reading-qdrant-quantization.md b/topics/14-vector-search/reading-qdrant-quantization.md index 7487564..63e299d 100644 --- a/topics/14-vector-search/reading-qdrant-quantization.md +++ b/topics/14-vector-search/reading-qdrant-quantization.md @@ -172,6 +172,14 @@ smallest and carries the score-without-decode idea), then 5. M14 decision: which rung of the ladder for graph node embeddings, given M17 SIMD comes later — commit + reason. +## Done when + +- [ ] You can name the three rungs of the ladder and the compression ratio each achieves. +- [ ] You can derive the u8 affine dot-product expansion and say what must be stored per vector for it to work. +- [ ] You can explain why PQ hurts HNSW traversal more than it hurts a flat IVF scan. +- [ ] You can say what oversample-and-rescore claws back, and predict where it lands against this topic's brute-force point before implementing `quant.rs`. +- [ ] You wrote answers to all five questions in notes.md, including the M14 rung decision. + ## References **Papers** diff --git a/topics/14-vector-search/reading-usearch.md b/topics/14-vector-search/reading-usearch.md index 36883ed..6f2a729 100644 --- a/topics/14-vector-search/reading-usearch.md +++ b/topics/14-vector-search/reading-usearch.md @@ -161,6 +161,14 @@ walked revision; navigate by symbol name when they drift): Decide, justify with expected access pattern, and note what M17's SIMD needs. +## Done when + +- [ ] You can list what an HNSW node must store and compute bytes per node for M=16, M0=32. +- [ ] You can explain what the node tape buys over `Vec>` per level, in allocations and in locality. +- [ ] You can say why link slots are preallocated to the maximum rather than grown. +- [ ] You can describe the concurrency scheme — striped writer locks, lock-free readers — and what it assumes about readers. +- [ ] You wrote answers to all five questions in notes.md, including your own tape-or-vec decision for `hnsw.rs`. + ## References **Papers** diff --git a/topics/15-replication-consensus/README.md b/topics/15-replication-consensus/README.md index 8b1d7a8..1754b30 100644 --- a/topics/15-replication-consensus/README.md +++ b/topics/15-replication-consensus/README.md @@ -6,6 +6,33 @@ asynchronously and calls it a day, qdrant wraps tikv's raft-rs around cluster METADATA only, and everyone chooses a different point on the consistency/latency line. +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin repl_lag` — 2000 entries × 128 B, leader +group-commits every 64, `WAIT 1` ack semantics. The only variable is the +follower's fsync policy: + +``` +follower fsync entries/s ack p50 us ack p99 us +every entry 341 2967.0 3889.5 +every 8 2730 22.2 2979.8 +every 64 12187 14.0 2133.0 +never 20174 13.8 64.5 +``` + +**59× throughput between the safest row and the loosest one — and the whole +durability argument hides in the p99 column.** The `every 8` row has already +recovered most of the *median* (22.2 µs, within 60% of `never`) while its p99 is +still 2980 µs. Batching removes the fsync from the common case and leaves it in +the tail, so a replication setting judged by its median is a setting whose worst +case you have not looked at. + +Then compare the top row to topic 5: 341 entries/s here, 337 commits/s for +`F_FULLFSYNC` there. The follower is not slow because replication is expensive. +It is slow because it is paying the same physical media flush, once per entry, +that topic 5 already priced. Same wall, one topic later — which is the argument +for reading these two together. + ## 1. The topology menu ``` diff --git a/topics/15-replication-consensus/experiments/Cargo.lock b/topics/15-replication-consensus/experiments/Cargo.lock new file mode 100644 index 0000000..df3565f --- /dev/null +++ b/topics/15-replication-consensus/experiments/Cargo.lock @@ -0,0 +1,133 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "replication-experiments" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/15-replication-consensus/experiments/src/bin/partition_test.rs b/topics/15-replication-consensus/experiments/src/bin/partition_test.rs index ccea1f0..e47872e 100644 --- a/topics/15-replication-consensus/experiments/src/bin/partition_test.rs +++ b/topics/15-replication-consensus/experiments/src/bin/partition_test.rs @@ -18,6 +18,29 @@ fn snapshot(sim: &Sim, label: &str) { } fn main() { + // The whole scenario drives YOUR src/raft.rs, so there is no provided + // lane here — probe once and explain the state instead of dumping a + // panic trace. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let implemented = std::panic::catch_unwind(|| { + let mut s = Sim::new(3, 1); + s.run(1); + }) + .is_ok(); + std::panic::set_hook(prev); + if !implemented { + println!( + "[stub — implement src/raft.rs to unlock the partition scenario]\n\n\ + This binary elects a leader, commits three entries, partitions the\n\ + leader into a minority with one buddy, has it propose entry 99, then\n\ + heals the partition. The two invariants it exists to show: 99 must\n\ + never commit, and after the heal it must be truncated everywhere.\n\ + `cargo test` is the smaller specification." + ); + return; + } + let mut sim = Sim::new(5, 2026); let leader = sim.run_until_leader(500); diff --git a/topics/15-replication-consensus/notes.md b/topics/15-replication-consensus/notes.md index ea81b5a..71274cf 100644 --- a/topics/15-replication-consensus/notes.md +++ b/topics/15-replication-consensus/notes.md @@ -1,5 +1,30 @@ # Topic 15 notes — replication, consensus & distribution +## Baseline (provided lane, Apple M3 Pro / APFS, measured 2026-07-28) + +`cargo run --release --bin repl_lag` — 2000 entries × 128 B, leader +group-commits every 64, ack semantics `WAIT 1` (one follower must acknowledge). +The follower's fsync policy is the only variable. + +| follower fsync | entries/s | ack p50 | ack p99 | +|---|---|---|---| +| every entry | 341 | 2967.0 µs | 3889.5 µs | +| every 8 | 2 730 | 22.2 µs | 2979.8 µs | +| every 64 | 12 187 | 14.0 µs | 2133.0 µs | +| never | 20 174 | 13.8 µs | 64.5 µs | + +**59× throughput between the safest and the loosest row, and the durability +argument is entirely in the p99 column.** Two things to sit with: + +- The `every 8` row already recovers most of the *median* (22.2 µs, within 60% + of `never`) while its p99 stays at 2980 µs — batching hides the fsync from the + common case and leaves it in the tail. If you judge a replication setting by + its median you will pick a configuration whose worst case you have not seen. +- `every entry` at 341 entries/s lines up with topic 5's `F_FULLFSYNC` rung + (337 commits/s) almost exactly. The follower is not slow because replication + is expensive; it is slow because it is paying the same physical media flush, + once per entry. Same wall, different topic. + ## Predictions (fill BEFORE implementing raft.rs) repl_lag baseline (provided, measured 2026-07-10, macOS diff --git a/topics/15-replication-consensus/reading-ddia-repl.md b/topics/15-replication-consensus/reading-ddia-repl.md index 99dfb57..03e323d 100644 --- a/topics/15-replication-consensus/reading-ddia-repl.md +++ b/topics/15-replication-consensus/reading-ddia-repl.md @@ -167,6 +167,15 @@ The closing vocabulary, three items: read — cost per read of each, and which M22 (the capstone's read-path milestone) should pick. +## Done when + +- [ ] You can name the three read anomalies lag produces and give a user-visible symptom for each. +- [ ] You can state what statement, WAL-byte and row shipping each make hard, and which one valkey uses. +- [ ] You can explain what a fencing token prevents that a timeout cannot. +- [ ] You can define linearizability precisely enough to say why it is a recency guarantee and not an isolation level. +- [ ] You can fill in the 2x3 matrix of {async, semi-sync, raft} x {read-your-writes, monotonic reads, consistent prefix}. +- [ ] You wrote answers to all five questions in notes.md, including why FLP does not doom Raft in practice. + ## References **Papers / Books** diff --git a/topics/15-replication-consensus/reading-qdrant-consensus.md b/topics/15-replication-consensus/reading-qdrant-consensus.md index 68525af..904de91 100644 --- a/topics/15-replication-consensus/reading-qdrant-consensus.md +++ b/topics/15-replication-consensus/reading-qdrant-consensus.md @@ -166,6 +166,14 @@ persisted (question 5). 5. Where does qdrant persist the raft log and HardState? Find the Storage impl behind ConsensusStateRef. +## Done when + +- [ ] You can state the arithmetic that forces the metadata/data plane split — why metadata volume fits raft and point writes do not. +- [ ] You can map the replica states Active/Dead/Partial onto Raft's Progress states. +- [ ] You can say what consistency a qdrant vector read actually gets, and whether it is configurable. +- [ ] You can describe the `on_ready` ordering rules and name which of them is a safety requirement rather than an optimization. +- [ ] You wrote answers to all five questions in notes.md, including where the raft log and HardState are persisted. + ## References **Code** diff --git a/topics/15-replication-consensus/reading-raft-paper.md b/topics/15-replication-consensus/reading-raft-paper.md index 7f160f7..bc1aadb 100644 --- a/topics/15-replication-consensus/reading-raft-paper.md +++ b/topics/15-replication-consensus/reading-raft-paper.md @@ -205,6 +205,15 @@ Step 6, and worth an hour. 5. Map to valkey: which Raft properties does async replication give up, and what do you get back for each? +## Done when + +- [ ] You can explain why agreeing on log order is sufficient for state-machine convergence. +- [ ] You can say what a term is and what it fences. +- [ ] You can state both safety rules — the election restriction and the current-term commit rule — and explain the quorum-intersection argument behind Figure 8. +- [ ] You can say exactly which state must be persisted before responding, and why the rest need not be. +- [ ] You can explain why a leader never overwrites its own entries, and what that means for the follower repair loop. +- [ ] You wrote answers to all five questions in notes.md, and can predict what `partition_test` must show: 99 never commits, and is truncated everywhere after the heal. + ## References **Papers** diff --git a/topics/15-replication-consensus/reading-raft-rs.md b/topics/15-replication-consensus/reading-raft-rs.md index d89ea3c..a71c942 100644 --- a/topics/15-replication-consensus/reading-raft-rs.md +++ b/topics/15-replication-consensus/reading-raft-rs.md @@ -182,6 +182,15 @@ driving loop for this exact API is the next chapter 5. Map Ready → M15 stage 2: which parts of your WAL commit path play the roles of persist/send/apply/advance? +## Done when + +- [ ] You can explain what sans-io buys and why raft-rs contains no fsync, no sockets and no threads. +- [ ] You can write out the `maybe_commit` sorted-matched-index computation from memory. +- [ ] You can state the Ready contract's ordering rules and say which reorderings are safety violations rather than performance bugs. +- [ ] You can explain how splitting the persistence acknowledgement (`advance_append`) enables pipelining without breaking the contract. +- [ ] You can say what the `next_idx` decrement-and-retry loop costs in round trips, and what optimization fixes it. +- [ ] You wrote answers to all five questions in notes.md, including the Ready-to-M15 mapping. + ## References **Papers** diff --git a/topics/15-replication-consensus/reading-valkey-replication.md b/topics/15-replication-consensus/reading-valkey-replication.md index 2cb8afd..ce47514 100644 --- a/topics/15-replication-consensus/reading-valkey-replication.md +++ b/topics/15-replication-consensus/reading-valkey-replication.md @@ -198,6 +198,15 @@ back what async gave up). 5. For M15 stage 1: which parts of PSYNC do you keep (replid+offset, backlog ring, +CONTINUE/+FULLRESYNC) and which do you simplify? +## Done when + +- [ ] You can explain what "ack first, replicate later" means for a client that received a success reply. +- [ ] You can describe PSYNC's `(replid, offset)` scheme and say what makes a partial resync possible or impossible. +- [ ] You can size the replication backlog from a write rate and a tolerable disconnect window. +- [ ] You can explain why full sync forks, and connect it to copy-on-write. +- [ ] You can say precisely what WAIT does and does not guarantee — then check it against this topic's measured table, where WAIT 1 with per-entry follower fsync costs 341 entries/s. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Code** diff --git a/topics/15-replication-consensus/reading-vsr.md b/topics/15-replication-consensus/reading-vsr.md index bb05802..45ed0ea 100644 --- a/topics/15-replication-consensus/reading-vsr.md +++ b/topics/15-replication-consensus/reading-vsr.md @@ -177,6 +177,15 @@ skill — it's how you'll evaluate M15 stage 2's design decisions. fails, torn write) survivable, where Raft's model assumes storage is faithful? Connect to topic 5's torn-page discussion. +## Done when + +- [ ] You can state which VSR concepts are Raft's under other names, and which are genuinely different choices. +- [ ] You can explain what round-robin primary selection removes from the protocol, and what it costs. +- [ ] You can compare DOVIEWCHANGE's whole-log shipping against Raft's incremental repair and say when each is cheaper. +- [ ] You can write the failure sequence that the no-disk recovery argument depends on, and say what makes it safe. +- [ ] You can say why the recovery protocol needs a nonce. +- [ ] You wrote answers to all five questions in notes.md, including the TigerBeetle checksum point. + ## References **Papers** diff --git a/topics/16-testing-correctness/README.md b/topics/16-testing-correctness/README.md index b24671c..078c638 100644 --- a/topics/16-testing-correctness/README.md +++ b/topics/16-testing-correctness/README.md @@ -23,6 +23,36 @@ Every technique in this topic is one choice of generator + oracle: | Jepsen/elle | concurrent client histories | linearizability checker | | Z3 / Cosette | symbolic (ALL inputs at once) | UNSAT = proven equal | +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin crash_matrix` — 5000 seeded workloads × 40 ops +(50% put / 20% delete / 20% commit / 10% crash) against a KV store with one bug +planted at a time: + +``` +bug caught rate first seed +None 0 0.0% - +LostDelete 3738 74.8% 0 +NoSyncOnCommit 4980 99.6% 0 +TornWriteAccepted 2442 48.8% 3 +StaleRead 4706 94.1% 0 +``` + +**Every planted bug is caught, three of them by the very first seed — and the +spread from 48.8% to 99.6% is the useful part.** A bug caught by 99.6% of seeds +is one you cannot ship; a bug caught by 48.8% is one that survives a test suite +run twice and fails in production on the third. Same harness, same effort, four +different probabilities of ever finding out. + +The `None` row is the load-bearing one. 0.0% is not a formality: any nonzero +number there means the harness reports divergence on a correct implementation, +and a false-positive oracle is worse than no oracle because it trains you to +ignore it. Check that row first, every time you change the generator. + +Note also what is *not* here: no timing claim. The value of deterministic +simulation is that a failure comes with a seed you can replay, and the exercise +lanes are about shrinking that seed's 40 ops to the 3 that matter. + ## 1. Deterministic simulation testing (DST) FoundationDB's gift to the industry (turso, TigerBeetle, Antithesis diff --git a/topics/16-testing-correctness/experiments/Cargo.lock b/topics/16-testing-correctness/experiments/Cargo.lock new file mode 100644 index 0000000..d855021 --- /dev/null +++ b/topics/16-testing-correctness/experiments/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "testing-experiments" +version = "0.1.0" +dependencies = [ + "rand", + "rand_chacha", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/16-testing-correctness/experiments/src/bin/dst_run.rs b/topics/16-testing-correctness/experiments/src/bin/dst_run.rs index 414b4b0..f55f892 100644 --- a/topics/16-testing-correctness/experiments/src/bin/dst_run.rs +++ b/topics/16-testing-correctness/experiments/src/bin/dst_run.rs @@ -7,6 +7,22 @@ use testing_experiments::kv::Bug; use testing_experiments::shrink::shrink; fn main() { + // find_bug + shrink are both yours (src/dst.rs, src/shrink.rs), so this + // binary has nothing of its own to print — probe once and say so. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let implemented = std::panic::catch_unwind(|| find_bug(Bug::LostDelete, 1, 4)).is_ok(); + std::panic::set_hook(prev); + if !implemented { + println!( + "[stub — implement src/dst.rs and src/shrink.rs to unlock the DST run]\n\n\ + This binary searches seeds for each planted bug, then shrinks the failing\n\ + op sequence to a minimal repro. crash_matrix is the provided answer key:\n\ + run it first for the detection rates your dst.rs should roughly match." + ); + return; + } + for bug in [Bug::LostDelete, Bug::NoSyncOnCommit, Bug::TornWriteAccepted, Bug::StaleRead] { match find_bug(bug, 500, 40) { None => println!("{bug:?}: NOT FOUND in 500 seeds (harness too weak?)"), diff --git a/topics/16-testing-correctness/reading-fdb-simulation.md b/topics/16-testing-correctness/reading-fdb-simulation.md index fe99c14..c17803e 100644 --- a/topics/16-testing-correctness/reading-fdb-simulation.md +++ b/topics/16-testing-correctness/reading-fdb-simulation.md @@ -191,6 +191,15 @@ but more nondeterminism left uncorralled. M6 buffer pool). List the remaining nondeterminism sources to corral (threadpool from M9! HashMap iteration! rand in plans!). +## Done when + +- [ ] You can explain why example-based tests lose to distributed systems, in terms of interleaving count. +- [ ] You can state the bet: the database and its test harness are one artifact, and say what that forbids in the production code. +- [ ] You can describe the seeded event loop over a time-ordered heap, and say why simulation runs *faster* than real time. +- [ ] You can explain what BUGGIFY is and give the argument for why compiling it out of production is not cheating. +- [ ] You can name three bug classes simulation provably cannot catch. +- [ ] You wrote answers to all five questions in notes.md, including which of our IO traits already sit in the right place for M16. + ## References **Papers & docs** diff --git a/topics/16-testing-correctness/reading-jepsen.md b/topics/16-testing-correctness/reading-jepsen.md index 2d28352..4026447 100644 --- a/topics/16-testing-correctness/reading-jepsen.md +++ b/topics/16-testing-correctness/reading-jepsen.md @@ -167,6 +167,15 @@ overlap. the history. What does the deterministic sim make TRIVIAL that real Jepsen fights (total real-time order is known!)? +## Done when + +- [ ] You can explain why checking a recorded history is the hard half of the method, not collecting one. +- [ ] You can describe elle's workload trick and say why append-and-read-full-list makes dependencies visible. +- [ ] You can explain why a cycle in the serialization graph *is* an anomaly, and identify which isolation level a pure-rw cycle violates. +- [ ] You can say why Jepsen uses SIGSTOP/SIGCONT rather than kill -9 for certain faults. +- [ ] You can state what elle cannot check, and where DST complements rather than competes. +- [ ] You wrote answers to all five questions in notes.md, including the redis-raft stale-read history. + ## References **Papers** diff --git a/topics/16-testing-correctness/reading-pqs-tlp-papers.md b/topics/16-testing-correctness/reading-pqs-tlp-papers.md index 7e44915..7ac108e 100644 --- a/topics/16-testing-correctness/reading-pqs-tlp-papers.md +++ b/topics/16-testing-correctness/reading-pqs-tlp-papers.md @@ -164,6 +164,15 @@ Read PQS first; TLP is partly a response to PQS's costs. for Cypher (WHERE / count(*) / collect?) and write the ⊎ for each. +## Done when + +- [ ] You can state the test-oracle problem and explain why differential testing against another DBMS is not a solution. +- [ ] You can explain rectification: how any random predicate is made TRUE on the pivot row, and why three-valued logic makes that delicate. +- [ ] You can construct a bug PQS provably misses, using containment rather than equality. +- [ ] You can write the TLP identity and say why `col = col` is a useless partitioning predicate. +- [ ] You can state the trade the two papers make — completeness for portability — and which one you would reach for first. +- [ ] You wrote answers to all five questions in notes.md, including your first three TLP recombinations for M16. + ## References **Papers** diff --git a/topics/16-testing-correctness/reading-sqlancer.md b/topics/16-testing-correctness/reading-sqlancer.md index 6d453b2..41a68ef 100644 --- a/topics/16-testing-correctness/reading-sqlancer.md +++ b/topics/16-testing-correctness/reading-sqlancer.md @@ -173,6 +173,15 @@ skip the adapter plumbing. plays the role of NULL in a graph pattern (missing property!), and what's the union assertion? +## Done when + +- [ ] You can describe all three oracles — PQS, TLP, NoREC — in one sentence each. +- [ ] You can explain why checking one row per query is enough in expectation. +- [ ] You can write the TLP identity for `COUNT(*)` and for `MAX(c)`, and say which one is harder and why. +- [ ] You can explain what NoREC compares and why turning the optimizer off is a valid oracle. +- [ ] You can sketch a Cypher TLP partition for `MATCH (a)-[e]->(b) WHERE p` and name what makes graph patterns harder than SQL rows here. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/16-testing-correctness/reading-turso-simulator.md b/topics/16-testing-correctness/reading-turso-simulator.md index 1caff70..a3666e8 100644 --- a/topics/16-testing-correctness/reading-turso-simulator.md +++ b/topics/16-testing-correctness/reading-turso-simulator.md @@ -203,6 +203,15 @@ doublecheck and shrink. 5. For M16: which three properties from generation/property.rs port directly to Cypher? Sketch the graph equivalents. +## Done when + +- [ ] You can explain what it takes to make a program a pure function of a seed, and name the three sources of nondeterminism that must be captured. +- [ ] You can say why ChaCha8 rather than the default RNG, and what property DST needs from its generator. +- [ ] You can describe fault injection at the file layer and why targeting the WAL separately from the db file matters. +- [ ] You can explain the doublecheck oracle — determinism itself — and why it is the cheapest one available. +- [ ] You can say why shrinking stateful op sequences is harder than shrinking values, and connect it to this topic's `shrink.rs` exercise. +- [ ] You wrote answers to all five questions in notes.md, including the three properties you will port. + ## References **Code** diff --git a/topics/16-testing-correctness/reading-z3.md b/topics/16-testing-correctness/reading-z3.md index e42a235..b7395b6 100644 --- a/topics/16-testing-correctness/reading-z3.md +++ b/topics/16-testing-correctness/reading-z3.md @@ -179,6 +179,15 @@ and 19. encoding for each; which needs the (value, is_null) pair and which doesn't? +## Done when + +- [ ] You can explain CDCL as a search loop that learns, and say what a learned clause is. +- [ ] You can state the SAT/theory division of labour: SAT proposes, theories veto. +- [ ] You can encode a small query plan as symbolic rows and say what the formula asserts. +- [ ] You can encode `WHERE NOT (a = b)` against `WHERE a <> b` over nullable columns and show where they differ. +- [ ] You can say why Cosette needs bags (K-relations) rather than sets, and which SQL feature forces it. +- [ ] You wrote answers to all five questions in notes.md, including the two topic-10 rewrite rules you would verify. + ## References **Papers** diff --git a/topics/17-simd/experiments/src/filter.rs b/topics/17-simd/experiments/src/filter.rs index 18ea508..2cc3aa3 100644 --- a/topics/17-simd/experiments/src/filter.rs +++ b/topics/17-simd/experiments/src/filter.rs @@ -41,6 +41,7 @@ pub fn count_branchy(vals: &[f32], t: f32) -> usize { /// alternative when you need bit POSITIONS, not just a count — you /// will need it in compact_neon below.) #[cfg(target_arch = "aarch64")] +#[allow(unused_variables, reason = "used once the todo!() is implemented")] pub fn count_neon(vals: &[f32], t: f32) -> usize { todo!("vcltq_f32 + vsubq_u32 mask accumulation + vaddvq_u32") } @@ -62,6 +63,7 @@ pub fn count_neon(vals: &[f32], t: f32) -> usize { /// flatten_bits shape). /// Output must EXACTLY match compact_branchy. #[cfg(target_arch = "aarch64")] +#[allow(unused_variables, reason = "used once the todo!() is implemented")] pub fn compact_neon(vals: &[f32], t: f32, out: &mut Vec) { todo!("4-bit mask → shuffle LUT → vqtbl1q_u8 → advance by popcount") } diff --git a/topics/17-simd/experiments/src/unpack.rs b/topics/17-simd/experiments/src/unpack.rs index 18d4ff4..e4f9138 100644 --- a/topics/17-simd/experiments/src/unpack.rs +++ b/topics/17-simd/experiments/src/unpack.rs @@ -34,6 +34,7 @@ pub fn pack4(vals: &[u32]) -> Vec { /// u8 → u16 → u32 (`vmovl_u8`, `vmovl_u16`) before `vst1q_u32`. /// Scalar remainder for len % 16. Output must equal unpack4_scalar. #[cfg(target_arch = "aarch64")] +#[allow(unused_variables, reason = "used once the todo!() is implemented")] pub fn unpack4_neon(packed: &[u8], out: &mut Vec) { todo!("vandq/vshrq nibble split + vzipq interleave + vmovl widen") } diff --git a/topics/17-simd/reading-fastlanes.md b/topics/17-simd/reading-fastlanes.md index 0ad769f..1f1fd2e 100644 --- a/topics/17-simd/reading-fastlanes.md +++ b/topics/17-simd/reading-fastlanes.md @@ -172,6 +172,14 @@ exactly there? simd_bench — then reconcile with FastLanes' claim that layout, not intrinsics, is the win. +## Done when + +- [ ] You can explain why the sequential bit-packed layout cannot vectorize, at the level of which value crosses which lane boundary. +- [ ] You can describe the transpose into 1024 virtual bit-serial lanes and say what makes one permutation work for every lane width. +- [ ] You can name the two constraints that fix the block at 1024 values regardless of bit width. +- [ ] You can say what random access to value i now costs, and what was traded to get the scan speed. +- [ ] You wrote answers to all five questions in notes.md, and can state how your `unpack.rs` differs from the paper's layout — this topic measures the scalar version at 7.99 GB/s of output. + ## References **Papers** diff --git a/topics/17-simd/reading-hashbrown-simd.md b/topics/17-simd/reading-hashbrown-simd.md index 75243f5..5f079a5 100644 --- a/topics/17-simd/reading-hashbrown-simd.md +++ b/topics/17-simd/reading-hashbrown-simd.md @@ -199,6 +199,15 @@ then sse2.rs → neon.rs → generic.rs in that order (native → shrunk fn pointers (SimSIMD): which fits a Cypher engine that ships one binary to unknown ARM servers? +## Done when + +- [ ] You can explain movemask and give the three different answers to "one bit per lane" across ISAs. +- [ ] You can explain SWAR: how a u64 acts as an 8-lane vector, and which operation is the dangerous one. +- [ ] You can narrate the SwissTable probe loop and count the instructions per 8-16 slots. +- [ ] You can explain why `match_tag` may tolerate false positives while `match_empty` may not. +- [ ] You can state the portability pattern — and say why hashbrown found a 16-byte NEON group *lost* to the generic path. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Code** diff --git a/topics/17-simd/reading-mojo-simd.md b/topics/17-simd/reading-mojo-simd.md index dbc00e1..5b7f0fa 100644 --- a/topics/17-simd/reading-mojo-simd.md +++ b/topics/17-simd/reading-mojo-simd.md @@ -156,6 +156,14 @@ this table is the map from the ideal to what M17 actually ships: sentence justifying that (deployment target) and the one-line escape hatch if SVE servers arrive. +## Done when + +- [ ] You can explain what making scalars width-1 vectors buys in the type system. +- [ ] You can say what `simdwidthof` + `vectorize` generate that you currently write by hand, including the remainder loop. +- [ ] You can separate the four layers of the matmul arc and say which one contributes most (it is not SIMD). +- [ ] You can name what parametric width does *not* solve. +- [ ] You wrote answers to all five questions in notes.md, including the one-line justification for hardcoding NEON width 4 in M17. + ## References **Papers & docs** diff --git a/topics/17-simd/reading-polars-compute.md b/topics/17-simd/reading-polars-compute.md index bc2c2ce..068cb7a 100644 --- a/topics/17-simd/reading-polars-compute.md +++ b/topics/17-simd/reading-polars-compute.md @@ -162,6 +162,15 @@ expanded in your head, then `mod.rs` for the dispatch. intrinsics + scalar. Why does compress specifically defeat portable SIMD abstractions? +## Done when + +- [ ] You can explain the reduction problem and why STRIPE=16 fixes it — then connect it to this topic's measured 8.88 -> 26.32 GB/s from eight accumulators alone. +- [ ] You can explain how pairwise recursion fixes accuracy and derive why the error bound differs from a single chain. +- [ ] You can describe the `simd_filter!` skeleton and how it advances the output pointer by popcount. +- [ ] You can say what NEON gets instead of a compress instruction, and what that costs. +- [ ] You can explain the dispatch strategy — one feature test per block — and why polars chose not to use `std::simd` for filter. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Code** diff --git a/topics/17-simd/reading-sigmod15-vectorization.md b/topics/17-simd/reading-sigmod15-vectorization.md index a949069..733a091 100644 --- a/topics/17-simd/reading-sigmod15-vectorization.md +++ b/topics/17-simd/reading-sigmod15-vectorization.md @@ -186,6 +186,15 @@ topic 13's software write-combining buffers make the scatter moot. our Cypher pipeline (filter, probe, partition, bloom) given M11's profile — where does Amdahl bite first? +## Done when + +- [ ] You can name the two primitives — selective store, selective load/gather — and say which operators need which. +- [ ] You can explain why branchless loses to branchy at extreme selectivities in the paper, and check that against this topic's measured sweep, where branchless never actually loses (0.95 vs 10.04 GB/s at 50%). +- [ ] You can describe vertical hash probing with W probes in flight and say what it requires of the hash table. +- [ ] You can state the gather cost model: parallel instructions, serial memory. +- [ ] You can say why scatter needs conflict detection during partitioning. +- [ ] You wrote answers to all five questions in notes.md, including your ranking of the four operators by expected engine-level win. + ## References **Papers** diff --git a/topics/17-simd/reading-simdjson.md b/topics/17-simd/reading-simdjson.md index 5e0ed6e..f53f731 100644 --- a/topics/17-simd/reading-simdjson.md +++ b/topics/17-simd/reading-simdjson.md @@ -181,6 +181,15 @@ stage 1 — the rest you can skim once the steps above are solid. 5. For M7: sketch stage-1 masks for RESP (`*3\r\n$3\r\nSET...`) — which characters are "structural"? +## Done when + +- [ ] You can explain the stage-1 idea: classify 64 bytes into bitmasks, branch once per block instead of once per byte. +- [ ] You can explain quote parity by carry-less multiply and why prefix-xor is the right primitive. +- [ ] You can describe the escaped-backslash problem and why it cannot be solved with a single mask. +- [ ] You can explain `flatten_bits`'s over-write-and-under-advance trick and why it is safe. +- [ ] You can say why stage 2 stays branchy and give the Amdahl argument for why that is acceptable. +- [ ] You wrote answers to all five questions in notes.md, including stage-1 masks sketched for RESP. + ## References **Papers** diff --git a/topics/17-simd/reading-simsimd.md b/topics/17-simd/reading-simsimd.md index f04d9c7..bbabca9 100644 --- a/topics/17-simd/reading-simsimd.md +++ b/topics/17-simd/reading-simsimd.md @@ -167,6 +167,15 @@ different table. {dot, l2sq, filter} × {neon, scalar} and where `is_aarch64_feature_detected!` runs exactly once. +## Done when + +- [ ] You can read the port/latency table and compute peak f32 FMA throughput for M-series from it. +- [ ] You can explain why precision is bought with wider accumulators rather than with reordering. +- [ ] You can say why batching candidates beats unrolling pairs. +- [ ] You can state the FCMLA lesson: a specialized instruction must beat the table, not merely exist. +- [ ] You can sketch the function-pointer dispatch table and say when the ISA choice is made. +- [ ] You wrote answers to all five questions in notes.md, including the Newton-Raphson round count for f64. + ## References **Code** diff --git a/topics/18-gpu/experiments/src/bin/gpu_bench.rs b/topics/18-gpu/experiments/src/bin/gpu_bench.rs index f62b5d7..634f905 100644 --- a/topics/18-gpu/experiments/src/bin/gpu_bench.rs +++ b/topics/18-gpu/experiments/src/bin/gpu_bench.rs @@ -16,7 +16,27 @@ fn cpu_us(mut f: impl FnMut()) -> f64 { } fn main() { - let ctx = GpuCtx::new(); + // This is the one lane in the repo that needs hardware the machine may not + // have: a headless Linux CI runner has no Metal and often no Vulkan driver. + // A benchmark with no device to measure has not failed, it has nothing to + // report — so say that and exit cleanly rather than panicking. verify.sh + // reads the marker below and records SKIP rather than PASS, because a green + // tick for a lane that measured nothing would be worse than a red one. + let Some(ctx) = GpuCtx::try_new() else { + println!( + "[skipped — no GPU adapter on this machine]\n\n\ + This topic is the only one that needs a device: it measures where the\n\ + CPU/GPU crossover sits once transfer costs are included, and with no\n\ + adapter there is nothing to transfer to. On macOS you get Metal; on\n\ + Linux you need a Vulkan or GL driver (mesa's lavapipe works, and is\n\ + itself instructive — a software adapter makes the transfer tax look\n\ + free and the kernel look terrible).\n\n\ + The recorded reference numbers for an Apple M3 Pro are in notes.md:\n\ + no crossover up to 2^24 elements, with upload alone costing 7197 µs\n\ + at 16 M against a 2723 µs CPU total." + ); + return; + }; println!("adapter: {}\n", ctx.adapter_name); println!("sum: CPU (8-acc autovec) vs GPU (workgroup reduce), end-to-end"); diff --git a/topics/18-gpu/experiments/src/gpu.rs b/topics/18-gpu/experiments/src/gpu.rs index 976d179..cb3041b 100644 --- a/topics/18-gpu/experiments/src/gpu.rs +++ b/topics/18-gpu/experiments/src/gpu.rs @@ -33,20 +33,37 @@ pub struct GpuCtx { } impl GpuCtx { + /// Panics if there is no usable GPU. Use this in tests and anywhere a + /// missing adapter should be treated as a broken environment. pub fn new() -> Self { + Self::try_new().expect( + "no GPU adapter — Metal is expected on macOS, and a Vulkan or GL \ + driver on Linux. A headless CI runner usually has neither; see \ + try_new() and the skip path in bin/gpu_bench.rs.", + ) + } + + /// Returns `None` when no GPU adapter is available, so a caller can report + /// "nothing to measure here" instead of crashing. This exists because the + /// whole topic is about a device that may simply not be present: a headless + /// Linux CI runner has no Metal and often no Vulkan driver either, and a + /// benchmark that cannot find hardware has not failed — it has nothing to + /// say. + pub fn try_new() -> Option { let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::default()); let adapter = pollster::block_on( instance.request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::HighPerformance, ..Default::default() }), - ) - .expect("no GPU adapter (Metal expected on macOS)"); + )?; let adapter_name = adapter.get_info().name; + // An adapter that reports itself but then refuses a device is the same + // situation as no adapter at all, as far as a caller is concerned. let (device, queue) = pollster::block_on( adapter.request_device(&wgpu::DeviceDescriptor::default(), None), ) - .expect("device"); + .ok()?; let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("sum"), @@ -62,7 +79,7 @@ impl GpuCtx { cache: None, }); - Self { device, queue, sum_pipeline, adapter_name } + Some(Self { device, queue, sum_pipeline, adapter_name }) } /// PROVIDED: sum via one workgroup-reduction pass (1024 elems → diff --git a/topics/18-gpu/reading-cagra.md b/topics/18-gpu/reading-cagra.md index b31e1f7..ca73773 100644 --- a/topics/18-gpu/reading-cagra.md +++ b/topics/18-gpu/reading-cagra.md @@ -179,6 +179,14 @@ single-CTA search (Steps 4–5). candidates. Which half goes to GPU first, and what's the batch size per the crossover table you'll measure with l2_batch? +## Done when + +- [ ] You can say what SIMT hates about a greedy proximity-graph walk, and why fixed degree 32 is the answer rather than levels. +- [ ] You can explain why the build is NN-descent rather than insert-one-at-a-time. +- [ ] You can describe the one-CTA-per-query shape and what is parallel inside a single step. +- [ ] You can explain why the visited set becomes a shared-memory hashmap, and what that costs. +- [ ] You wrote answers to all five questions in notes.md, including where `search_multi_cta` becomes the right choice. + ## References **Papers** diff --git a/topics/18-gpu/reading-crystal-sigmod20.md b/topics/18-gpu/reading-crystal-sigmod20.md index 9cf2237..6006458 100644 --- a/topics/18-gpu/reading-crystal-sigmod20.md +++ b/topics/18-gpu/reading-crystal-sigmod20.md @@ -224,6 +224,15 @@ on a GPU at all? filter (streaming), distance scoring (dense). Apply Step 7's roofline to each and write the one-line go/no-go. +## Done when + +- [ ] You can state the two regimes and say which one the bus puts you in — then connect it to this topic's measured result: no crossover up to 2^24, with upload alone costing 7197 µs at 16 M elements. +- [ ] You can explain the two memory rules (coalescing, shared memory) and what violating each costs. +- [ ] You can explain why a filter needs a prefix scan rather than an output cursor. +- [ ] You can state why kernel fusion is mandatory rather than an optimization, and what it costs in modularity. +- [ ] You can use the roofline formula to predict a winner before measuring. +- [ ] You wrote answers to all five questions in notes.md, including the unified-memory rewrite of their regime analysis. + ## References **Papers** diff --git a/topics/18-gpu/reading-faiss-gpu.md b/topics/18-gpu/reading-faiss-gpu.md index 232081d..5cd3ae0 100644 --- a/topics/18-gpu/reading-faiss-gpu.md +++ b/topics/18-gpu/reading-faiss-gpu.md @@ -179,6 +179,14 @@ needs no index redesign. brute-force case at batch 1. Predict from the roofline whether Metal wins BEFORE running your implementation — then check. +## Done when + +- [ ] You can state the residency rule and what it implies about index size versus device memory. +- [ ] You can explain why heaps fail on warps and what WarpSelect does instead. +- [ ] You can say what breaks when k exceeds roughly 1024 per warp. +- [ ] You can explain why fusing distance computation into k-select matters more than either part alone. +- [ ] You wrote answers to all five questions in notes.md, including the `l2_batch` comparison against their reported figures. + ## References **Papers** diff --git a/topics/18-gpu/reading-gunrock.md b/topics/18-gpu/reading-gunrock.md index 07d598a..e0ea038 100644 --- a/topics/18-gpu/reading-gunrock.md +++ b/topics/18-gpu/reading-gunrock.md @@ -181,6 +181,15 @@ representations. In the paper: §3 is the operator model (Step 2), per LDBC scale factor, and does the answer change with the frontier's hub fraction per BFS level? +## Done when + +- [ ] You can express a graph algorithm as rounds of advance and filter. +- [ ] You can explain why frontier raggedness breaks warps, using this repo's own measured degree skew (max degree 6565 against a median of 11 in topic 13). +- [ ] You can name the load-balancing strategies and say which one a hub vertex of degree 10^6 demands. +- [ ] You can explain why sparse versus dense frontier representation is the same choice as push versus pull. +- [ ] You can say why a lost CAS race on `parent[]` is benign in BFS. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/18-gpu/reading-libcudf.md b/topics/18-gpu/reading-libcudf.md index e51cd67..923ea0a 100644 --- a/topics/18-gpu/reading-libcudf.md +++ b/topics/18-gpu/reading-libcudf.md @@ -187,6 +187,15 @@ questions demand. values, not count) using a workgroup prefix scan — Crystal's BlockScan. +## Done when + +- [ ] You can explain the no-push rule and why it forces two-phase (size, then retrieve) kernels. +- [ ] You can count the kernel launches in one `inner_join` and say which of them recomputes work. +- [ ] You can explain cooperative-groups probing: the warp as the vector register. +- [ ] You can say why group-by aggregates in shared memory until it spills, and what the spill costs. +- [ ] You can explain why conditional joins fall back to nested loops with a device AST. +- [ ] You wrote answers to all five questions in notes.md, including what is wrong with one atomic per workgroup in the `filter_count` stub. + ## References **Code** diff --git a/topics/18-gpu/reading-wgpu-compute.md b/topics/18-gpu/reading-wgpu-compute.md index 890bef1..f2c2e15 100644 --- a/topics/18-gpu/reading-wgpu-compute.md +++ b/topics/18-gpu/reading-wgpu-compute.md @@ -194,6 +194,15 @@ next to sum.wgsl, and big_compute_buffers only when Step 7 bites. regime A) or `upload(&[f32]) -> GpuVec` + `sum(&GpuVec)` (regime B)? Justify from this guide's measurements. +## Done when + +- [ ] You can list the object ladder a dispatch requires and say which parts can be hoisted out of a loop. +- [ ] You can state the fixed per-dispatch tax and check it against this topic's measurement: ~1.4 ms of kernel time at every size from 2^14 to 2^20. +- [ ] You can explain why the absence of float atomics forces the tree-reduction shape. +- [ ] You can say why there is no device-wide barrier and what multi-pass therefore means. +- [ ] You can explain why upload dominates readback in the measured table (7197 µs against 25.6 µs at 16 M elements). +- [ ] You wrote answers to all five questions in notes.md, including the regime-B rerun with upload hoisted out. + ## References **Code** diff --git a/topics/19-jit/README.md b/topics/19-jit/README.md index 7ca1211..804acc4 100644 --- a/topics/19-jit/README.md +++ b/topics/19-jit/README.md @@ -9,6 +9,37 @@ VM since 2000, and SuiteSparse:GraphBLAS JIT-compiles its semiring kernels — which makes this FalkorDB home turf twice over (M19 JITs Cypher expressions with cranelift). +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin jit_bench` — random arithmetic expression trees +evaluated over columns, interpreter (per row) vs vectorized (per column): + +``` +depth (nodes) rows interp M/s vector M/s ratio + 2 (7) 1024 89.37 558.34 6.2x + 2 (7) 2097152 86.83 452.10 5.2x + 4 (31) 1024 18.80 182.04 9.7x + 4 (31) 2097152 17.75 129.32 7.3x + 6 (127) 1024 4.05 47.72 11.8x + 6 (127) 2097152 3.71 32.40 8.7x + 8 (511) 1024 0.95 11.84 12.5x +``` + +**Interpretation cost scales with expression size, and the penalty compounds: +7 nodes to 511 nodes costs the interpreter 94× (89.4 → 0.95 M rows/s) while the +vectorized evaluator loses 47× (558 → 11.8).** The gap between them therefore +*widens* with depth, from 6× to 12×, which is the opposite of what "interpretive +overhead is a constant factor" would predict. + +The reason is the thing this topic is about: a tree-walking interpreter pays +dispatch *per node per row*, so its work is `rows × nodes` of branching on tags, +while the vectorized version pays `nodes` dispatches and amortizes each over a +whole column. That leaves the JIT lane with a precise question rather than a +vague one — compile time is a fixed cost paid once, so break-even rows = +`compile_µs / (µs_per_row_interp − µs_per_row_jit)`. Predict where that lands for +depth 8 before you implement it; the answer is why SQLite still ships a bytecode +VM and HyPer does not. + ## 1. The spectrum (and where each system sits) ``` diff --git a/topics/19-jit/experiments/src/bin/jit_bench.rs b/topics/19-jit/experiments/src/bin/jit_bench.rs index 83d39bc..01b5163 100644 --- a/topics/19-jit/experiments/src/bin/jit_bench.rs +++ b/topics/19-jit/experiments/src/bin/jit_bench.rs @@ -6,11 +6,30 @@ use jit_experiments::{expr::gen_expr, gen_cols, interp, jit, to_rows, vectorized}; use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; const N_COLS: usize = 4; +static STUBBED: AtomicBool = AtomicBool::new(false); + +/// The exercise lanes below are each wrapped in `catch_unwind` and reported as +/// STUB in the table. This replaces the default panic hook so an unimplemented +/// `todo!()` does not dump a trace between two table rows, and records that at +/// least one lane is still a stub. +fn quiet_stubs() { + std::panic::set_hook(Box::new(|_| STUBBED.store(true, Ordering::Relaxed))); +} + +/// One line at the end, for the reader and for verify.sh to count. +fn stub_summary(what: &str) { + if STUBBED.load(Ordering::Relaxed) { + println!("\n[stub — implement {what} to unlock the lanes marked STUB]"); + } +} + fn main() { + quiet_stubs(); println!("=== jit_bench: interpreter vs vectorized vs JIT ===\n"); for depth in [2usize, 4, 6, 8, 10] { @@ -100,6 +119,7 @@ fn main() { println!("crossover math: break-even rows = compile_µs / (µs/row_interp - µs/row_jit)"); println!("fill notes.md prediction table BEFORE implementing jit.rs"); + stub_summary("src/jit.rs"); } fn eval_sum(v: &[f64]) -> f64 { diff --git a/topics/19-jit/reading-cranelift-jit-demo.md b/topics/19-jit/reading-cranelift-jit-demo.md index b8e0bab..ac3f140 100644 --- a/topics/19-jit/reading-cranelift-jit-demo.md +++ b/topics/19-jit/reading-cranelift-jit-demo.md @@ -210,6 +210,14 @@ that our pure `Expr` doesn't need. directly, and what's the fallback boundary (per-node fallback vs whole-expression bailout — pick one and defend it)? +## Done when + +- [ ] You can recite the compile ladder: declare, define, finalize — and say why `define_function` alone does not give you a callable. +- [ ] You can explain what `FunctionBuilder` handling SSA construction saves you from doing. +- [ ] You can state the lifetime contract on the returned pointer and every invariant the `transmute` is assuming. +- [ ] You can time `compile()` across expression depths and say whether it is linear in node count. +- [ ] You wrote answers to all five questions in notes.md, including how you will handle non-f64 values in M19. + ## References **Code** diff --git a/topics/19-jit/reading-graphblas-jit.md b/topics/19-jit/reading-graphblas-jit.md index 64e0cd8..b70f35e 100644 --- a/topics/19-jit/reading-graphblas-jit.md +++ b/topics/19-jit/reading-graphblas-jit.md @@ -188,6 +188,15 @@ counterpart under `Source/generic/` to see Steps 2–3 as a diff. what does getting this wrong cost (constant folded in → cache miss per literal value → compile storm)? +## Done when + +- [ ] You can explain why semirings make the kernel space combinatorial, and why that makes a JIT structurally necessary rather than merely fast. +- [ ] You can state the JIT grain: the specialization, not the query — and say why that grain makes caching effective. +- [ ] You can describe the four-level load ladder and the lifetime of each cache. +- [ ] You can state the cache key rule — shape in, values out — and why including values would defeat it. +- [ ] You can explain what PreJIT does and why harvesting from the cache feeds back into the build. +- [ ] You wrote answers to all five questions in notes.md, including your Cypher expression cache key design. + ## References **Code** diff --git a/topics/19-jit/reading-neumann-vldb11.md b/topics/19-jit/reading-neumann-vldb11.md index 15831d0..2ce5af7 100644 --- a/topics/19-jit/reading-neumann-vldb11.md +++ b/topics/19-jit/reading-neumann-vldb11.md @@ -194,6 +194,14 @@ Read the whole thing — it's short. and what did HyPer add to fix it (group prefetching / SIMD probe batching)? +## Done when + +- [ ] You can explain why operator boundaries are data boundaries, and why that is the deeper cost than dispatch. +- [ ] You can define pipelines and pipeline breakers and identify both in a plan you draw yourself. +- [ ] You can explain how produce/consume turns a tree walk into one flat loop, and why push-based codegen produces one loop where pull-based produces several. +- [ ] You can state the LLVM cocktail rule and say which parts of an expression should stay interpreted. +- [ ] You wrote answers to all five questions in notes.md, including the VLDB '18 counter-result on hash-probe-heavy queries. + ## References **Papers** diff --git a/topics/19-jit/reading-postgres-jit.md b/topics/19-jit/reading-postgres-jit.md index 7af58e7..ba29576 100644 --- a/topics/19-jit/reading-postgres-jit.md +++ b/topics/19-jit/reading-postgres-jit.md @@ -180,6 +180,14 @@ llvmjit.c for the lifecycle. key (expression shape with constants as parameters — count how many distinct shapes a workload of 1000 queries has)? +## Done when + +- [ ] You can explain that Postgres already had bytecode (`ExprState`) before it had a JIT, and what the JIT therefore actually replaces. +- [ ] You can explain tuple deforming and why it is the underrated half of the win. +- [ ] You can name all four ways the `jit_above_cost` gate misfires, and propose a better gate. +- [ ] You can say why the JIT emits one function per `ExprState` with a block per step. +- [ ] You wrote answers to all five questions in notes.md, including the per-query-no-cache versus GraphBLAS-cache-forever contrast. + ## References **Code** diff --git a/topics/19-jit/reading-sqlite-vdbe.md b/topics/19-jit/reading-sqlite-vdbe.md index 7919605..0e7cd9c 100644 --- a/topics/19-jit/reading-sqlite-vdbe.md +++ b/topics/19-jit/reading-sqlite-vdbe.md @@ -178,6 +178,14 @@ it uses; Step 5's coroutine pair is the detour worth taking whole. match). Predict where it lands between interp and JIT in rows/s, then (stretch) build it and check. +## Done when + +- [ ] You can explain what flattening an AST into bytecode buys before any compilation is involved — and check it against this topic's measured interpreter numbers, which fall 94x from 7 nodes to 511. +- [ ] You can say why a register machine beats a stack machine here, and count the ops for `a*b + c*d` under each. +- [ ] You can explain what dispatch cost bytecode removes and what it leaves behind. +- [ ] You can trace `OP_Yield` and explain how flattening gives coroutines for free. +- [ ] You wrote answers to all five questions in notes.md, including a sketch of a bytecode lane for this topic's `Expr` enum. + ## References **Code** diff --git a/topics/19-jit/reading-umbra-tidy-tuples.md b/topics/19-jit/reading-umbra-tidy-tuples.md index 7c3e393..4ae544b 100644 --- a/topics/19-jit/reading-umbra-tidy-tuples.md +++ b/topics/19-jit/reading-umbra-tidy-tuples.md @@ -201,6 +201,15 @@ you don't want two backends. the break-even row count formula and compute it. Does a FalkorDB `WHERE` clause over a 1M-node scan clear it? +## Done when + +- [ ] You can state the compile-latency budget in numbers and explain why LLVM's cost is structural rather than a flag away. +- [ ] You can name three concrete ways Umbra IR differs from LLVM IR. +- [ ] You can explain what Flying Start's single-pass register allocation gives up. +- [ ] You can explain why continuation-passing plus `musttail` is what makes copy-and-patch work. +- [ ] You can say what state must be transferable for an adaptive swap at a morsel boundary. +- [ ] You wrote answers to all five questions in notes.md, including your measured cranelift compile time for a depth-8 expression. + ## References **Papers** diff --git a/topics/20-graphblas/experiments/src/bfs.rs b/topics/20-graphblas/experiments/src/bfs.rs index b6d2d3a..6c12e48 100644 --- a/topics/20-graphblas/experiments/src/bfs.rs +++ b/topics/20-graphblas/experiments/src/bfs.rs @@ -102,7 +102,7 @@ mod tests { #[test] fn pull_matches_oracle() { - check_matches_oracle(|g, at, s| bfs_pull(at, s).0); + check_matches_oracle(|_g, at, s| bfs_pull(at, s).0); } #[test] diff --git a/topics/20-graphblas/experiments/src/bin/gb_bench.rs b/topics/20-graphblas/experiments/src/bin/gb_bench.rs index 0fb99da..95f48a1 100644 --- a/topics/20-graphblas/experiments/src/bin/gb_bench.rs +++ b/topics/20-graphblas/experiments/src/bin/gb_bench.rs @@ -10,14 +10,34 @@ use graphblas_experiments::{ spgemm, spmv, }; use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; +static STUBBED: AtomicBool = AtomicBool::new(false); + +/// The exercise lanes below are each wrapped in `catch_unwind` and reported as +/// STUB in the table. This replaces the default panic hook so an unimplemented +/// `todo!()` does not dump a trace between two table rows, and records that at +/// least one lane is still a stub. +fn quiet_stubs() { + std::panic::set_hook(Box::new(|_| STUBBED.store(true, Ordering::Relaxed))); +} + +/// One line at the end, for the reader and for verify.sh to count. +fn stub_summary(what: &str) { + if STUBBED.load(Ordering::Relaxed) { + println!("\n[stub — implement {what} to unlock the lanes marked STUB]"); + } +} + fn main() { + quiet_stubs(); println!("=== gb_bench ===\n"); spmv_sweep(); spgemm_bench(); bfs_bench(); hyper_bench(); + stub_summary("src/spgemm.rs and src/bfs.rs"); } fn spmv_sweep() { diff --git a/topics/20-graphblas/reading-beamer-sc12.md b/topics/20-graphblas/reading-beamer-sc12.md index 7477030..f48530f 100644 --- a/topics/20-graphblas/reading-beamer-sc12.md +++ b/topics/20-graphblas/reading-beamer-sc12.md @@ -188,6 +188,15 @@ implements all three explicitly in ~100 lines. a SPARSE frontier from a bitmap — O(n) scan), and design the experiment that would confirm it. +## Done when + +- [ ] You can explain why push dies mid-search, in terms of edges checked per useful discovery. +- [ ] You can explain what pull inverts and why its early exit needs the ANY/OR monoid. +- [ ] You can state what pull requires — the reverse graph and a dense frontier — and what that costs in memory. +- [ ] You can describe the two thresholds and why hysteresis is needed. +- [ ] You can write the algebraic translation: push is `vxm`, pull is `mxv`. +- [ ] You wrote answers to all five questions in notes.md, and can reproduce Beamer's waste argument from `gb_bench`'s per-level trace once `bfs_diropt` runs. + ## References **Papers** diff --git a/topics/20-graphblas/reading-davis-toms19.md b/topics/20-graphblas/reading-davis-toms19.md index 81a36f8..b6e4150 100644 --- a/topics/20-graphblas/reading-davis-toms19.md +++ b/topics/20-graphblas/reading-davis-toms19.md @@ -212,6 +212,15 @@ Numbers to retain while you read: the CSR memory in v9 (64-bit) vs v10 — and where the same 2× shows up in our Rust CSR if we switch usize→u32. +## Done when + +- [ ] You can name the four sparsity formats and the rule that switches between them. +- [ ] You can map semiring, mask and accumulator onto executor concepts. +- [ ] You can explain zombies and pending tuples, and why lazy mutation is necessary at all. +- [ ] You can say what non-blocking mode defers and what forces completion. +- [ ] You can explain the iso-value optimization and identify which FalkorDB matrices are iso. +- [ ] You wrote answers to all five questions in notes.md, including the 32-bit index memory computation. + ## References **Papers** diff --git a/topics/20-graphblas/reading-falkordb-delta-matrix.md b/topics/20-graphblas/reading-falkordb-delta-matrix.md index 21c704a..dbda269 100644 --- a/topics/20-graphblas/reading-falkordb-delta-matrix.md +++ b/topics/20-graphblas/reading-falkordb-delta-matrix.md @@ -205,6 +205,15 @@ via the LDBC update workloads. update+read mixes prefer? Predict, then bench both under gb_bench's update workload. +## Done when + +- [ ] You can state the trio and write the read identity `(M ∪ DP) ∖ DM` from memory. +- [ ] You can answer the load-bearing question: why FalkorDB needs its own deltas rather than SuiteSparse's pending tuples. +- [ ] You can explain `delta_mxm` as algebra instead of a flush, and identify where it over-masks. +- [ ] You can describe the two-sided compaction that `wait` performs and what triggers it. +- [ ] You can cost the transposed twin's extra write work per mutation. +- [ ] You wrote answers to all five questions in notes.md, including your Rust representation choice for DP/DM. + ## References **Code** diff --git a/topics/20-graphblas/reading-gustavson-spgemm.md b/topics/20-graphblas/reading-gustavson-spgemm.md index 50ad393..4f2fc9d 100644 --- a/topics/20-graphblas/reading-gustavson-spgemm.md +++ b/topics/20-graphblas/reading-gustavson-spgemm.md @@ -185,6 +185,15 @@ median. per wedge, and reconcile with LAGraph shipping BOTH Sandia_LL (saxpy) and Sandia_LUT (dot) as the fastest per-graph choices. +## Done when + +- [ ] You can derive Gustavson's total work and say why it equals the flop count rather than the output size. +- [ ] You can explain what the SPA does and why it makes scattering O(1). +- [ ] You can state the design space in one sentence: what data structure the SPA is — and connect it to this topic's measured SpGEMM (356.9 ms hash at scale 14, 17.1 M flops). +- [ ] You can explain the unknown-output-size problem and when symbolic-then-numeric beats guessing. +- [ ] You can show, on an example, why masked Gustavson cannot skip work but masked dot can. +- [ ] You wrote answers to all five questions in notes.md, including the dense-SPA memory cost per thread. + ## References **Papers** diff --git a/topics/20-graphblas/reading-lagraph.md b/topics/20-graphblas/reading-lagraph.md index a6627be..e7ea7df 100644 --- a/topics/20-graphblas/reading-lagraph.md +++ b/topics/20-graphblas/reading-lagraph.md @@ -194,6 +194,14 @@ LAGr_PageRankGAP.c's loop. Leave LG_CC_FastSV7.c until M24. assign), and what does each move per level (indices vs nothing — iso!)? +## Done when + +- [ ] You can write the BFS loop as one line of algebra inside a while loop. +- [ ] You can explain what `ANY_SECONDI` computes and why that semiring gives you parents for free. +- [ ] You can give more than one masked spelling of triangle counting and say which one LAGraph picks. +- [ ] You can explain why PageRank needs no mask and is therefore pure bandwidth — check against this topic's measured SpMV ladder (20.72 GB/s at scale 14 falling to 12.26 at scale 20). +- [ ] You wrote answers to all five questions in notes.md, including which inputs the direction switch actually reads. + ## References **Code** diff --git a/topics/20-graphblas/reading-openmp-vs-rayon.md b/topics/20-graphblas/reading-openmp-vs-rayon.md index 3c8dd65..523a78c 100644 --- a/topics/20-graphblas/reading-openmp-vs-rayon.md +++ b/topics/20-graphblas/reading-openmp-vs-rayon.md @@ -208,6 +208,15 @@ What to read, in order: the workspace? Write the four decisions in notes.md — that's the checklist item. +## Done when + +- [ ] You can state the skew problem: equal slices are not equal work, and connect it to the RMAT max degree of 9751 measured in topic 24. +- [ ] You can explain the static answer (cost the work, freeze the plan) and what the flopcount pre-pass costs. +- [ ] You can explain work stealing and name what it gives up (determinism). +- [ ] You can say why both worlds need a small-job guard and what `GB_nthreads` does with a small `work`. +- [ ] You can fill in the trade table from memory. +- [ ] You wrote answers to all five questions in notes.md, including your M20 kernel list. + ## References **Code** diff --git a/topics/20-graphblas/reading-suitesparse-internals.md b/topics/20-graphblas/reading-suitesparse-internals.md index 7414ffd..6ecd2c4 100644 --- a/topics/20-graphblas/reading-suitesparse-internals.md +++ b/topics/20-graphblas/reading-suitesparse-internals.md @@ -206,6 +206,15 @@ else in `Source/mxm/` is implementation of that comment. Then read lose to the HashMap version (SPA = m×8B cold bytes per row team — when m outgrows L2, topic 13's blocking argument bites)? +## Done when + +- [ ] You can explain format switching as a bitmask plus two floats, and name the two thresholds. +- [ ] You can explain why the flopcount pre-pass exists before any allocation. +- [ ] You can describe the coarse/fine task taxonomy and why a fine Gustavson task needs atomics while a coarse one does not. +- [ ] You can say how each task picks between Gustavson and hash accumulation, and how the hash table is sized. +- [ ] You can explain dot3's mask-as-outer-loop and where it crosses over against saxpy3. +- [ ] You wrote answers to all five questions in notes.md, including the RMAT scale at which the dense SPA stops fitting. + ## References **Papers** diff --git a/topics/21-formal/README.md b/topics/21-formal/README.md index 45ca2f6..61e35a5 100644 --- a/topics/21-formal/README.md +++ b/topics/21-formal/README.md @@ -28,6 +28,37 @@ graph LR LEAN["Lean 4"] --> PROOF["machine-checked proof
(unbounded, forever)"] ``` +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin eqsat_bench` — a hand-ordered rewriter on the +canonical trap expression `(a*2)/2`, then on random trees: + +``` +-- the ordering trap: (a*2)/2 -- +hand: cost 5 (1 firing, 11.9 µs) — Div(Shl(Var("a"), Num(1)), Num(2)) + +-- random exprs, 20 seeds per depth -- +depth in cost hand cost hand µs firings + 4 31 21 4.3 4 + 6 127 90 17.3 15 + 8 511 355 102.1 61 + 10 2047 1390 420.7 248 +``` + +**The answer to `(a*2)/2` is `a`. The hand-ordered rewriter returns +`(a << 1) / 2`, cost 5, and stops.** It is not buggy and it did not run out of +rules — it applied a locally excellent rewrite (`*2` → `<<1`, a real strength +reduction) which destroyed the syntactic shape that the cancellation rule was +waiting for. One firing, then a local optimum, forever. + +That is the entire motivation for equality saturation, and it is why the sweep +matters too: hand rewriting only removes about 32% of the cost at every depth +(2047 → 1390 at depth 10), consistently, because the same class of ordering +accident keeps happening. An e-graph does not choose an order — it keeps *all* +the equivalent forms and extracts the cheapest at the end, which is why the egg +column exists and why "phase ordering" is a compiler problem with a data +structure for an answer rather than a heuristic for one. + ## 1. E-graphs: the data structure An e-graph = union-find over e-classes + hashcons (memo) + congruence diff --git a/topics/21-formal/experiments/src/bin/eqsat_bench.rs b/topics/21-formal/experiments/src/bin/eqsat_bench.rs index 9e35899..e7444fd 100644 --- a/topics/21-formal/experiments/src/bin/eqsat_bench.rs +++ b/topics/21-formal/experiments/src/bin/eqsat_bench.rs @@ -8,12 +8,32 @@ use formal_experiments::{ hand::hand_optimize, }; use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; +static STUBBED: AtomicBool = AtomicBool::new(false); + +/// The exercise lanes below are each wrapped in `catch_unwind` and reported as +/// STUB in the table. This replaces the default panic hook so an unimplemented +/// `todo!()` does not dump a trace between two table rows, and records that at +/// least one lane is still a stub. +fn quiet_stubs() { + std::panic::set_hook(Box::new(|_| STUBBED.store(true, Ordering::Relaxed))); +} + +/// One line at the end, for the reader and for verify.sh to count. +fn stub_summary(what: &str) { + if STUBBED.load(Ordering::Relaxed) { + println!("\n[stub — implement {what} to unlock the lanes marked STUB]"); + } +} + fn main() { + quiet_stubs(); println!("=== eqsat_bench ===\n"); trap_case(); sweep(); + stub_summary("src/eqsat.rs"); } fn trap_case() { diff --git a/topics/21-formal/reading-aws-cacm15.md b/topics/21-formal/reading-aws-cacm15.md index 5a6f5a1..549f426 100644 --- a/topics/21-formal/reading-aws-cacm15.md +++ b/topics/21-formal/reading-aws-cacm15.md @@ -148,6 +148,15 @@ counterpart in that 94-line model. 5. Spec-code drift: sketch how the capstone's CI could keep WalReplication.tla honest against the real replication code. +## Done when + +- [ ] You can explain what model checking does and why exhaustive state enumeration is different in kind from testing. +- [ ] You can state the 35-step claim and say what makes an interleaving reachable but rare. +- [ ] You can explain the small-scope hypothesis and say where it holds (protocols) and where it does not. +- [ ] You can state the pitch — exhaustively testable pseudo-code — and say what a TLA+ `Next` action is that pseudo-code is not. +- [ ] You can name what TLA+ did not do for AWS. +- [ ] You wrote answers to all five questions in notes.md, including which capstone protocol clears the cost/benefit bar. + ## References **Papers** diff --git a/topics/21-formal/reading-egg-popl21.md b/topics/21-formal/reading-egg-popl21.md index 2da3d11..a7ba684 100644 --- a/topics/21-formal/reading-egg-popl21.md +++ b/topics/21-formal/reading-egg-popl21.md @@ -201,6 +201,15 @@ tree and orthogonal. 5. Cascades memo vs e-graph: what does Cascades have that egg lacks (physical properties, promises), and vice versa (congruence)? +## Done when + +- [ ] You can explain the e-graph as a set of terms closed under equivalence, and say what congruence adds to union-find. +- [ ] You can trace `(a*2)/2` by hand through iteration 1 and say which unions happen — then compare with the measured hand-rewriter result here, which gets stuck at `(a << 1) / 2`, cost 5. +- [ ] You can explain deferred rebuilding and why `memo` re-canonicalization needs a repair loop. +- [ ] You can explain what an e-class analysis is and give one useful example. +- [ ] You can say why extraction is the weak spot, and what a cost function cannot express. +- [ ] You wrote answers to all five questions in notes.md, including the growth estimate under associativity plus commutativity. + ## References **Papers** diff --git a/topics/21-formal/reading-lean-perceus.md b/topics/21-formal/reading-lean-perceus.md index a5e37ea..5a2a503 100644 --- a/topics/21-formal/reading-lean-perceus.md +++ b/topics/21-formal/reading-lean-perceus.md @@ -176,6 +176,15 @@ preserved by set/remove/wait. 5. Koka's effect types let Perceus assume no hidden aliasing. What's the moral equivalent in Rust that makes `Arc::make_mut` sound? +## Done when + +- [ ] You can explain why immutability means copying and what reference counting costs to avoid it. +- [ ] You can explain borrow inference: not counting what you only look at. +- [ ] You can explain reuse tokens and the runtime RC==1 check they depend on. +- [ ] You can state what "garbage-free" means precisely (peak memory equals live data) and what it assumes. +- [ ] You can say why this belongs in a database curriculum, in terms of where `Arc` costs a Rust engine. +- [ ] You wrote answers to all five questions in notes.md, including your ranking of Lean, TLC and proptest for the `DP ∩ M = ∅` invariant. + ## References **Papers** diff --git a/topics/21-formal/reading-tlaplus-raft.md b/topics/21-formal/reading-tlaplus-raft.md index ec0feeb..98937fd 100644 --- a/topics/21-formal/reading-tlaplus-raft.md +++ b/topics/21-formal/reading-tlaplus-raft.md @@ -191,6 +191,15 @@ class of bug (stuck protocols, not corrupt ones). 5. `[][Next]_vars` allows stuttering. Why is that essential for refinement (mapping a detailed spec onto an abstract one)? +## Done when + +- [ ] You can explain a state as a variable snapshot and an action as a predicate relating now to next. +- [ ] You can explain why `Next` being a disjunction gives you concurrency for free. +- [ ] You can state the model-size discipline that lets TLC finish, and name the knobs. +- [ ] You can explain why `[][Next]_vars` allows stuttering and why that is essential. +- [ ] You can state the difference between safety and liveness and which one TLC checks cheaply. +- [ ] You wrote answers to all five questions in notes.md, including the `Rejoin` action and the longest-log-among-survivors counterexample. + ## References **Papers** diff --git a/topics/21-formal/reading-z3-tacas08.md b/topics/21-formal/reading-z3-tacas08.md index 59b0b94..dfe23a8 100644 --- a/topics/21-formal/reading-z3-tacas08.md +++ b/topics/21-formal/reading-z3-tacas08.md @@ -186,6 +186,15 @@ cited inside the 2008 solver. 5. E-matching triggers: why is trigger selection the "index choice" problem of SMT (too general = blowup, too specific = incomplete)? +## Done when + +- [ ] You can explain DPLL(T): the SAT core proposes, theories dispose. +- [ ] You can explain why Nelson-Oppen requires theories to agree on shared equalities. +- [ ] You can say why Z3's e-graph must carry justifications while egg's need not, and connect it to backtracking. +- [ ] You can explain e-matching and why trigger selection is the index-choice problem of theorem proving. +- [ ] You can encode `x/x -> 1` as an SMT query and say what changes between integers and reals. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/22-benchmarks/experiments/src/bin/bench_suite.rs b/topics/22-benchmarks/experiments/src/bin/bench_suite.rs index 8ba42e3..a556ef3 100644 --- a/topics/22-benchmarks/experiments/src/bin/bench_suite.rs +++ b/topics/22-benchmarks/experiments/src/bin/bench_suite.rs @@ -10,12 +10,32 @@ use bench_experiments::{ zipf::{Scrambled, Uniform, Zipfian}, }; use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; +static STUBBED: AtomicBool = AtomicBool::new(false); + +/// The exercise lanes below are each wrapped in `catch_unwind` and reported as +/// STUB in the table. This replaces the default panic hook so an unimplemented +/// `todo!()` does not dump a trace between two table rows, and records that at +/// least one lane is still a stub. +fn quiet_stubs() { + std::panic::set_hook(Box::new(|_| STUBBED.store(true, Ordering::Relaxed))); +} + +/// One line at the end, for the reader and for verify.sh to count. +fn stub_summary(what: &str) { + if STUBBED.load(Ordering::Relaxed) { + println!("\n[stub — implement {what} to unlock the lanes marked STUB]"); + } +} + fn main() { + quiet_stubs(); println!("=== bench_suite ===\n"); tpch_section(); ycsb_section(); + stub_summary("src/tpch.rs and src/zipf.rs"); } fn tpch_section() { diff --git a/topics/22-benchmarks/reading-boncz-tpch.md b/topics/22-benchmarks/reading-boncz-tpch.md index 9755969..06508f5 100644 --- a/topics/22-benchmarks/reading-boncz-tpch.md +++ b/topics/22-benchmarks/reading-boncz-tpch.md @@ -214,6 +214,14 @@ TPCTC 2013, ~20 pages, one evening — but only with the queries open 5. TPC-H says nothing about updates. What does TPC-C's NewOrder mix test that no TPC-H query can (see reading-oltpbench-tpcc.md)? +## Done when + +- [ ] You can define a choke point and name what Q1, Q6 and Q9 each stress. +- [ ] You can explain why Q6's low selectivity favours branchy evaluation while 50% does not — and check it against topic 17's measured sweep. +- [ ] You can state dbgen's dirty secret (uniform, independent columns) and what it flatters. +- [ ] You can read a published TPC-H number and say what refresh streams and scale factor do to its meaning. +- [ ] You wrote answers to all five questions in notes.md, including the Cypher analogues of Q1/Q6/Q9. + ## References **Papers** diff --git a/topics/22-benchmarks/reading-duckdb-tpch.md b/topics/22-benchmarks/reading-duckdb-tpch.md index 8bce774..0ca2ddd 100644 --- a/topics/22-benchmarks/reading-duckdb-tpch.md +++ b/topics/22-benchmarks/reading-duckdb-tpch.md @@ -155,6 +155,14 @@ aggregation, Q9 is join-order sensitive (try 5. Sketch M22's `CALL ldbc_datagen(sf=1)` equivalent for the capstone: what determinism/answer-shipping properties must it keep? +## Done when + +- [ ] You can explain what a table function is and why shipping the generator inside the engine makes determinism the product. +- [ ] You can say why shipping `answers/` matters more than shipping `queries/`. +- [ ] You can explain what streaming chunks avoids that writing `.tbl` files does not. +- [ ] You can measure DuckDB Q1 and Q6 at SF1 on this machine and compare effective GB/s against this topic's own measured lane (5.2-5.7 GB/s for Q1, 9.0-14.4 for Q6). +- [ ] You wrote answers to all five questions in notes.md, including your `CALL ldbc_datagen` sketch for M22. + ## References **Code** diff --git a/topics/22-benchmarks/reading-oltpbench-tpcc.md b/topics/22-benchmarks/reading-oltpbench-tpcc.md index 75faca7..9c5aa1a 100644 --- a/topics/22-benchmarks/reading-oltpbench-tpcc.md +++ b/topics/22-benchmarks/reading-oltpbench-tpcc.md @@ -185,6 +185,15 @@ VLDB 2013, ~12 pages; §3 is the part that aged well: 5. OLTP-Bench's phased rates: sketch the config that reproduces a cache-warmup-then-spike incident (topic 6's eviction storm). +## Done when + +- [ ] You can explain that an OLTP benchmark measures contention, not speed, and identify TPC-C's hot counter. +- [ ] You can explain NURand and why the skew cannot be preloaded away. +- [ ] You can say what think times are for and what removing them changes. +- [ ] You can state what an honest harness must add (rate control) and why — connect it to topic 34's coordinated-omission lane. +- [ ] You can contrast TPC-C's contention with YCSB-A's; they are not the same shape. +- [ ] You wrote answers to all five questions in notes.md, including your design for a graph analogue of the hot counter. + ## References **Papers** diff --git a/topics/22-benchmarks/reading-ycsb.md b/topics/22-benchmarks/reading-ycsb.md index e01c596..f96090f 100644 --- a/topics/22-benchmarks/reading-ycsb.md +++ b/topics/22-benchmarks/reading-ycsb.md @@ -190,6 +190,15 @@ SoCC 2010, ~12 pages; the design half aged well, the eval half didn't: zipfian to a growing keyspace subtly wrong (hint: zetan staleness, go-ycsb :135)? +## Done when + +- [ ] You can state the factoring: workload equals op mix times key distribution, and name all six mixes. +- [ ] You can derive `P(rank 0) = 1/ζ(n,θ)` and compute the hot-key probability at n=1M, θ=0.99. +- [ ] You can explain the O(1) inverse-CDF sampling trick and why the two fast paths in `next()` exist. +- [ ] You can explain what scrambling changes and what it deliberately preserves. +- [ ] You can predict the uniform-to-Zipf effect per workload against this topic's measured uniform baseline (3.16 Mops/s on B, 0.86 on E) before implementing `zipf.rs`. +- [ ] You wrote answers to all five questions in notes.md, including the coordinated-omission fix for the driver. + ## References **Papers** diff --git a/topics/23-fulltext/experiments/src/bin/fts_bench.rs b/topics/23-fulltext/experiments/src/bin/fts_bench.rs index c76571d..75231bf 100644 --- a/topics/23-fulltext/experiments/src/bin/fts_bench.rs +++ b/topics/23-fulltext/experiments/src/bin/fts_bench.rs @@ -9,6 +9,10 @@ fn time_ms(f: impl FnOnce()) -> f64 { } fn main() { + // An unimplemented exercise lane is an expected state, not a crash: every + // stub lane below is wrapped in catch_unwind, so drop the default panic + // hook to keep their todo!() traces out of the measured output. + std::panic::set_hook(Box::new(|_| {})); println!("== corpus + index build =="); let t = Instant::now(); let c = corpus::gen_corpus(100_000, 50_000, 1.0, 42); diff --git a/topics/23-fulltext/reading-blockmax-wand.md b/topics/23-fulltext/reading-blockmax-wand.md index 9ece0ce..32fc92e 100644 --- a/topics/23-fulltext/reading-blockmax-wand.md +++ b/topics/23-fulltext/reading-blockmax-wand.md @@ -190,6 +190,15 @@ already fixed. max_score may belong to a deleted doc. Is WAND still exact? What's the merge-time fix? +## Done when + +- [ ] You can explain the threshold θ and why top-k means most documents are provably irrelevant. +- [ ] You can compute a pivot from term upper bounds and say what makes the jump safe. +- [ ] You can explain what per-block ceilings fix about the global upper bound. +- [ ] You can say why block-max helps most on common terms, and check it against this topic's measured oracle: 10.378 ms and 272 310 postings for `t0∧t1∧t5` against 0.009 ms and 159 postings for two rare terms. +- [ ] You can state what breaks if the scorer stops having a ceiling. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/23-fulltext/reading-bm25.md b/topics/23-fulltext/reading-bm25.md index 0d14db5..a336447 100644 --- a/topics/23-fulltext/reading-bm25.md +++ b/topics/23-fulltext/reading-bm25.md @@ -195,6 +195,15 @@ The 2009 monograph is ~90 pages; you need two sections: no-information special case. Where would M23 get click/edge feedback to use the full RSJ weight, and is it worth it? +## Done when + +- [ ] You can explain where idf comes from, rather than asserting it. +- [ ] You can derive the tf saturation limit and say what it approaches as tf grows. +- [ ] You can explain what b controls and predict its effect on a corpus with near-uniform lengths. +- [ ] You can say why WAND needs BM25's score ceiling and what a scorer without one costs. +- [ ] You can state what the +0.5 smoothing terms are doing. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/23-fulltext/reading-redisearch.md b/topics/23-fulltext/reading-redisearch.md index 5635447..adf0d53 100644 --- a/topics/23-fulltext/reading-redisearch.md +++ b/topics/23-fulltext/reading-redisearch.md @@ -191,6 +191,14 @@ monomorphized type. graph (node ids = doc ids, roaring hit-sets into masked mxv) change the design? +## Done when + +- [ ] You can state the constraint that shapes the whole design: mutable now, or nothing. +- [ ] You can describe the chained growable block structure per term and the write path through it. +- [ ] You can explain the codec ladder — one trait, many encoders, chosen at compile time — and why that is a codegen decision rather than a runtime one. +- [ ] You can explain how GC, `gc_marker` and `unique_id` let readers survive concurrent deletes. +- [ ] You wrote answers to all questions in notes.md. + ## References **Code** diff --git a/topics/23-fulltext/reading-roaring.md b/topics/23-fulltext/reading-roaring.md index 369d971..ae49dce 100644 --- a/topics/23-fulltext/reading-roaring.md +++ b/topics/23-fulltext/reading-roaring.md @@ -181,6 +181,14 @@ Two short papers, both readable in one sitting: RediSearch → node-id set → GraphBLAS vector, and what would a native roaring-masked mxv save? +## Done when + +- [ ] You can derive the 4096 crossover from bytes per value. +- [ ] You can explain why the representation is chosen per 64K range rather than per set. +- [ ] You can name the kernel matrix idea: one algorithm per container pair. +- [ ] You can say what a 99.9%-dense posting list like this topic's `t0` (df 99 888 of 100 000 docs) should become, and what that costs against the sorted-vec baseline measured here (0.1178 ms for dense∧dense AND). +- [ ] You wrote answers to all five questions in notes.md, including the M20 bitmap-container tie-in. + ## References **Papers** diff --git a/topics/23-fulltext/reading-tantivy.md b/topics/23-fulltext/reading-tantivy.md index 0afe715..02dcefa 100644 --- a/topics/23-fulltext/reading-tantivy.md +++ b/topics/23-fulltext/reading-tantivy.md @@ -175,6 +175,14 @@ Suggested 90-minute read order: actually need to fetch, and in what order — how does the layout minimize round trips? +## Done when + +- [ ] You can say why the term dictionary is an FST rather than a hash map, and list what the FST gives you that a hash cannot. +- [ ] You can describe 128-delta posting blocks with one bit width, and what happens to a final partial block. +- [ ] You can explain what skip data answers without decoding. +- [ ] You can explain why the write path is topic 4's LSM wearing a hat, and contrast LogMergePolicy with leveled compaction. +- [ ] You wrote answers to all five questions in notes.md, including which of WAND's needs `TermInfo.doc_freq` serves. + ## References **Code** diff --git a/topics/23-fulltext/reading-zobel-moffat.md b/topics/23-fulltext/reading-zobel-moffat.md index 6846127..53d8eb6 100644 --- a/topics/23-fulltext/reading-zobel-moffat.md +++ b/topics/23-fulltext/reading-zobel-moffat.md @@ -192,6 +192,15 @@ compression specifics in §4 are 2006's menu — read for the *why* its cost models still bind a BM25+vector hybrid (M23), and which are obsoleted by the ANN side? +## Done when + +- [ ] You can explain granularity: what each posting carries and what that costs. +- [ ] You can state the difference between doc-sorted and impact-sorted postings and which query strategy each enables. +- [ ] You can explain why storing gaps works, and how Zipf makes it work. +- [ ] You can explain the TAAT/DAAT distinction and which one this topic's oracle lane implements. +- [ ] You can say what capped accumulators bound and how that differs from WAND's guarantee. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/24-graph-algorithms/experiments/src/bin/algo_bench.rs b/topics/24-graph-algorithms/experiments/src/bin/algo_bench.rs index 96e74c5..0e21829 100644 --- a/topics/24-graph-algorithms/experiments/src/bin/algo_bench.rs +++ b/topics/24-graph-algorithms/experiments/src/bin/algo_bench.rs @@ -3,6 +3,10 @@ use std::panic::{catch_unwind, AssertUnwindSafe}; use std::time::Instant; fn main() { + // An unimplemented exercise lane is an expected state, not a crash: every + // stub lane below is wrapped in catch_unwind, so drop the default panic + // hook to keep their todo!() traces out of the measured output. + std::panic::set_hook(Box::new(|_| {})); println!("== graphs =="); let t = Instant::now(); let (n, e) = graph::gen_rmat(16, 16, 42); diff --git a/topics/24-graph-algorithms/reading-brandes.md b/topics/24-graph-algorithms/reading-brandes.md index 6ecb642..10909a0 100644 --- a/topics/24-graph-algorithms/reading-brandes.md +++ b/topics/24-graph-algorithms/reading-brandes.md @@ -215,6 +215,15 @@ The stub's failure modes are all boundary conditions of Steps 3–5: changed under a delta matrix that hasn't been flushed (topic 20's wait) — flush first, or compute on the stale main matrix? +## Done when + +- [ ] You can explain what betweenness measures and why the definitional cost is O(n³). +- [ ] You can derive the dependency recurrence from the definition, using the partition-by-predecessor argument. +- [ ] You can explain how one BFS counts paths along the level DAG. +- [ ] You can say why the brute-force version is O(n²) in *memory* as well as O(n³) in time. +- [ ] You can name the four practical traps and say which one bites on a scale-free graph like this topic's RMAT (max degree 9751). +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/24-graph-algorithms/reading-delta-stepping.md b/topics/24-graph-algorithms/reading-delta-stepping.md index c5237e3..79e27ad 100644 --- a/topics/24-graph-algorithms/reading-delta-stepping.md +++ b/topics/24-graph-algorithms/reading-delta-stepping.md @@ -205,6 +205,15 @@ never a wrong answer. the M20 core: which semiring, which vector becomes the bucket, and where does Δ live in the API? +## Done when + +- [ ] You can state relaxation as the one move all SSSP algorithms share. +- [ ] You can explain the Dijkstra/Bellman-Ford trade as order against parallelism, and where Δ sits on that dial. +- [ ] You can say exactly which algorithm Δ=1 gives you with integer weights. +- [ ] You can name the three traps inside the bucket loop and why benign write races are acceptable. +- [ ] You can write SSSP as MIN_PLUS matrix multiplication. +- [ ] You wrote answers to all five questions in notes.md, and predicted a Δ for weights uniform in 1..=255 before running the lane against the measured Dijkstra oracle (42.5 ms, 342 909 heap pops). + ## References **Papers** diff --git a/topics/24-graph-algorithms/reading-gap.md b/topics/24-graph-algorithms/reading-gap.md index f90d8d5..b2e2b19 100644 --- a/topics/24-graph-algorithms/reading-gap.md +++ b/topics/24-graph-algorithms/reading-gap.md @@ -175,6 +175,15 @@ Each file's header comment = required reading (Step 6): benchmark-hostile (hint: nondeterminism, tie-breaking, quality-vs-speed frontier)? +## Done when + +- [ ] You can explain what a kernel is and why the spec binds the kernel rather than the implementation. +- [ ] You can explain why degree skew changes the work and not just the clock — this topic measures max degree 9751 on RMAT against 59 on uniform, with triangle counts of 15.6 M against 5428. +- [ ] You can explain why diameter sets the round count and why road networks are therefore in the suite. +- [ ] You can say why 64 trials from random sources are required, and what source luck does to a single measurement. +- [ ] You can state the baseline problem: reference code that is itself state of the art. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/24-graph-algorithms/reading-lagraph-algos.md b/topics/24-graph-algorithms/reading-lagraph-algos.md index ccf7b52..e4e2c28 100644 --- a/topics/24-graph-algorithms/reading-lagraph-algos.md +++ b/topics/24-graph-algorithms/reading-lagraph-algos.md @@ -188,6 +188,15 @@ read it first. main+DP-DM masked) and their consistency semantics (topic 8's read-your-writes for procedures). +## Done when + +- [ ] You can name the four verbs and express connected components as algebra. +- [ ] You can explain FastSV's `min_2nd` semiring and why it takes the neighbour's grandparent. +- [ ] You can contrast sampling in FastSV against Afforest and count matrix ops against pointer chases. +- [ ] You can write triangle counting as a masked multiplication. +- [ ] You can explain why the flush boundary is the cost in the FalkorDB tie-in. +- [ ] You wrote answers to all five questions in notes.md, including what `CALL algo.wcc()` must do about pending deltas. + ## References **Code** diff --git a/topics/24-graph-algorithms/reading-ligra.md b/topics/24-graph-algorithms/reading-ligra.md index e56d495..909b9ce 100644 --- a/topics/24-graph-algorithms/reading-ligra.md +++ b/topics/24-graph-algorithms/reading-ligra.md @@ -183,6 +183,15 @@ fusability, and neither dominates: Ligra's F-with-CAS cost a SAFE embedding (Rust: Send+Sync bounds, no UDF aborts mid-frontier)? +## Done when + +- [ ] You can define the frontier and both physical representations. +- [ ] You can explain push against pull as whose edges you traverse. +- [ ] You can construct a frontier where the m/20 threshold is the wrong call. +- [ ] You can explain why `edgeMapDenseForward` pushes from all vertices and when that is cheaper. +- [ ] You can compare Ligra's model honestly against GraphBLAS's and say what each makes awkward. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/24-graph-algorithms/reading-louvain-leiden.md b/topics/24-graph-algorithms/reading-louvain-leiden.md index 2d5ac51..0fccd83 100644 --- a/topics/24-graph-algorithms/reading-louvain-leiden.md +++ b/topics/24-graph-algorithms/reading-louvain-leiden.md @@ -198,6 +198,15 @@ machinery that already exists: (connectivity check per community = one BFS each, or one FastSV on the induced subgraph). +## Done when + +- [ ] You can define modularity and say what it scores. +- [ ] You can reproduce Figure 1's failure: how a bridge vertex ends up disconnected from its own community. +- [ ] You can state the general root cause — greedy plus irreversible is unfixable — and what Leiden's refinement changes. +- [ ] You can explain the resolution limit and what γ does about it. +- [ ] You can map one Leiden iteration onto SPA and SpGEMM steps. +- [ ] You wrote answers to all five questions in notes.md, including the topic-16 property test for community connectivity. + ## References **Papers** diff --git a/topics/25-graph-ml/experiments/src/bin/gnn_bench.rs b/topics/25-graph-ml/experiments/src/bin/gnn_bench.rs index 2603ec0..1346215 100644 --- a/topics/25-graph-ml/experiments/src/bin/gnn_bench.rs +++ b/topics/25-graph-ml/experiments/src/bin/gnn_bench.rs @@ -13,6 +13,10 @@ use std::panic::{catch_unwind, AssertUnwindSafe}; use std::time::Instant; fn main() { + // An unimplemented exercise lane is an expected state, not a crash: every + // stub lane below is wrapped in catch_unwind, so drop the default panic + // hook to keep their todo!() traces out of the measured output. + std::panic::set_hook(Box::new(|_| {})); // 64 blocks x 256 = 16,384 vertices; ~30 intra + ~4 inter deg let t = Instant::now(); let (g, labels) = gen_sbm(64, 256, 0.12, 0.00025, 42); diff --git a/topics/25-graph-ml/experiments/src/walks.rs b/topics/25-graph-ml/experiments/src/walks.rs index f49c52e..31588cb 100644 --- a/topics/25-graph-ml/experiments/src/walks.rs +++ b/topics/25-graph-ml/experiments/src/walks.rs @@ -80,16 +80,16 @@ pub fn visit_dist(walks: &[Vec], n: usize) -> Vec { counts } -fn degree_dist(g: &Csr) -> Vec { +pub fn degree_dist(g: &Csr) -> Vec { let m = g.m() as f64; (0..g.n as u32).map(|v| g.degree(v) as f64 / m).collect() } -fn l1(a: &[f64], b: &[f64]) -> f64 { +pub fn l1(a: &[f64], b: &[f64]) -> f64 { a.iter().zip(b).map(|(x, y)| (x - y).abs()).sum() } -fn avg_distinct(walks: &[Vec]) -> f64 { +pub fn avg_distinct(walks: &[Vec]) -> f64 { let mut sum = 0.0; for w in walks { let mut s: Vec = w.clone(); diff --git a/topics/25-graph-ml/reading-gat.md b/topics/25-graph-ml/reading-gat.md index 2a3b9e2..f02e6a3 100644 --- a/topics/25-graph-ml/reading-gat.md +++ b/topics/25-graph-ml/reading-gat.md @@ -175,6 +175,14 @@ is the explanation (question 4: what Cypher surface exposes it). 5. For M25: is GAT worth engine support at all, or is GCN/SAGE + the vector index the 95% case? Argue from the kernel inventory each needs. +## Done when + +- [ ] You can say what GCN's structural constant weights cannot express. +- [ ] You can explain why the softmax is over in-edges of v, and what normalizing over out-edges would mean. +- [ ] You can decompose a GAT layer into SDDMM, segmented softmax and SpMM. +- [ ] You can count edge passes per GAT layer against per GCN layer on this topic's 566 K-edge SBM, whose SpMM measures 4.31 ms at 16.82 GFLOP/s. +- [ ] You wrote answers to all five questions in notes.md, including whether GAT is worth engine support at all. + ## References **Papers** diff --git a/topics/25-graph-ml/reading-gcn.md b/topics/25-graph-ml/reading-gcn.md index 3b1d2b1..9dcaab8 100644 --- a/topics/25-graph-ml/reading-gcn.md +++ b/topics/25-graph-ml/reading-gcn.md @@ -171,6 +171,14 @@ Three built-in ceilings, each motivating a successor: deltas participate in A_hat, and is that the same decision as topic 24's `CALL algo.wcc` three-option question? +## Done when + +- [ ] You can write `A_hat = D^-1/2 (A+I) D^-1/2` and say why its eigenvalues lie in [-1, 1]. +- [ ] You can decompose a layer into one SpMM plus one small dense matmul. +- [ ] You can explain why associativity is a query plan, and count FLOPs both ways on this topic's SBM — the measured SpMM is 4.31 ms against 5.65 ms for the dense transform. +- [ ] You can say what being baked into `A_hat` at training time costs when a node arrives. +- [ ] You wrote answers to all five questions in notes.md, including what pending deltas mean for a forward pass over the M20 graph. + ## References **Papers** diff --git a/topics/25-graph-ml/reading-graphrag-sdk.md b/topics/25-graph-ml/reading-graphrag-sdk.md index bdd25b4..e227f26 100644 --- a/topics/25-graph-ml/reading-graphrag-sdk.md +++ b/topics/25-graph-ml/reading-graphrag-sdk.md @@ -167,6 +167,14 @@ database. 5. M25 acceptance test: pattern + similarity in one query, verified against this SDK's answers on the same data — sketch it. +## Done when + +- [ ] You can draw the pipeline as a dataflow and name what the SDK asks the database for. +- [ ] You can write the single Cypher query that replaces the client-side relationship expansion. +- [ ] You can name all four systems smells and say which one `SET c.embedding = vecf32(...)` inside a loop is. +- [ ] You can say what statistic would give the router a cost model. +- [ ] You wrote answers to all five questions in notes.md, including the M25 acceptance test that puts pattern and similarity in one query. + ## References **Code** diff --git a/topics/25-graph-ml/reading-graphsage.md b/topics/25-graph-ml/reading-graphsage.md index 1528bb8..a4b6786 100644 --- a/topics/25-graph-ml/reading-graphsage.md +++ b/topics/25-graph-ml/reading-graphsage.md @@ -169,6 +169,15 @@ stale is acceptable — question 4 makes this precise. as the stored artifact — which do you ship first, and what does the vector index (topic 14) need to know about staleness either way? +## Done when + +- [ ] You can state the transductive/inductive distinction as a lookup table against a function. +- [ ] You can explain the fan-out explosion and compute nodes touched for B=512, S=(25,10) against a full 2-hop on this topic's SBM (avg degree 34.6). +- [ ] You can explain why neighbour sampling is a page budget for graph access. +- [ ] You can say what bias the sample introduces and how you would measure it. +- [ ] You can explain why inductive is the database-compatible variant, in terms of which embeddings an insert invalidates. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/25-graph-ml/reading-node2vec.md b/topics/25-graph-ml/reading-node2vec.md index de68a98..4e59b4f 100644 --- a/topics/25-graph-ml/reading-node2vec.md +++ b/topics/25-graph-ml/reading-node2vec.md @@ -188,6 +188,15 @@ Everything above maps onto machinery an engine already owns: API, and which should be fixed opinions? Compare FalkorDB's proc_pagerank arg surface (topic 24). +## Done when + +- [ ] You can explain why the walk bias must be second-order to interpolate between BFS-ish and DFS-ish neighbourhoods. +- [ ] You can say what p and q buy — roles against communities — and predict the effect before running the lane. +- [ ] You can state the skip-gram-with-negative-sampling objective. +- [ ] You can explain the alias-table against rejection-sampling trade and compute the expected draw count at p=1, q=0.25. +- [ ] You can explain embeddings as a materialized view and say which ones an edge insert invalidates. +- [ ] You wrote answers to all five questions in notes.md, and compared your walk rate against the measured uniform-walk baseline of 35.1 Msteps/s. + ## References **Papers** diff --git a/topics/25-graph-ml/reading-pyg-message-passing.md b/topics/25-graph-ml/reading-pyg-message-passing.md index 360a497..a18eea9 100644 --- a/topics/25-graph-ml/reading-pyg-message-passing.md +++ b/topics/25-graph-ml/reading-pyg-message-passing.md @@ -169,6 +169,15 @@ are the supporting cast. 5. If M25 exposes ONE kernel to Cypher (`CALL algo.spmm`?), which PyG surface is the right shape to copy, and what stays engine-internal? +## Done when + +- [ ] You can name the three overridable functions and trace one `GCNConv.forward`. +- [ ] You can compute the memory footprint of the COO path's m×d temporary against a CSR SpMM on this topic's graph. +- [ ] You can explain what `message_and_aggregate` fuses and why that is exactly an SpMM. +- [ ] You can explain why attention forces SDDMM as a second primitive. +- [ ] You can say why `reduce='max'` is not a semiring on floats-with-gradients. +- [ ] You wrote answers to all five questions in notes.md, including which single kernel you would expose to Cypher. + ## References **Code** diff --git a/topics/25-graph-ml/reading-transe.md b/topics/25-graph-ml/reading-transe.md index ac4fda8..5d0c0eb 100644 --- a/topics/25-graph-ml/reading-transe.md +++ b/topics/25-graph-ml/reading-transe.md @@ -154,6 +154,15 @@ problem wearing KG clothes (question 3). vectors live (graph metadata? a relations table?) and do they update transactionally with edge-type DDL? +## Done when + +- [ ] You can state the model in one equation and say what it assumes about relations. +- [ ] You can prove the symmetric-relation collapse. +- [ ] You can name the failure modes one arrow per relation cannot express. +- [ ] You can explain why serving is a nearest-neighbour query and what filter the vector index needs. +- [ ] You can say what degenerates when TransE is applied to an untyped single-relation graph like this topic's SBM. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/26-probabilistic/experiments/src/bin/filter_bench.rs b/topics/26-probabilistic/experiments/src/bin/filter_bench.rs index 91afee0..e515cdf 100644 --- a/topics/26-probabilistic/experiments/src/bin/filter_bench.rs +++ b/topics/26-probabilistic/experiments/src/bin/filter_bench.rs @@ -15,6 +15,10 @@ const N: usize = 10_000_000; const Q: usize = 1_000_000; fn main() { + // An unimplemented exercise lane is an expected state, not a crash: every + // stub lane below is wrapped in catch_unwind, so drop the default panic + // hook to keep their todo!() traces out of the measured output. + std::panic::set_hook(Box::new(|_| {})); let mut rng = ChaCha8Rng::seed_from_u64(42); let mut keys: Vec = (0..N).map(|_| rng.gen::() | 1).collect(); // odd = present keys.sort_unstable(); diff --git a/topics/26-probabilistic/reading-bloom-to-ribbon.md b/topics/26-probabilistic/reading-bloom-to-ribbon.md index ba64ba9..184312b 100644 --- a/topics/26-probabilistic/reading-bloom-to-ribbon.md +++ b/topics/26-probabilistic/reading-bloom-to-ribbon.md @@ -207,6 +207,15 @@ for your keys-per-block Poisson mean. `bloom_before_level`). Why does that split follow directly from "ribbon: ~30% less space but several× slower to build and query"? +## Done when + +- [ ] You can state the filter contract: one-sided error, and which side. +- [ ] You can explain why exactly half the bits are set at optimal k, and why that is intuitive. +- [ ] You can name the two sins — 1.44x space and k cache misses — and say which one blocked bloom fixes. +- [ ] You can explain filters as a linear solve, and why the ribbon band makes it O(n). +- [ ] You can say what happens when ribbon construction fails and what RocksDB does about it. +- [ ] You wrote answers to all five questions in notes.md, including why RocksDB picks ribbon for the bottom level — and you have this topic's measured miss costs to compare against: 246 ns binary search, 299 ns BTreeMap, 28 ns HashSet at 224 MB. + ## References **Papers** diff --git a/topics/26-probabilistic/reading-cuckoo-xor.md b/topics/26-probabilistic/reading-cuckoo-xor.md index 8e33066..e831192 100644 --- a/topics/26-probabilistic/reading-cuckoo-xor.md +++ b/topics/26-probabilistic/reading-cuckoo-xor.md @@ -192,6 +192,16 @@ exercise — it's the test a bloom filter *cannot* pass. Rank bloom/cuckoo/xor/ribbon along (updatable, space, query misses) and match each to: memtable filter, routing table with churn, immutable SST. +## Done when + +- [ ] You can explain why bloom cannot delete, in terms of shared bits. +- [ ] You can state the one trick that makes cuckoo *filters* possible: partial-key cuckoo hashing. +- [ ] You can explain why the alternate bucket is `i1 XOR hash(fp)` rather than something simpler. +- [ ] You can say why deletion is only safe for keys actually inserted. +- [ ] You can explain why 4 slots per bucket, with the load-factor numbers. +- [ ] You can explain why the peeling stack makes XOR filters build-once. +- [ ] You wrote answers to all questions in notes.md. + ## References **Papers** diff --git a/topics/26-probabilistic/reading-geo-indexes.md b/topics/26-probabilistic/reading-geo-indexes.md index b41c9db..90c13ed 100644 --- a/topics/26-probabilistic/reading-geo-indexes.md +++ b/topics/26-probabilistic/reading-geo-indexes.md @@ -202,6 +202,15 @@ Read them in pipeline order (encode → step estimate → ranges → neighbors) code (encode + 9-cell range computation + haversine), and what's reused verbatim? +## Done when + +- [ ] You can explain the reframe: making 2D nearness look like key order. +- [ ] You can compute a Morton code by hand and say why interleaving works. +- [ ] You can explain why 26 bits per axis, connected to the zset score's precision. +- [ ] You can describe the candidate-cells, range-scan, exact-verify search and estimate the over-fetch factor. +- [ ] You can explain the curve's seams and what Hilbert fixes. +- [ ] You wrote answers to all questions in notes.md, including the `GEO.ADD`/`GEO.SEARCH` sketch for M26. + ## References **Papers** diff --git a/topics/26-probabilistic/reading-hyperloglog.md b/topics/26-probabilistic/reading-hyperloglog.md index d5b728a..ba0fea8 100644 --- a/topics/26-probabilistic/reading-hyperloglog.md +++ b/topics/26-probabilistic/reading-hyperloglog.md @@ -185,6 +185,15 @@ the ranges the old estimator needed three different formulas for. you'd maintain a per-label HLL inside a graph engine's write path (topic 26 M-log) without making every node-insert O(m). +## Done when + +- [ ] You can explain why rare hash patterns imply many distinct elements. +- [ ] You can say why index bits and pattern bits must not overlap. +- [ ] You can explain what the registers average away and why the harmonic mean. +- [ ] You can explain the sparse encoding and why a PFCOUNT key starts at 30 bytes. +- [ ] You can state the killer feature — merge is max, therefore algebraic — and say what that buys a distributed count. +- [ ] You wrote answers to all five questions in notes.md, including the ZERO/XZERO/VAL comparison against roaring's containers. + ## References **Papers** diff --git a/topics/26-probabilistic/reading-learned-indexes.md b/topics/26-probabilistic/reading-learned-indexes.md index b74e1a6..1692b3e 100644 --- a/topics/26-probabilistic/reading-learned-indexes.md +++ b/topics/26-probabilistic/reading-learned-indexes.md @@ -184,6 +184,16 @@ ALEX — Step 5 binary search within the leaf, and when is it worth zero? (Uniform small leaves fit in one cache line either way.) +## Done when + +- [ ] You can state the reframe: an index is a model of the CDF, and a B-tree is already one. +- [ ] You can explain what RMI provokes and what safety net it lacks. +- [ ] You can explain PGM's inversion: fix the error bound first, then minimize the model. +- [ ] You can construct four points where the shrinking cone closes a segment. +- [ ] You can explain how ε trades segment count against final search width. +- [ ] You can describe an adversarial insert sequence and how ALEX's gapped arrays respond. +- [ ] You wrote answers to all five questions in notes.md. + ## References **Papers** diff --git a/topics/26-probabilistic/reading-postgres-indexam.md b/topics/26-probabilistic/reading-postgres-indexam.md index a547efe..0840778 100644 --- a/topics/26-probabilistic/reading-postgres-indexam.md +++ b/topics/26-probabilistic/reading-postgres-indexam.md @@ -165,6 +165,15 @@ rare case where that order pays. graph workload, and why is that the one topic 4 already measured? (Point-miss cost × miss rate of MATCH lookups.) +## Done when + +- [ ] You can explain what an index AM is and name the three price points behind the one API. +- [ ] You can say what nbtree's cache misses buy you that a filter cannot. +- [ ] You can explain what exactness costs under concurrency and on writes. +- [ ] You can explain why GIN is an inverted index and BRIN is an admitted filter. +- [ ] You can state the precise condition under which BRIN on a column is useful. +- [ ] You wrote answers to all questions in notes.md, including the M26 synthesis — and you have this topic's in-memory baseline (BTreeMap miss at 299 ns) to put beside postgres's page-based numbers. + ## References **Code** ([postgres](https://github.com/postgres/postgres), `src/backend/access/`) diff --git a/topics/26-probabilistic/reading-roaring-internals.md b/topics/26-probabilistic/reading-roaring-internals.md index 2708e66..08073bc 100644 --- a/topics/26-probabilistic/reading-roaring-internals.md +++ b/topics/26-probabilistic/reading-roaring-internals.md @@ -189,6 +189,15 @@ produces runs. ever converts back down, and why is demotion rarer than promotion everywhere? +## Done when + +- [ ] You can state the density crossover and why it is applied per 64K chunk. +- [ ] You can explain what run containers add and which data shape wants them. +- [ ] You can explain the density algebra: kernels chosen pairwise per container type. +- [ ] You can say what happens when a union overflows the array limit. +- [ ] You can explain why cardinality-only operations are the hot path and what they skip. +- [ ] You wrote answers to all questions in notes.md, including the three-adaptive-encodings cross-topic thread with GraphBLAS and HLL. + ## References **Papers** diff --git a/topics/27-streaming/README.md b/topics/27-streaming/README.md index d5474e3..90e7239 100644 --- a/topics/27-streaming/README.md +++ b/topics/27-streaming/README.md @@ -6,6 +6,35 @@ Differential dataflow and DBSP made that rigorous — and FalkorDB's delta matrices (topic 20) are already halfway there conceptually: DP/DM *are* positive and negative Z-sets waiting for an algebra. +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin ivm_bench` — 50 000 nodes / 500 000 edges, then 10 +batches of +90/−10 edges, recomputing each view from scratch per batch: + +``` +view full recompute per batch +triangles 141.6 ms +wedge join 1111.0 ms +reachability (re-BFS) 31.2 ms +``` + +**One hundred changes to half a million edges, and the wedge join spends 1.1 +seconds re-deriving an answer that barely moved.** The batch is 0.02% of the +graph. The recompute is 100% of the work, every time. + +That ratio — change size against work size — is the only motivation incremental +view maintenance needs, and stating it in milliseconds first is what keeps the +rest of the topic honest. The three views are deliberately different shapes: +reachability is cheap to redo (31 ms) so incrementalization has little room and +mostly has to avoid regressions; the wedge join is expensive and *local*, so +delta rules should win enormously; triangles sit between them and are where the +bookkeeping cost of maintaining state starts to show against the recompute it +saves. + +Predict the speedup for each of the three before implementing, and predict which +one's incremental version is *slower* to initialize than a full recompute — that +one is the interesting result, not the 100× one. + ## Our motivation numbers first (Apple M3 Pro, 50K nodes / 500K edges, batches of 100 changes, 2026-07-10) | standing query | full recompute / batch | incremental target | diff --git a/topics/27-streaming/experiments/src/bin/ivm_bench.rs b/topics/27-streaming/experiments/src/bin/ivm_bench.rs index 02a811f..4c672e8 100644 --- a/topics/27-streaming/experiments/src/bin/ivm_bench.rs +++ b/topics/27-streaming/experiments/src/bin/ivm_bench.rs @@ -20,6 +20,10 @@ const BATCH_INS: usize = 90; const BATCH_DEL: usize = 10; fn main() { + // An unimplemented exercise lane is an expected state, not a crash: every + // stub lane below is wrapped in catch_unwind, so drop the default panic + // hook to keep their todo!() traces out of the measured output. + std::panic::set_hook(Box::new(|_| {})); let base = gen_edges(N, M, 42); println!("graph: {} nodes, {} edges; {} batches of +{}/-{}", N, M, BATCHES, BATCH_INS, BATCH_DEL); diff --git a/topics/27-streaming/reading-dbsp.md b/topics/27-streaming/reading-dbsp.md index ec26046..7276026 100644 --- a/topics/27-streaming/reading-dbsp.md +++ b/topics/27-streaming/reading-dbsp.md @@ -181,6 +181,15 @@ questions about what nesting trades against lattice times. what M27 must add (the arranged join state — nothing! wedges need only A itself: the integrals ARE the adjacency matrices). +## Done when + +- [ ] You can explain why Z-sets make deletion a first-class value and why sets cannot. +- [ ] You can name the four operators and write incrementalization in one line. +- [ ] You can prove the bilinear rule by expanding `Q^Δ = D∘Q∘I`. +- [ ] You can explain the chain rule and why it covers a whole dialect rather than one query. +- [ ] You can say how recursion is handled by nested circuits. +- [ ] You wrote answers to all questions in notes.md, including the wedge count — which this topic measures at 1111.0 ms per batch under full recompute. + ## References **Papers** diff --git a/topics/27-streaming/reading-differential-dataflow.md b/topics/27-streaming/reading-differential-dataflow.md index f817b07..4a0a970 100644 --- a/topics/27-streaming/reading-differential-dataflow.md +++ b/topics/27-streaming/reading-differential-dataflow.md @@ -160,6 +160,15 @@ bug you can now name. input change at epoch 2 while iteration from epoch 1 is still running — which updates must NOT be merged? +## Done when + +- [ ] You can explain the delta discipline: weighted, timestamped updates. +- [ ] You can explain what an arrangement is and why sharing one across queries matters. +- [ ] You can explain the incremental join as the bilinear rule on traces, and what "fuel" is for. +- [ ] You can explain why lattice timestamps are required for retractable recursion, not merely convenient. +- [ ] You can show how semi-naive evaluation falls out for free. +- [ ] You wrote answers to all questions in notes.md, including the ordering issue in `IncrementalJoin::step`. + ## References **Papers** diff --git a/topics/27-streaming/reading-kafka-log.md b/topics/27-streaming/reading-kafka-log.md index 91e6544..06bf198 100644 --- a/topics/27-streaming/reading-kafka-log.md +++ b/topics/27-streaming/reading-kafka-log.md @@ -167,6 +167,16 @@ after the paper so the architecture claims have mechanics under them. subscriber disconnects for an hour — and where's the retention-window trade from Step 6 hiding in your answer? +## Done when + +- [ ] You can explain why position is identity in an append-only log. +- [ ] You can explain the dumb-broker/smart-consumer split and what it moves to the client. +- [ ] You can state the mechanical bet: sequential IO plus the OS page cache. +- [ ] You can explain why ordering is per partition only and what that forbids. +- [ ] You can say where the offset lives for each delivery semantic. +- [ ] You can explain log compaction as turning a topic into a table changelog. +- [ ] You wrote answers to all questions in notes.md, including what FalkorDB's existing log already gives M27. + ## References **Papers** diff --git a/topics/27-streaming/reading-materialize-risingwave.md b/topics/27-streaming/reading-materialize-risingwave.md index bab5259..c27a318 100644 --- a/topics/27-streaming/reading-materialize-risingwave.md +++ b/topics/27-streaming/reading-materialize-risingwave.md @@ -160,6 +160,15 @@ RisingWave — Steps 4–5 gets harder (memory pressure from arrangements competes with the graph itself — who evicts?). +## Done when + +- [ ] You can say what production adds to the theory, in failure modes rather than features. +- [ ] You can explain Materialize's identity: indexes are arrangements are memory. +- [ ] You can explain delta joins and why they need an arrangement per input per key. +- [ ] You can contrast RisingWave's hand-written executors plus LSM-on-S3 state with that. +- [ ] You can explain what barriers give you for consistency and recovery. +- [ ] You wrote answers to all questions in notes.md, including the degree-table against diff-arithmetic comparison. + ## References **Code** diff --git a/topics/27-streaming/reading-naiad-timely.md b/topics/27-streaming/reading-naiad-timely.md index c4c9bd9..53bb8de 100644 --- a/topics/27-streaming/reading-naiad-timely.md +++ b/topics/27-streaming/reading-naiad-timely.md @@ -177,6 +177,16 @@ Then the Rust reincarnation — the code anchors, by step: Where does FalkorDB's single-writer serialization make the proof trivial? (That's why M27 can skip most of §4.) +## Done when + +- [ ] You can explain what a logical timestamp says about a message. +- [ ] You can state the completeness problem and what a frontier answers. +- [ ] You can explain could-result-in and progress tracking as a refcount. +- [ ] You can explain why loop nodes must edit the timestamp and what order becomes inside a loop. +- [ ] You can say why progress counts may transiently go negative and why that is safe. +- [ ] You can contrast this with heuristic watermarks and say what each guarantees. +- [ ] You wrote answers to all questions in notes.md. + ## References **Papers** diff --git a/topics/28-cloud-native/experiments/src/bin/tier_bench.rs b/topics/28-cloud-native/experiments/src/bin/tier_bench.rs index 705860f..1c0654e 100644 --- a/topics/28-cloud-native/experiments/src/bin/tier_bench.rs +++ b/topics/28-cloud-native/experiments/src/bin/tier_bench.rs @@ -32,6 +32,10 @@ fn pctl_line(label: &str, lat: &mut Vec) { } fn main() { + // An unimplemented exercise lane is an expected state, not a crash: every + // stub lane below is wrapped in catch_unwind, so drop the default panic + // hook to keep their todo!() traces out of the measured output. + std::panic::set_hook(Box::new(|_| {})); println!("=== tier_bench: {N_KEYS} keys, {READS} zipf({ZIPF_THETA}) point reads ===\n"); // Pre-draw the key stream so every lane sees identical reads. diff --git a/topics/28-cloud-native/experiments/src/cache.rs b/topics/28-cloud-native/experiments/src/cache.rs index d1a451c..e33c994 100644 --- a/topics/28-cloud-native/experiments/src/cache.rs +++ b/topics/28-cloud-native/experiments/src/cache.rs @@ -20,6 +20,8 @@ pub struct LruBlockCache { pub capacity: usize, pub hits: u64, pub misses: u64, + // your get()/insert() bump and compare this to pick a victim + #[allow(dead_code, reason = "read by the eviction you are about to implement")] tick: u64, map: HashMap)>, } diff --git a/topics/28-cloud-native/reading-aurora.md b/topics/28-cloud-native/reading-aurora.md index 46ad2b8..5a2156e 100644 --- a/topics/28-cloud-native/reading-aurora.md +++ b/topics/28-cloud-native/reading-aurora.md @@ -179,6 +179,15 @@ adjacency. What operation must the storage tier then support that S3 doesn't — and is that why Aurora runs its own storage fleet while Neon keeps S3 behind a pageserver? +## Done when + +- [ ] You can count the writes a naive lift-and-shift of a page-based engine to cloud storage produces. +- [ ] You can state the thesis — only the log crosses the network — and what that removes. +- [ ] You can explain the quorum and protection-group scheme and what failure it survives. +- [ ] You can explain LSN and VDL and why one monotonic counter replaces 2PC. +- [ ] You can say what waits at commit and what does not, and why REDO has already run at recovery. +- [ ] You have this topic's measured latency gap to argue against: local NVMe p50 0.10 ms against raw S3 p50 14.17 ms and p99 112.99 ms. + ## References **Papers** diff --git a/topics/28-cloud-native/reading-neon.md b/topics/28-cloud-native/reading-neon.md index 37862ff..1373c09 100644 --- a/topics/28-cloud-native/reading-neon.md +++ b/topics/28-cloud-native/reading-neon.md @@ -183,6 +183,14 @@ would M28's graph branches need the same trick — what query pattern makes a 64-deep ancestor walk show up, and what's the graph equivalent of an image layer (a materialized matrix snapshot at the branch point)? +## Done when + +- [ ] You can state the Postgres contract Neon preserves: WAL in, pages out. +- [ ] You can explain the safekeeper/pageserver split as durability against serving. +- [ ] You can explain why the pageserver is an LSM keyed by (key, LSN). +- [ ] You can explain reconstruction: REDO on the read path, and what it costs a cold read. +- [ ] You can explain why a branch is two numbers, and what that makes free. + ## References **Code** diff --git a/topics/28-cloud-native/reading-slatedb-quickwit.md b/topics/28-cloud-native/reading-slatedb-quickwit.md index 49bf961..629f706 100644 --- a/topics/28-cloud-native/reading-slatedb-quickwit.md +++ b/topics/28-cloud-native/reading-slatedb-quickwit.md @@ -192,6 +192,15 @@ FalkorDB analogue for a graph snapshot object — what belongs in the footer so a reader can route its *second* GET precisely (matrix block index / offsets, label→matrix directory, node-count header)? +## Done when + +- [ ] You can re-price the LSM when fsync costs 50-100 ms and say which decisions invert. +- [ ] You can explain why the manifest is the whole database and what that buys. +- [ ] You can explain single-writer fencing from one conditional PUT. +- [ ] You can describe the cache ladder that buys back the 15 ms, against this topic's measured S3 p50 of 14.17 ms. +- [ ] You can explain checkpoints and clones as copying the manifest, not the data. +- [ ] You can explain hedged requests and predict their effect on the measured p99 of 112.99 ms before implementing `hedge.rs`. + ## References **Code** diff --git a/topics/28-cloud-native/reading-snowflake-s3.md b/topics/28-cloud-native/reading-snowflake-s3.md index b37355d..3ef552f 100644 --- a/topics/28-cloud-native/reading-snowflake-s3.md +++ b/topics/28-cloud-native/reading-snowflake-s3.md @@ -187,6 +187,14 @@ live engine. Which graph representations tolerate immutable ~16 MB chunks well (edge lists / CSR segments, topic 2) and which don't (in-place delta-mutated matrices)? One paragraph in notes.md. +## Done when + +- [ ] You can say what object storage actually is, in terms of which primitives exist. +- [ ] You can explain why the 2008 pages-on-S3 attempt failed. +- [ ] You can state the fix — never modify an object — and where the mutable bit goes instead. +- [ ] You can explain a table version as a list of files, and pruning as the replacement for indexes. +- [ ] You can explain how caches and consistent hashing claw back the gap, and check the size of that gap against this topic's measured 140x local-to-S3 median ratio. + ## References **Papers** diff --git a/topics/28-cloud-native/reading-socrates.md b/topics/28-cloud-native/reading-socrates.md index 9552990..6d6d805 100644 --- a/topics/28-cloud-native/reading-socrates.md +++ b/topics/28-cloud-native/reading-socrates.md @@ -163,6 +163,14 @@ does the compute node's own RBPEX-style local cache over object storage suffice until read replicas (M15) enter? Write the one-paragraph answer in notes.md. +## Done when + +- [ ] You can state why durability and availability are different jobs, and what that separation permits. +- [ ] You can name the four tiers and what each owns. +- [ ] You can explain why commit latency reduces to one small append to the XLOG landing zone. +- [ ] You can explain why page servers are caches and therefore disposable. +- [ ] You can explain what RBPEX preserves across a restart and why that matters given this topic's measured cold-S3 tail. + ## References **Papers** diff --git a/topics/29-distributed-txn/experiments/src/bin/txn_bench.rs b/topics/29-distributed-txn/experiments/src/bin/txn_bench.rs index a39b85f..6db7938 100644 --- a/topics/29-distributed-txn/experiments/src/bin/txn_bench.rs +++ b/topics/29-distributed-txn/experiments/src/bin/txn_bench.rs @@ -25,6 +25,10 @@ const TXNS: usize = 100_000; const THETAS: [f64; 4] = [0.5, 0.9, 1.1, 1.3]; fn main() { + // An unimplemented exercise lane is an expected state, not a crash: every + // stub lane below is wrapped in catch_unwind, so drop the default panic + // hook to keep their todo!() traces out of the measured output. + std::panic::set_hook(Box::new(|_| {})); println!("=== txn_bench: {TXNS} transfers over {ACCOUNTS} accounts, batches of {BATCH} ===\n"); // ---- Lane 1 (provided): workload conflict probability ----------------- diff --git a/topics/29-distributed-txn/experiments/src/hlc.rs b/topics/29-distributed-txn/experiments/src/hlc.rs index 9986ccc..7b07670 100644 --- a/topics/29-distributed-txn/experiments/src/hlc.rs +++ b/topics/29-distributed-txn/experiments/src/hlc.rs @@ -89,6 +89,9 @@ mod tests { } #[test] + // rustc reports the seed assignment to `m` as dead while recv() is still + // a todo!(), because the loop below never gets to read it + #[allow(unused_assignments)] fn l_is_bounded_by_max_physical_time_seen() { // The paper's key bound: l never exceeds the largest pt in the // system, so HLC stays within clock-skew of true time (unlike a diff --git a/topics/29-distributed-txn/reading-calvin.md b/topics/29-distributed-txn/reading-calvin.md index 2ba3678..b10d15d 100644 --- a/topics/29-distributed-txn/reading-calvin.md +++ b/topics/29-distributed-txn/reading-calvin.md @@ -159,6 +159,16 @@ symptom. (reconnaissance traversal, then deterministic re-execution), and what invalidation check would "did the read set move?" become on a graph? +## Done when + +- [ ] You can state why nondeterminism is the reason databases coordinate. +- [ ] You can explain how the sequencer takes consensus off the critical path. +- [ ] You can explain deterministic locking and why lock *ordering* is what makes it work. +- [ ] You can explain why cross-shard reads are pushed rather than requested. +- [ ] You can explain why recovery is replaying inputs, and why replication is cheaper than shipping a write set. +- [ ] You can state the catch — dependent transactions — and connect it to why graph traversals are the hard case for M29. +- [ ] You wrote answers to all six questions in notes.md. + ## References **Papers** diff --git a/topics/29-distributed-txn/reading-foundationdb.md b/topics/29-distributed-txn/reading-foundationdb.md index f1d00b7..7ca0c50 100644 --- a/topics/29-distributed-txn/reading-foundationdb.md +++ b/topics/29-distributed-txn/reading-foundationdb.md @@ -191,6 +191,15 @@ Two design reads to carry out of the topic: What is the graph analogue of a range conflict, and does a 2-hop traversal's read set even fit in a resolver's memory window? +## Done when + +- [ ] You can explain OCC: check at commit, lock nothing. +- [ ] You can name the unbundled roles and what batching happens between them. +- [ ] You can explain the resolver's in-memory conflict window and why sharded resolvers need not talk to each other. +- [ ] You can explain why storage servers may apply writes lazily after commit. +- [ ] You can explain why "failure equals recovery, not repair" is a design choice and what it buys. +- [ ] You wrote answers to all six questions in notes.md, including why a read-only transaction still gets a consistent snapshot without contacting resolvers. + ## References **Papers** diff --git a/topics/29-distributed-txn/reading-percolator-tikv.md b/topics/29-distributed-txn/reading-percolator-tikv.md index 5a5e9b7..8609750 100644 --- a/topics/29-distributed-txn/reading-percolator-tikv.md +++ b/topics/29-distributed-txn/reading-percolator-tikv.md @@ -219,6 +219,16 @@ TiKV, in reading order: `get` suffice for consistent multi-shard *reads*, and what does the TSO become in that design? +## Done when + +- [ ] You can say where classic 2PC blocks and what Percolator replaces the coordinator with. +- [ ] You can name the three column families and what each holds. +- [ ] You can explain why prewrite must fail on any lock, even a newer one. +- [ ] You can identify the commit point precisely and say why it is one atomic write. +- [ ] You can explain how a reader resolves an abandoned transaction from the data alone. +- [ ] You can predict the abort rate at θ=1.1, where this topic measures 86.2% of batches containing a key collision, before implementing `percolator.rs`. +- [ ] You wrote answers to all six questions in notes.md. + ## References **Papers** diff --git a/topics/29-distributed-txn/reading-spanner-hlc.md b/topics/29-distributed-txn/reading-spanner-hlc.md index e8c7c16..51972a5 100644 --- a/topics/29-distributed-txn/reading-spanner-hlc.md +++ b/topics/29-distributed-txn/reading-spanner-hlc.md @@ -201,6 +201,15 @@ commit-wait; schema/evaluation sections are skimmable). TiKV's PD and (b) HLC + uncertainty restarts, which fits a single-region graph store, and what changes if we go multi-region? +## Done when + +- [ ] You can define external consistency and say why clocks do not give it for free. +- [ ] You can explain TrueTime as a clock that confesses its error, and what commit-wait does with that. +- [ ] You can explain why commit-wait's ~2ε sleep does not cap throughput. +- [ ] You can derive HLC's `l <= max pt seen` bound by induction. +- [ ] You can explain the uncertainty-interval alternative: restart the read rather than sleep the write. +- [ ] You wrote answers to all questions in notes.md, and can connect the HLC bound to the invariant the `hlc.rs` test asserts. + ## References **Papers** diff --git a/topics/30-timeseries/README.md b/topics/30-timeseries/README.md index 4c9d3ee..99f0f8e 100644 --- a/topics/30-timeseries/README.md +++ b/topics/30-timeseries/README.md @@ -5,6 +5,34 @@ TSDBs are what you get when every design decision exploits that regularity: append-mostly, time-ordered, compress-by-predicting, partition-by-time, delete-by-dropping-partitions. +## The problem, measured (bench lane 1, provided — runs today) + +`cargo run --release --bin tsdb_bench` — 1 M samples per series shape, 10-second +scrape interval, raw representation = 16.00 B/sample (8 B timestamp + 8 B value): + +``` +shape delta+varint decode +constant 11.00 B/sample 272 Msamples/s +gauge 11.00 B/sample 239 Msamples/s +counter 11.00 B/sample 320 Msamples/s +random 11.00 B/sample 329 Msamples/s +``` + +**11.00 bytes per sample for all four shapes, including the constant one.** A +series that never changes value compresses exactly as well as uniform random +noise, which is absurd on its face and is precisely the finding: delta + varint +encodes the *timestamp* well (regular scrapes delta to a small constant) and the +*float* not at all, because an f64 that repeats is still 8 bytes of mantissa to a +varint that only knows about leading zero bytes. + +So the baseline has cut 16 B to 11 B — a 1.45× win, entirely from the timestamp +column — and left the larger half of every sample untouched. That is the gap +Gorilla's XOR-and-leading-zeros trick is aimed at, and it is why the paper +reports single-digit *bits* per value rather than bytes. Predict the B/sample +Gorilla achieves for each of these four shapes before you implement it; the +spread between `constant` and `random` should go from zero to enormous, and if it +does not, your encoder is not exploiting what the shape gives it. + ## 0. The shape of the problem ``` diff --git a/topics/30-timeseries/experiments/src/bin/tsdb_bench.rs b/topics/30-timeseries/experiments/src/bin/tsdb_bench.rs index 2c6dc27..3d1d209 100644 --- a/topics/30-timeseries/experiments/src/bin/tsdb_bench.rs +++ b/topics/30-timeseries/experiments/src/bin/tsdb_bench.rs @@ -18,6 +18,10 @@ use timeseries_experiments::index::TagIndex; const N: usize = 1_000_000; fn main() { + // An unimplemented exercise lane is an expected state, not a crash: every + // stub lane below is wrapped in catch_unwind, so drop the default panic + // hook to keep their todo!() traces out of the measured output. + std::panic::set_hook(Box::new(|_| {})); println!("=== tsdb_bench: {N} samples/series shape, 10s scrape interval ===\n"); let ts = scrape_timestamps(N, 1_700_000_000_000, 10_000, 100, 42); diff --git a/topics/30-timeseries/reading-gorilla.md b/topics/30-timeseries/reading-gorilla.md index 5a30819..d0c6eb3 100644 --- a/topics/30-timeseries/reading-gorilla.md +++ b/topics/30-timeseries/reading-gorilla.md @@ -191,6 +191,14 @@ prometheus `tsdb/chunkenc/xor.go`, line by line: Gorilla survives (dod timestamps) and what replaces XOR for non-numeric payloads? +## Done when + +- [ ] You can explain delta-of-delta on timestamps and why regular scrapes make it nearly free. +- [ ] You can explain XOR plus leading/trailing zero counts on values, and why that is the half delta+varint cannot compress. +- [ ] You can predict bits per sample for a constant series and for a random one — this topic measures the baseline at a flat 11.00 B/sample for both, which is the gap you are closing. +- [ ] You can say what makes the encoding block-oriented and what a partial block costs. +- [ ] You wrote answers to all questions in notes.md. + ## References **Papers** diff --git a/topics/30-timeseries/reading-monarch-btrdb.md b/topics/30-timeseries/reading-monarch-btrdb.md index 6e42504..5285670 100644 --- a/topics/30-timeseries/reading-monarch-btrdb.md +++ b/topics/30-timeseries/reading-monarch-btrdb.md @@ -180,6 +180,17 @@ for few fat streams, not 10M skinny ones (Q4). churn (edges-added-per-hour rollups). Sketch where an aggregate tree over the M27 changelog would live in FalkorDB. +## Done when + +- [ ] You can state Monarch's founding constraint — you cannot depend on what you monitor — and name two design choices that fall directly out of it. +- [ ] You can explain autonomy by geography: what a zone owns and what it keeps working through a partition. +- [ ] You can say why push beats pull at Monarch's scale, and what it costs. +- [ ] You can explain why typed schemas are the cure for cardinality, and connect that to the tag-index cost you will measure in `index.rs`. +- [ ] You can explain query pushdown as shipping aggregates rather than samples, and say what class of query it cannot serve. +- [ ] You can describe BtrDB's aggregate tree and say which regime it is built for that Monarch is not. +- [ ] You can explain copy-on-write versioning as treating corrections as history, and connect it to the out-of-order tax this topic's `head.rs` lane prices. +- [ ] You wrote answers to all the questions in notes.md. + ## References **Papers** diff --git a/topics/30-timeseries/reading-prometheus-tsdb.md b/topics/30-timeseries/reading-prometheus-tsdb.md index 54ca7a1..0b59d9e 100644 --- a/topics/30-timeseries/reading-prometheus-tsdb.md +++ b/topics/30-timeseries/reading-prometheus-tsdb.md @@ -180,6 +180,14 @@ partition key *is* the age. topology (adjacency) belong in the "labels" (indexed dimensions) or in the "values" (payload)? +## Done when + +- [ ] You can explain the head block against persistent blocks and what compaction does between them. +- [ ] You can explain why series churn is the scaling problem rather than sample volume. +- [ ] You can explain the inverted index over labels and why a selector is a postings intersection (topic 23, again). +- [ ] You can say what out-of-order samples cost and why the head block's design decides it. +- [ ] You wrote answers to all questions in notes.md. + ## References **Papers** diff --git a/topics/30-timeseries/reading-victoriametrics-influx.md b/topics/30-timeseries/reading-victoriametrics-influx.md index b7b4740..633138f 100644 --- a/topics/30-timeseries/reading-victoriametrics-influx.md +++ b/topics/30-timeseries/reading-victoriametrics-influx.md @@ -195,6 +195,16 @@ InfluxDB 3 (Rust) anchors: Which do you pick for `MATCH ... AT TIME t` and why does the answer differ for hot recent history vs year-old history? +## Done when + +- [ ] You can say what both systems agree on with Prometheus, before naming what they reject. +- [ ] You can explain VM's codec ladder: integers first, then lossy *on purpose* — and say what "on purpose" means for a monitoring workload. +- [ ] You can explain why VM caches the query rather than only the postings, and what that assumes about query shape. +- [ ] You can state IOx's bet — delete the engine, keep the pipeline (Parquet plus DataFusion) — and what it inherits for free from topic 12. +- [ ] You can explain IOx's sort-at-snapshot approach to out-of-order data and contrast it with a head block that must accept writes in place. +- [ ] You can fill in the side-by-side bet table from memory, and say which bet you would make for the capstone. +- [ ] You wrote answers to all the questions in notes.md, and have a predicted bytes-per-sample for VM's codec to set against this topic's measured 11.00 B/sample baseline. + ## References **Papers** diff --git a/topics/31-crdts/experiments/src/bin/crdt_bench.rs b/topics/31-crdts/experiments/src/bin/crdt_bench.rs index 2aba8b6..aab49d7 100644 --- a/topics/31-crdts/experiments/src/bin/crdt_bench.rs +++ b/topics/31-crdts/experiments/src/bin/crdt_bench.rs @@ -33,7 +33,13 @@ fn main() { fn stub_lane(name: &str, f: fn()) { println!("\n=== {name} ==="); - if catch_unwind(AssertUnwindSafe(f)).is_err() { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s in src/ to unlock this lane]"); } } diff --git a/topics/32-htap/experiments/src/bin/htap_bench.rs b/topics/32-htap/experiments/src/bin/htap_bench.rs index ee5c143..caadee6 100644 --- a/topics/32-htap/experiments/src/bin/htap_bench.rs +++ b/topics/32-htap/experiments/src/bin/htap_bench.rs @@ -32,7 +32,13 @@ fn main() { fn stub_lane(name: &str, f: fn()) { println!("\n=== {name} ==="); - if catch_unwind(AssertUnwindSafe(f)).is_err() { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s in src/ to unlock this lane]"); } } diff --git a/topics/33-temporal-graphs/experiments/Cargo.lock b/topics/33-temporal-graphs/experiments/Cargo.lock new file mode 100644 index 0000000..75d5991 --- /dev/null +++ b/topics/33-temporal-graphs/experiments/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "temporal-experiments" +version = "0.1.0" +dependencies = [ + "rand", + "rand_chacha", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/33-temporal-graphs/experiments/src/bin/temporal_bench.rs b/topics/33-temporal-graphs/experiments/src/bin/temporal_bench.rs index 0184ae1..1ec3d99 100644 --- a/topics/33-temporal-graphs/experiments/src/bin/temporal_bench.rs +++ b/topics/33-temporal-graphs/experiments/src/bin/temporal_bench.rs @@ -28,7 +28,13 @@ fn main() { fn stub_lane(name: &str, f: fn()) { println!("\n=== {name} ==="); - if catch_unwind(AssertUnwindSafe(f)).is_err() { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s in src/ to unlock this lane]"); } } diff --git a/topics/34-debugging/experiments/Cargo.lock b/topics/34-debugging/experiments/Cargo.lock new file mode 100644 index 0000000..ce5602b --- /dev/null +++ b/topics/34-debugging/experiments/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "debug-experiments" +version = "0.1.0" +dependencies = [ + "rand", + "rand_chacha", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/34-debugging/experiments/src/bin/debug_bench.rs b/topics/34-debugging/experiments/src/bin/debug_bench.rs index 68154fd..b0951eb 100644 --- a/topics/34-debugging/experiments/src/bin/debug_bench.rs +++ b/topics/34-debugging/experiments/src/bin/debug_bench.rs @@ -156,8 +156,14 @@ fn lane3_observability_tax() { println!(); } -fn stub_lane(name: &str, f: impl FnOnce() + std::panic::UnwindSafe) { - if catch_unwind(AssertUnwindSafe(f)).is_err() { +fn stub_lane(name: &str, f: impl FnOnce()) { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s to unlock {name}]\n"); } } diff --git a/topics/35-overload/experiments/Cargo.lock b/topics/35-overload/experiments/Cargo.lock new file mode 100644 index 0000000..999c3e6 --- /dev/null +++ b/topics/35-overload/experiments/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "overload-experiments" +version = "0.1.0" diff --git a/topics/35-overload/experiments/src/bin/overload_bench.rs b/topics/35-overload/experiments/src/bin/overload_bench.rs index 552eec1..26b264c 100644 --- a/topics/35-overload/experiments/src/bin/overload_bench.rs +++ b/topics/35-overload/experiments/src/bin/overload_bench.rs @@ -140,8 +140,14 @@ fn lane3_admission() { println!(); } -fn stub_lane(name: &str, f: impl FnOnce() + std::panic::UnwindSafe) { - if catch_unwind(AssertUnwindSafe(f)).is_err() { +fn stub_lane(name: &str, f: impl FnOnce()) { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s to unlock {name}]\n"); } } diff --git a/topics/36-sharding/experiments/Cargo.lock b/topics/36-sharding/experiments/Cargo.lock new file mode 100644 index 0000000..3a2a11d --- /dev/null +++ b/topics/36-sharding/experiments/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "sharding-experiments" +version = "0.1.0" +dependencies = [ + "rand", + "rand_chacha", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/36-sharding/experiments/src/bin/shard_bench.rs b/topics/36-sharding/experiments/src/bin/shard_bench.rs index 4610669..24e68fd 100644 --- a/topics/36-sharding/experiments/src/bin/shard_bench.rs +++ b/topics/36-sharding/experiments/src/bin/shard_bench.rs @@ -109,8 +109,14 @@ fn lane3_partitioner() { println!(); } -fn stub_lane(name: &str, f: impl FnOnce() + std::panic::UnwindSafe) { - if catch_unwind(AssertUnwindSafe(f)).is_err() { +fn stub_lane(name: &str, f: impl FnOnce()) { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s to unlock {name}]\n"); } } diff --git a/topics/37-distributed-query/experiments/Cargo.lock b/topics/37-distributed-query/experiments/Cargo.lock new file mode 100644 index 0000000..0c9cee9 --- /dev/null +++ b/topics/37-distributed-query/experiments/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "distributed-query-experiments" +version = "0.1.0" +dependencies = [ + "rand", + "rand_chacha", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/37-distributed-query/experiments/src/bin/distq_bench.rs b/topics/37-distributed-query/experiments/src/bin/distq_bench.rs index 690b7b3..846676d 100644 --- a/topics/37-distributed-query/experiments/src/bin/distq_bench.rs +++ b/topics/37-distributed-query/experiments/src/bin/distq_bench.rs @@ -112,8 +112,14 @@ fn lane3_hedge() { println!(); } -fn stub_lane(name: &str, f: impl FnOnce() + std::panic::UnwindSafe) { - if catch_unwind(AssertUnwindSafe(f)).is_err() { +fn stub_lane(name: &str, f: impl FnOnce()) { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s to unlock {name}]\n"); } } diff --git a/topics/38-graphrag-agent-memory/experiments/Cargo.lock b/topics/38-graphrag-agent-memory/experiments/Cargo.lock new file mode 100644 index 0000000..fb517d3 --- /dev/null +++ b/topics/38-graphrag-agent-memory/experiments/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "graphrag-experiments" +version = "0.1.0" +dependencies = [ + "rand", + "rand_chacha", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/38-graphrag-agent-memory/experiments/src/bin/graphrag_bench.rs b/topics/38-graphrag-agent-memory/experiments/src/bin/graphrag_bench.rs index a51d253..13c9290 100644 --- a/topics/38-graphrag-agent-memory/experiments/src/bin/graphrag_bench.rs +++ b/topics/38-graphrag-agent-memory/experiments/src/bin/graphrag_bench.rs @@ -94,8 +94,14 @@ fn lane3_temporal() { println!(); } -fn stub_lane(name: &str, f: impl FnOnce() + std::panic::UnwindSafe) { - if catch_unwind(AssertUnwindSafe(f)).is_err() { +fn stub_lane(name: &str, f: impl FnOnce()) { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s to unlock {name}]\n"); } } diff --git a/topics/39-fraud-identity-graphs/experiments/Cargo.lock b/topics/39-fraud-identity-graphs/experiments/Cargo.lock new file mode 100644 index 0000000..9626d9d --- /dev/null +++ b/topics/39-fraud-identity-graphs/experiments/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "fraud-experiments" +version = "0.1.0" +dependencies = [ + "rand", + "rand_chacha", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/39-fraud-identity-graphs/experiments/src/bin/fraud_bench.rs b/topics/39-fraud-identity-graphs/experiments/src/bin/fraud_bench.rs index ef4080d..e8b8cc5 100644 --- a/topics/39-fraud-identity-graphs/experiments/src/bin/fraud_bench.rs +++ b/topics/39-fraud-identity-graphs/experiments/src/bin/fraud_bench.rs @@ -125,8 +125,14 @@ fn lane3_er() { println!(); } -fn stub_lane(name: &str, f: impl FnOnce() + std::panic::UnwindSafe) { - if catch_unwind(AssertUnwindSafe(f)).is_err() { +fn stub_lane(name: &str, f: impl FnOnce()) { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s to unlock {name}]\n"); } } diff --git a/topics/40-security-attack-graphs/experiments/Cargo.lock b/topics/40-security-attack-graphs/experiments/Cargo.lock new file mode 100644 index 0000000..f3533ac --- /dev/null +++ b/topics/40-security-attack-graphs/experiments/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "attack-experiments" +version = "0.1.0" +dependencies = [ + "rand", + "rand_chacha", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/40-security-attack-graphs/experiments/src/bin/attack_bench.rs b/topics/40-security-attack-graphs/experiments/src/bin/attack_bench.rs index 10164e7..7f4048e 100644 --- a/topics/40-security-attack-graphs/experiments/src/bin/attack_bench.rs +++ b/topics/40-security-attack-graphs/experiments/src/bin/attack_bench.rs @@ -259,8 +259,14 @@ fn lane3_authz() { println!(); } -fn stub_lane(name: &str, f: impl FnOnce() + std::panic::UnwindSafe) { - if catch_unwind(AssertUnwindSafe(f)).is_err() { +fn stub_lane(name: &str, f: impl FnOnce()) { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s to unlock {name}]\n"); } } diff --git a/topics/41-onchain-analytics/experiments/Cargo.lock b/topics/41-onchain-analytics/experiments/Cargo.lock new file mode 100644 index 0000000..70a039c --- /dev/null +++ b/topics/41-onchain-analytics/experiments/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chain-experiments" +version = "0.1.0" +dependencies = [ + "rand", + "rand_chacha", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/41-onchain-analytics/experiments/src/bin/chain_bench.rs b/topics/41-onchain-analytics/experiments/src/bin/chain_bench.rs index a70ea73..415f1f5 100644 --- a/topics/41-onchain-analytics/experiments/src/bin/chain_bench.rs +++ b/topics/41-onchain-analytics/experiments/src/bin/chain_bench.rs @@ -182,8 +182,14 @@ fn lane3_clustering() { println!(); } -fn stub_lane(name: &str, f: impl FnOnce() + std::panic::UnwindSafe) { - if catch_unwind(AssertUnwindSafe(f)).is_err() { +fn stub_lane(name: &str, f: impl FnOnce()) { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s to unlock {name}]\n"); } } diff --git a/topics/42-recommendations-social/experiments/Cargo.lock b/topics/42-recommendations-social/experiments/Cargo.lock new file mode 100644 index 0000000..48d5700 --- /dev/null +++ b/topics/42-recommendations-social/experiments/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "social-experiments" +version = "0.1.0" +dependencies = [ + "rand", + "rand_chacha", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/42-recommendations-social/experiments/src/bin/social_bench.rs b/topics/42-recommendations-social/experiments/src/bin/social_bench.rs index 797a387..5c987f6 100644 --- a/topics/42-recommendations-social/experiments/src/bin/social_bench.rs +++ b/topics/42-recommendations-social/experiments/src/bin/social_bench.rs @@ -260,8 +260,14 @@ fn lane3_linkpred() { println!(); } -fn stub_lane(name: &str, f: impl FnOnce() + std::panic::UnwindSafe) { - if catch_unwind(AssertUnwindSafe(f)).is_err() { +fn stub_lane(name: &str, f: impl FnOnce()) { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s to unlock {name}]\n"); } } diff --git a/topics/42-recommendations-social/experiments/src/linkpred.rs b/topics/42-recommendations-social/experiments/src/linkpred.rs index 4bc7a37..2b9a583 100644 --- a/topics/42-recommendations-social/experiments/src/linkpred.rs +++ b/topics/42-recommendations-social/experiments/src/linkpred.rs @@ -117,7 +117,7 @@ mod tests { // specialist. Common neighbours cannot tell them apart; // Adamic/Adar must weight the specialist far more heavily. let mut adj: Vec> = vec![Default::default(); 110]; - let mut link = |a: u32, b: u32, adj: &mut Vec>| { + let link = |a: u32, b: u32, adj: &mut Vec>| { adj[a as usize].insert(b); adj[b as usize].insert(a); }; diff --git a/topics/42-recommendations-social/experiments/src/pixie.rs b/topics/42-recommendations-social/experiments/src/pixie.rs index 87fa4b6..15554a7 100644 --- a/topics/42-recommendations-social/experiments/src/pixie.rs +++ b/topics/42-recommendations-social/experiments/src/pixie.rs @@ -265,7 +265,7 @@ mod tests { } let (a, b) = (a.unwrap(), b.unwrap()); let mut rng = seeded_rng(3); - let mut list_for = |u: u32, rng: &mut ChaCha8Rng| { + let list_for = |u: u32, rng: &mut ChaCha8Rng| { let q: Vec<(u32, f64)> = g.user_adj[u as usize][..4].iter().map(|&i| (i, 1.0)).collect(); let r = pixie_walk(rng, &g, &q, 60_000, 0.3, None); topk(&r.scores, &g.user_adj[u as usize], 50) diff --git a/topics/43-ops-dependency-graphs/experiments/Cargo.lock b/topics/43-ops-dependency-graphs/experiments/Cargo.lock new file mode 100644 index 0000000..a859e7f --- /dev/null +++ b/topics/43-ops-dependency-graphs/experiments/Cargo.lock @@ -0,0 +1,134 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "opsgraph-experiments" +version = "0.1.0" +dependencies = [ + "rand", + "rand_chacha", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/topics/43-ops-dependency-graphs/experiments/src/bin/ops_bench.rs b/topics/43-ops-dependency-graphs/experiments/src/bin/ops_bench.rs index 9ae8e98..6cb0850 100644 --- a/topics/43-ops-dependency-graphs/experiments/src/bin/ops_bench.rs +++ b/topics/43-ops-dependency-graphs/experiments/src/bin/ops_bench.rs @@ -252,8 +252,14 @@ fn lane3_sampling() { println!(); } -fn stub_lane(name: &str, f: impl FnOnce() + std::panic::UnwindSafe) { - if catch_unwind(AssertUnwindSafe(f)).is_err() { +fn stub_lane(name: &str, f: impl FnOnce()) { + // Silence the default panic hook for the duration of the lane: an + // unimplemented exercise is an expected state, not a crash to report. + let prev = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let r = catch_unwind(AssertUnwindSafe(f)); + std::panic::set_hook(prev); + if r.is_err() { println!("[stub — implement the todo!()s to unlock {name}]\n"); } } diff --git a/verify.sh b/verify.sh index 51f11be..7ae60aa 100755 --- a/verify.sh +++ b/verify.sh @@ -10,31 +10,80 @@ # # ./verify.sh run everything, print the measured lanes # ./verify.sh --summary just the pass/fail table +# ./verify.sh --list show every lane and what it measures, run nothing # ./verify.sh 40 41 only these topics +# ./verify.sh --criterion also run the criterion lanes (slow, topic 0) # # Requires: a Rust toolchain (rustup.rs). First run compiles from scratch and -# takes a few minutes; later runs are cached. +# takes a few minutes; later runs are cached. A full run is ~20 minutes on an +# M3 Pro, most of it in topics 6, 11, 14 and 25, which build big inputs. # # A note on what you will see: each benchmark has a first "lane" that is # implemented and runs today — those are the numbers quoted in the guides. The # later lanes are the reader's exercises and print "[stub — implement the # todo!()s ...]" until you do them. That is intended, not a broken build. +# +# Seven binaries are deliberately NOT in this list, because they measure the +# reader's own implementation and so have nothing to report on a fresh clone: +# +# 03 disk_btree (bench) 05 crash_test 07 server 10 explain +# 04 write_amp 15 partition_test 16 dst_run +# +# Each of those prints one line saying which file to implement, and exits 0. +# Topics 3 and 7 have separate provided lanes here (btree_baseline, +# loopback_bench) that measure the same claims without the reader's code. set -uo pipefail cd "$(dirname "$0")" SUMMARY_ONLY=0 +LIST_ONLY=0 +RUN_CRITERION=0 FILTER=() for arg in "$@"; do case "$arg" in --summary) SUMMARY_ONLY=1 ;; - -h|--help) sed -n '3,20p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + --list) LIST_ONLY=1 ;; + --criterion) RUN_CRITERION=1 ;; + -h|--help) sed -n '3,40p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; *) FILTER+=("$arg") ;; esac done # topic dir : bench binary : what it measures BENCHES=( + "01-storage-engine-landscape:engine-shootout:the RUM conjecture priced — space amp, B-tree vs LSM" + "02-in-memory-structures:rehash_spike:the doubling rehash you can see in the tail" + "03-btree-internals:btree_baseline:height is one lever; cache residency is the other" + "05-durability-wal:fsync_ladder:what a durable commit costs, per sync policy" + "06-buffer-pool:pool_vs_mmap:mmap's tail — the page fault you cannot schedule" + "07-networking-protocols:loopback_bench:pipelining, or the same work at 279x the rate" + "08-transactions-mvcc:txn_bench:MVCC vs one big lock, across read/write mixes" + "09-concurrency:false_sharing:the cost of sharing a cache line you never share" + "09-concurrency:scaling:a global mutex scaling BACKWARDS with cores" + "11-execution-models:exec_bench:tuple-at-a-time vs batch-at-a-time" + "12-columnar-analytics:scan_bench:the memory-bandwidth floor a scan has to beat" + "13-graph-engines:hop_bench:two-hop traversal, adjacency list vs CSR vs SpMV" + "14-vector-search:ann_bench:brute-force recall and the QPS floor it sets" + "15-replication-consensus:repl_lag:follower fsync policy vs ack latency" + "16-testing-correctness:crash_matrix:which planted bugs seeded crash testing catches" + "17-simd:simd_bench:autovectorization vs hand SIMD, per selectivity" + "18-gpu:gpu_bench:the transfer tax, and where the GPU crossover is" + "19-jit:jit_bench:interpreter vs vectorized, and the compile-time break-even" + "20-graphblas:gb_bench:SpMV bandwidth, SpGEMM, and hypersparse index size" + "21-formal:eqsat_bench:the rewrite-ordering trap that hand optimizers fall into" + "22-benchmarks:bench_suite:TPC-H choke points and YCSB tails, measured" + "23-fulltext:fts_bench:BM25 top-k and what exhaustive scoring costs" + "24-graph-algorithms:algo_bench:PageRank, triangles and Dijkstra on RMAT vs uniform" + "25-graph-ml:gnn_bench:the message-passing kernel is an SpMM" + "26-probabilistic:filter_bench:what a point-miss costs before you add a filter" + "27-streaming:ivm_bench:full recompute per batch — the bill incremental view maintenance pays" + "28-cloud-native:tier_bench:local NVMe vs raw S3, at the tail" + "29-distributed-txn:txn_bench:how much conflict the workload itself contains" + "30-timeseries:tsdb_bench:delta+varint as the baseline Gorilla must beat" + "31-crdts:crdt_bench:convergence and the metadata it costs" + "32-htap:htap_bench:freshness vs analytical throughput" + "33-temporal-graphs:temporal_bench:snapshot replay cost vs anchor+delta" "34-debugging:debug_bench:coordinated omission — a closed loop hides its own tail" "35-overload:overload_bench:metastable failure — the outage that outlives its trigger" "36-sharding:shard_bench:mod-N resharding moves almost everything" @@ -47,63 +96,133 @@ BENCHES=( "43-ops-dependency-graphs:ops_bench:one gray failure, thirty-four alerts" ) +# topic dir : criterion bench : what it measures (--criterion only: minutes each) +CRITERION=( + "00-performance-toolbox:cache_ladder:the memory hierarchy, one rung at a time" + "00-performance-toolbox:lookup_shootout:where a HashMap lookup's time actually goes" + "00-performance-toolbox:branch_misprediction:the branch you cannot predict, priced" +) + bold() { printf '\033[1m%s\033[0m\n' "$1"; } green() { printf '\033[32m%s\033[0m' "$1"; } red() { printf '\033[31m%s\033[0m' "$1"; } +if [ $LIST_ONLY -eq 1 ]; then + bold "═══ bin lanes (run by default)" + for entry in "${BENCHES[@]}"; do + IFS=: read -r dir bin what <<<"$entry" + printf ' %-30s %-18s %s\n' "$dir" "$bin" "$what" + done + echo + bold "═══ criterion lanes (--criterion)" + for entry in "${CRITERION[@]}"; do + IFS=: read -r dir bin what <<<"$entry" + printf ' %-30s %-18s %s\n' "$dir" "$bin" "$what" + done + exit 0 +fi + command -v cargo >/dev/null || { red "cargo not found"; echo " — install Rust from https://rustup.rs"; exit 1; } declare -a RESULTS=() FAILED=0 -for entry in "${BENCHES[@]}"; do - IFS=: read -r dir bin what <<<"$entry" - - if [ ${#FILTER[@]} -gt 0 ]; then - match=0 - for f in "${FILTER[@]}"; do [[ "$dir" == *"$f"* ]] && match=1; done - [ $match -eq 1 ] || continue - fi +wanted() { + [ ${#FILTER[@]} -eq 0 ] && return 0 + for f in "${FILTER[@]}"; do [[ "$1" == *"$f"* ]] && return 0; done + return 1 +} - crate="topics/$dir/experiments" - [ -d "$crate" ] || { RESULTS+=("SKIP|$dir|missing"); continue; } +run_lane() { + local dir=$1 bin=$2 what=$3 kind=$4 + local crate="topics/$dir/experiments" + [ -d "$crate" ] || { RESULTS+=("SKIP|$dir/$bin|missing"); return; } [ $SUMMARY_ONLY -eq 1 ] || { echo; bold "═══ topics/$dir — $what"; echo; } + local start out rc dur stubs start=$(date +%s) - out=$( cd "$crate" && cargo run --release --quiet --bin "$bin" 2>&1 ) + if [ "$kind" = criterion ]; then + out=$( cd "$crate" && cargo bench --quiet --bench "$bin" -- --quick --noplot 2>&1 ) + else + out=$( cd "$crate" && cargo run --release --quiet --bin "$bin" 2>&1 ) + fi rc=$? dur=$(( $(date +%s) - start )) if [ $rc -ne 0 ]; then FAILED=1 - RESULTS+=("FAIL|$dir|${dur}s") - [ $SUMMARY_ONLY -eq 1 ] || { echo "$out" | tail -20; } - continue + RESULTS+=("FAIL|$dir/$bin|${dur}s") + # A failure is exactly when the output is wanted, so print it even under + # --summary. (This script used to swallow it, which meant CI reported a red + # build with no way to tell what broke.) + echo + red "── $dir/$bin failed (rc=$rc); last 25 lines:"; echo + printf '%s\n' "$out" | tail -25 + return + fi + + # A lane that needs hardware this machine does not have reports SKIP, not + # PASS: a green tick for a lane that measured nothing is worse than a red one. + if printf '%s\n' "$out" | grep -q 'skipped —'; then + RESULTS+=("SKIP|$dir/$bin|${dur}s, needs hardware this machine lacks") + [ $SUMMARY_ONLY -eq 1 ] || printf '%s\n' "$out" + return fi stubs=$(printf '%s\n' "$out" | grep -c 'stub —' || true) - RESULTS+=("PASS|$dir|${dur}s, $stubs exercise lane(s) unimplemented") + if [ "$stubs" -gt 0 ]; then + RESULTS+=("PASS|$dir/$bin|${dur}s, $stubs exercise lane(s) unimplemented") + else + RESULTS+=("PASS|$dir/$bin|${dur}s, all lanes implemented") + fi [ $SUMMARY_ONLY -eq 1 ] || printf '%s\n' "$out" +} + +for entry in "${BENCHES[@]}"; do + IFS=: read -r dir bin what <<<"$entry" + wanted "$dir" || continue + run_lane "$dir" "$bin" "$what" bin +done + +for entry in "${CRITERION[@]}"; do + IFS=: read -r dir bin what <<<"$entry" + # criterion lanes are slow: only on --criterion, or when named explicitly + if [ $RUN_CRITERION -eq 0 ]; then + wanted "$dir" && [ ${#FILTER[@]} -gt 0 ] || continue + else + wanted "$dir" || continue + fi + run_lane "$dir" "$bin" "$what" criterion done echo bold "═══ summary" echo -printf ' %-8s %-28s %s\n' "result" "topic" "notes" -printf ' %-8s %-28s %s\n' "------" "-----" "-----" +printf ' %-8s %-40s %s\n' "result" "lane" "notes" +printf ' %-8s %-40s %s\n' "------" "----" "-----" for r in "${RESULTS[@]}"; do - IFS='|' read -r status dir note <<<"$r" + IFS='|' read -r status lane note <<<"$r" case "$status" in - PASS) printf ' %-17s %-28s %s\n' "$(green PASS)" "$dir" "$note" ;; - FAIL) printf ' %-17s %-28s %s\n' "$(red FAIL)" "$dir" "$note" ;; - *) printf ' %-8s %-28s %s\n' "$status" "$dir" "$note" ;; + PASS) printf ' %-17s %-40s %s\n' "$(green PASS)" "$lane" "$note" ;; + FAIL) printf ' %-17s %-40s %s\n' "$(red FAIL)" "$lane" "$note" ;; + SKIP) printf ' %-8s %-40s %s\n' "SKIP" "$lane" "$note" ;; + *) printf ' %-8s %-40s %s\n' "$status" "$lane" "$note" ;; esac done echo -if [ $FAILED -eq 0 ]; then +if [ ${#RESULTS[@]} -eq 0 ]; then + echo " Nothing matched. ./verify.sh --list shows every lane." +elif [ $FAILED -eq 0 ]; then echo " Every measured lane ran. The numbers above are the ones quoted in the" echo " guides; timings will differ from the recorded ones on other hardware." + skipped=$(printf '%s\n' "${RESULTS[@]}" | grep -c '^SKIP' || true) + if [ "$skipped" -gt 0 ]; then + echo + echo " $skipped lane(s) reported SKIP: they need hardware this machine does not" + echo " have, so they measured nothing rather than failing. topics/18-gpu is the" + echo " only one that can do this, and its reference numbers are in its notes.md." + fi else echo " Something failed to run — please open an issue with the output above." fi