diff --git a/cmd/bdbench/live.html b/cmd/bdbench/live.html index a992fea..f789bd6 100644 --- a/cmd/bdbench/live.html +++ b/cmd/bdbench/live.html @@ -2,7 +2,12 @@

bdbench live

-
refreshes every 5 s from bdbench.csv
-
+
live state every 2 s (live.json); a report row every minute (bdbench.csv)
+
+
+

seal p50 / p90 / max (ms) — must stay flat with age

block p50 / p90 / max (ms) — interval is 1000

@@ -28,16 +35,30 @@

bdbench live

function line(id,rows,keys,colors,ymax){const c=document.getElementById(id);const W=c.width=c.clientWidth*2,H=c.height=c.clientHeight*2;const x=c.getContext("2d");x.clearRect(0,0,W,H);if(!rows.length)return; let m=ymax||Math.max(1,...keys.flatMap(k=>rows.map(r=>r[k]||0)))*1.1;x.strokeStyle="#2a3240";x.lineWidth=1;for(let g=0;g<=4;g++){const y=H-6-(H-12)*g/4;x.beginPath();x.moveTo(40,y);x.lineTo(W,y);x.stroke();x.fillStyle="#8b949e";x.font="18px sans-serif";x.fillText((m*g/4).toFixed(0),0,y+6)} keys.forEach((k,i)=>{x.strokeStyle=colors[i];x.lineWidth=3;x.beginPath();rows.forEach((r,j)=>{const px=40+(W-44)*j/Math.max(1,rows.length-1),py=H-6-(H-12)*Math.min(1,(r[k]||0)/m);j?x.lineTo(px,py):x.moveTo(px,py)});x.stroke()})} +async function now(){try{const l=await (await fetch("live.json?"+Date.now())).json();const m=Math.floor(l.elapsedSec/60),s=l.elapsedSec%60;document.getElementById("state").textContent=`live · ${m}m ${String(s).padStart(2,"0")}s · block ${l.height} · ${l.blocks} blocks`;const L=l.last10s; + document.getElementById("now").innerHTML=[ + group("Now: seals, last 10 s",[["p50",L.sealP50ms.toFixed(0)+" ms","big"],["p90 (budget 100)",L.sealP90ms.toFixed(0)+" ms",L.sealP90ms>100?"warn":"ok"],["max",L.sealMaxMs.toFixed(0)+" ms"]]), + group("Now: blocks, last 10 s",[["p50",L.blockP50ms.toFixed(0)+" ms","big"],["p90 (interval 1000)",L.blockP90ms.toFixed(0)+" ms",L.blockP90ms>1000?"bad":"ok"],["max",L.blockMaxMs.toFixed(0)+" ms"],["over the interval",L.over+" of "+L.blocks,L.over>0?"bad":"ok"]]), + group("Now: maintenance and heap",[["passes in flight",l.maintenanceInFlight,"big"],["heap live / dead",l.heapLiveMB.toFixed(0)+" / "+l.heapHoleMB.toFixed(0)+" MB"],["cleaner scanned / copied",l.heapScannedMB.toFixed(0)+" / "+l.heapMovedMB.toFixed(0)+" MB"],["shard sync: heap fsync / delta",l.heapFsyncMsAvg.toFixed(1)+" / "+l.heapDeltaMsAvg.toFixed(1)+" ms avg"],["shard sync covers",l.heapSyncKBAvg.toFixed(0)+" KB avg"],["perm merges / folds / packs",l.permMerges+" / "+l.permFolds+" / "+l.permPacks],["perm index written",(l.permIndexMB||0).toFixed(0)+" MB"],["wrong answers so far",l.mismatches,l.mismatches>0?"bad":"ok"]])].join("")}catch(e){}} async function tick(){let t;try{t=await (await fetch("bdbench.csv?"+Date.now())).text()}catch(e){document.getElementById("state").textContent="unreachable";return} const rows=parse(t),last=rows[rows.length-1]||{};const st=document.getElementById("state"); - st.textContent=rows.length?("minute "+last.minute):"waiting for the first minute"; - const cards=[["seal p90",last.seal_p90_ms+" ms","budget 100"],["block p90",last.block_p90_ms+" ms","interval 1000"],["blocks over interval",last.over_budget,"this minute"],["read p99",last.read_p99_us+" µs",""],["put p99 dyna/perm",last.dyna_put_p99_us+"/"+last.perm_put_p99_us+" µs",""],["maintenance",(+last.compress_s+ +last.merge_s+ +last.pack_s).toFixed(1)+" s/min",last.compress_passes+" passes, "+last.skipped+" skipped"],["disk write",last.write_MBps+" MB/s","read "+last.read_MBps],["store",(last.store_MB/1000).toFixed(2)+" GB",last.files+" files"],["history segments","perm "+last.perm_history+" dyna "+last.dyna_history,"bloom "+last.bloom_MB+" MB"],["wrong answers",last.mismatches,""]]; - document.getElementById("cards").innerHTML=cards.map(c=>`
${c[0]}
0?'class=bad':''}>${c[1]}
${c[2]}
`).join(""); + if(!rows.length)st.textContent="waiting for the first minute"; + const tot=(+last.compress_s+ +last.merge_s+ +last.pack_s).toFixed(1); + document.getElementById("cards").innerHTML=[ + group("Protocol path, this minute",[["seal p90 (budget 100 ms)",last.seal_p90_ms+" ms",last.seal_p90_ms>100?"big warn":"big ok"],["block p90 (interval 1000 ms)",last.block_p90_ms+" ms",last.block_p90_ms>1000?"big bad":"big"],["blocks over the interval",last.over_budget,last.over_budget>0?"bad":"ok"],["blocks this minute",last.blocks],["read p99",last.read_p99_us+" µs"],["put p99 dyna / perm",last.dyna_put_p99_us+" / "+last.perm_put_p99_us+" µs"]]), + group("Maintenance, this minute",[["seconds of work",tot+" s","big"],["compress passes / s",last.compress_passes+" / "+last.compress_s],["merge passes / s",last.merge_passes+" / "+last.merge_s],["pack passes / s",last.pack_passes+" / "+last.pack_s],["skipped (one in flight)",last.skipped]]), + group("Disk",[["write",last.write_MBps+" MB/s","big"],["read",last.read_MBps+" MB/s"],["store size",(last.store_MB/1000).toFixed(2)+" GB"],["files",last.files]]), + group("Store",[["history segments perm / dyna",last.perm_history+" / "+last.dyna_history],["resident filters",last.bloom_MB+" MB"],["wrong answers",last.mismatches,last.mismatches>0?"big bad":"big ok"]])].join(""); line("c_seal",rows,["seal_p50_ms","seal_p90_ms","seal_max_ms"],["#3fb950","#d29922","#f85149"]); line("c_block",rows,["block_p50_ms","block_p90_ms","block_max_ms"],["#3fb950","#d29922","#f85149"],1000); line("c_maint",rows,["compress_s","merge_s","pack_s"],["#58a6ff","#bc8cff","#39c5cf"]); line("c_disk",rows.map(r=>({w:r.write_MBps,g:r.store_MB/1000*10})),["w","g"],["#d29922","#58a6ff"]); const T=document.getElementById("t");T.innerHTML=""+cols.map(c=>""+c.replace(/_/g," ")+"").join("")+""+rows.slice().reverse().map(r=>""+cols.map(c=>{let cls="";if(c=="seal_p90_ms"&&r[c]>100)cls="warn";if(c=="over_budget"&&r[c]>0)cls="bad";if(c=="mismatches"&&r[c]>0)cls="bad";return `${r[c]}`}).join("")+"").join("")} -fetch("run.json").then(r=>r.json()).then(c=>{document.getElementById("sub").textContent=`${c.stores} store(s) × ${c.shards} shards, seal limit ${c.sealLimit}, window ${c.window}, maintenance every ${c.compressEvery} blocks, pack every ${c.packEvery} · per ${c.interval} block: ${c.dynaPuts} dyna puts, ${c.permPuts} perm puts, ${c.reads} reads over ${c.hotKeys} hot keys, values ${c.valueMin}-${c.valueMax} B · ${c.dir} · started ${c.started} for ${c.duration} · refreshes every 5 s from bdbench.csv`}).catch(()=>{}); -tick();setInterval(tick,5000); +function group(title,rows){return `

${title}

${rows.map(r=>``).join("")}
${r[0]}${r[1]}
`} +fetch("run.json").then(r=>r.json()).then(c=>{document.getElementById("config").innerHTML=[ + group("Store",[["dynamic layer",c.dynaHeap?"heap with holes":"sealed segments"],["stores on the disk",c.stores],["shards per store",c.shards],["seal limit (records/shard layer)",c.sealLimit],["window (blocks)",c.window]]), + group("Load per block per store",[["dynamic puts",c.dynaPuts],["permanent puts",c.permPuts],["reads",c.reads],["hot dynamic keys",c.hotKeys],["value bytes",c.valueMin+" to "+c.valueMax]]), + group("Schedule",[["block interval",c.interval],["duration",c.duration],["maintenance every (blocks)",c.compressEvery],["pack every (blocks)",c.packEvery],["started (UTC)",c.started]]), + group("Run",[["directory",c.dir.split("/").slice(-1)[0]],["seed",c.seed]])].join("")}).catch(()=>{}); +now();tick();setInterval(now,2000);setInterval(tick,5000); diff --git a/cmd/bdbench/main.go b/cmd/bdbench/main.go index 643f6e0..f0e7d46 100644 --- a/cmd/bdbench/main.go +++ b/cmd/bdbench/main.go @@ -20,6 +20,7 @@ package main import ( + "bytes" _ "embed" "encoding/csv" "encoding/json" @@ -64,6 +65,9 @@ type config struct { seed uint64 pprof string http string + dynaHeap bool + permFiles bool + phase bool } //go:embed live.html @@ -91,6 +95,9 @@ func parseFlags() (config, error) { flag.Uint64Var(&c.seed, "seed", 1, "random seed") flag.StringVar(&c.pprof, "pprof", "", "serve net/http/pprof on this address (e.g. 127.0.0.1:6061)") flag.StringVar(&c.http, "http", "127.0.0.1:8098", "serve the live page and the run's files here; empty disables") + flag.BoolVar(&c.dynaHeap, "dyna-heap", false, "dynamic layer as a heap with holes (proposal 2026-09-16) instead of sealed segments") + flag.BoolVar(&c.permFiles, "perm-files", false, "both layers as files of entries: the heap and the permanent layer with buckets of keys (implies -dyna-heap)") + flag.BoolVar(&c.phase, "maintenance-phase", false, "offset each store's maintenance cadence by its share of the period, so stores in lockstep do not all maintain at once") flag.Parse() if flag.NArg() != 0 { return c, fmt.Errorf("unexpected arguments: %q", flag.Args()) @@ -129,6 +136,15 @@ func (s *samples) add(d time.Duration) { s.mu.Unlock() } +// addAll takes a block's worth of samples under one lock: forty +// thousand reads a block, nine stores, made the recorder's lock a +// fifth of the benchmark's CPU. +func (s *samples) addAll(v []time.Duration) { + s.mu.Lock() + s.v = append(s.v, v...) + s.mu.Unlock() +} + func (s *samples) take() []time.Duration { s.mu.Lock() v := s.v @@ -149,10 +165,23 @@ func pct(sorted []time.Duration, p float64) time.Duration { func ms(d time.Duration) string { return strconv.FormatFloat(float64(d)/1e6, 'f', 1, 64) } func us(d time.Duration) string { return strconv.FormatInt(int64(d/time.Microsecond), 10) } +// recent is one block as the live page sees it: when it ended and +// what it and its seal cost. +type recent struct { + at time.Time + block, seal time.Duration + store int +} + // tallies is what every store adds to and the report takes from. type tallies struct { blockTimes, sealTimes, dynaPut, permPut, readT samples blocks, over, mismatches atomic.Uint64 + inFlight atomic.Int64 // Maintenance passes running now + + ringMu sync.Mutex + ring [1024]recent // The last blocks, for the live state + ringN uint64 mu sync.Mutex passes map[string]int @@ -188,10 +217,13 @@ type store struct { // A few hot keys are checked on read: the value the store returns // must be the value last written. A platform that only times // answers cannot tell a fast wrong answer from a fast right one. - last map[[32]byte][]byte - height uint64 - maintaining atomic.Bool - maintWG sync.WaitGroup + last map[[32]byte][]byte + height uint64 + phase uint64 // Blocks this store's maintenance cadence is offset by + // A block's samples, handed to the tallies once per block + dynaT, permT, readT []time.Duration + maintaining atomic.Bool + maintWG sync.WaitGroup } const ( @@ -201,7 +233,14 @@ const ( func openStore(c config, id int) (*store, error) { dir := filepath.Join(c.dir, fmt.Sprintf("store-%d", id)) - kv, err := blockchainDB.NewKVShardN(dir, c.shards, c.sealLimit) + open := blockchainDB.NewKVShardN + if c.dynaHeap { + open = blockchainDB.NewKVShardHeapN + } + if c.permFiles { + open = blockchainDB.NewKVShardFilesN + } + kv, err := open(dir, c.shards, c.sealLimit) if err != nil { return nil, fmt.Errorf("open store %d: %w", id, err) } @@ -215,6 +254,9 @@ func openStore(c config, id int) (*store, error) { } s := &store{id: id, kv: kv, rnd: blockchainDB.NewFastRandom(seed), hot: make([][32]byte, c.hotKeys), permKeys: make([][32]byte, 0, permSample), last: make(map[[32]byte][]byte, checked)} + if c.phase && c.compressEvery > 0 { + s.phase = uint64(id) * c.compressEvery / uint64(c.stores) + } // Hot dynamic keys are rewritten with a skew (index = hot * r^2, so // the low indexes take most writes); permanent keys are always new, // and a bounded sample of them, across all ages, is what the @@ -228,6 +270,12 @@ func openStore(c config, id int) (*store, error) { func (s *store) block(c config, t *tallies) error { s.height++ start := time.Now() + s.dynaT, s.permT, s.readT = s.dynaT[:0], s.permT[:0], s.readT[:0] + defer func() { + t.dynaPut.addAll(s.dynaT) + t.permPut.addAll(s.permT) + t.readT.addAll(s.readT) + }() for i := 0; i < c.dynaPuts; i++ { r := float64(s.rnd.UintN(1<<20)) / (1 << 20) k := s.hot[int(r*r*float64(c.hotKeys))%c.hotKeys] @@ -236,19 +284,19 @@ func (s *store) block(c config, t *tallies) error { if err := s.kv.PutDyna(k, v); err != nil { return fmt.Errorf("store %d PutDyna: %w", s.id, err) } - t.dynaPut.add(time.Since(at)) + s.dynaT = append(s.dynaT, time.Since(at)) if len(s.last) < checked || s.last[k] != nil { s.last[k] = v } } for i := 0; i < c.permPuts; i++ { k := s.rnd.NextHash() - v := s.rnd.RandBuff(c.valueMin, c.valueMax) + v := permValue(k, c.valueMin, c.valueMax) at := time.Now() if err := s.kv.PutPerm(k, v); err != nil { return fmt.Errorf("store %d PutPerm: %w", s.id, err) } - t.permPut.add(time.Since(at)) + s.permT = append(s.permT, time.Since(at)) if len(s.permKeys) < permSample { s.permKeys = append(s.permKeys, k) } else if s.rnd.UintN(64) == 0 { // Keep the sample spread across every age @@ -259,20 +307,32 @@ func (s *store) block(c config, t *tallies) error { for i := 0; i < c.reads; i++ { var k [32]byte var get func([32]byte) ([]byte, error) + perm := false switch pick := s.rnd.UintN(6); { case pick < 3: k, get = s.hot[int(s.rnd.UintN(uint(c.hotKeys)))], s.kv.GetDyna case pick < 5 && len(s.permKeys) > 0: - k, get = s.permKeys[int(s.rnd.UintN(uint(len(s.permKeys))))], s.kv.GetPerm + // A permanent key of any age, through the deep read the + // adapter uses for anything older than the window (Get + // answers the permanent layer's window only, by design) + k, get, perm = s.permKeys[int(s.rnd.UintN(uint(len(s.permKeys))))], s.kv.GetDeep, true default: k, get = s.rnd.NextHash(), s.kv.Get } at := time.Now() v, err := get(k) - t.readT.add(time.Since(at)) + s.readT = append(s.readT, time.Since(at)) if err != nil && !errors.Is(err, os.ErrNotExist) && !strings.Contains(err.Error(), "not found") { return fmt.Errorf("store %d read: %w", s.id, err) } + // A permanent key that was written must come back, with the + // value its key derives; a checked hot key must come back as + // last written. A platform that only times answers cannot + // tell a fast wrong answer, or a fast "not found", from a + // right one. + if perm && (err != nil || !bytes.Equal(v, permValue(k, c.valueMin, c.valueMax))) { + t.mismatches.Add(1) + } if want, ok := s.last[k]; ok && err == nil && string(v) != string(want) { t.mismatches.Add(1) } @@ -281,16 +341,21 @@ func (s *store) block(c config, t *tallies) error { if err := s.kv.SealBlock(s.height); err != nil { return fmt.Errorf("store %d SealBlock: %w", s.id, err) } - t.sealTimes.add(time.Since(at)) + sealTook := time.Since(at) + t.sealTimes.add(sealTook) took := time.Since(start) t.blockTimes.add(took) t.blocks.Add(1) + t.ringMu.Lock() + t.ring[t.ringN%uint64(len(t.ring))] = recent{at: time.Now(), block: took, seal: sealTook, store: s.id} + t.ringN++ + t.ringMu.Unlock() if took > c.interval { t.over.Add(1) } else { time.Sleep(c.interval - took) } - if c.compressEvery > 0 && s.height%c.compressEvery == 0 { + if c.compressEvery > 0 && (s.height+s.phase)%c.compressEvery == 0 { s.maintain(c, t) } return nil @@ -307,9 +372,11 @@ func (s *store) maintain(c config, t *tallies) { } height := s.height s.maintWG.Add(1) + t.inFlight.Add(1) go func() { defer s.maintWG.Done() defer s.maintaining.Store(false) + defer t.inFlight.Add(-1) at := time.Now() err := s.kv.Compress() t.note("compress", time.Since(at)) @@ -353,6 +420,8 @@ func procIO() (read, write uint64) { return } +// dirSize counts the bytes the files occupy, not their apparent size: +// a heap releases regions with punched holes and keeps its length. func dirSize(dir string) (files int, bytes int64) { _ = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { if err != nil || d.IsDir() { @@ -360,13 +429,140 @@ func dirSize(dir string) (files int, bytes int64) { } if info, err := d.Info(); err == nil { files++ - bytes += info.Size() + if st, ok := info.Sys().(*syscall.Stat_t); ok { + bytes += st.Blocks * 512 + } else { + bytes += info.Size() + } } return nil }) return } +// liveState is what the page shows between report rows: the last ten +// seconds of blocks and seals, and the run's running totals. Written +// to live.json every two seconds. +func (t *tallies) liveState(c config, stores []*store, start time.Time) []byte { + t.ringMu.Lock() + cut := time.Now().Add(-10 * time.Second) + var bt, st []time.Duration + var perStore [][]time.Duration + n := t.ringN + if n > uint64(len(t.ring)) { + n = uint64(len(t.ring)) + } + for i := uint64(0); i < n; i++ { + r := t.ring[(t.ringN-1-i)%uint64(len(t.ring))] + if r.at.Before(cut) { + break + } + bt = append(bt, r.block) + st = append(st, r.seal) + for len(perStore) <= r.store { + perStore = append(perStore, nil) + } + perStore[r.store] = append(perStore[r.store], r.seal) + } + t.ringMu.Unlock() + // Each store's seal p90 over the same window: a tail that is one + // store's looks different from a tail every store shares + storeP90 := make([]float64, len(perStore)) + for i, v := range perStore { + sort.Slice(v, func(a, b int) bool { return v[a] < v[b] }) + storeP90[i] = float64(pct(v, .9)) / 1e6 + } + sort.Slice(bt, func(i, j int) bool { return bt[i] < bt[j] }) + sort.Slice(st, func(i, j int) bool { return st[i] < st[j] }) + over := 0 + for _, d := range bt { + if d > c.interval { + over++ + } + } + var height uint64 + hs := sumHeap(stores) + holes, live, scanned, moved, syncs, syncBytes, heapFsync, deltaSync := hs.holes, hs.live, hs.scanned, hs.moved, hs.syncs, hs.bytes, hs.fsync, hs.delta + var permMerges, permFolds, permPacks, permIndexBytes uint64 + for _, s := range stores { + if s.height > height { + height = s.height + } + for _, sh := range s.kv.Shards { + if sh.Perm != nil { + m, f, pk, ib := sh.Perm.Counters() + permMerges, permFolds, permPacks, permIndexBytes = permMerges+m, permFolds+f, permPacks+pk, permIndexBytes+ib + } + } + } + b, _ := json.Marshal(map[string]any{ + "elapsedSec": int(time.Since(start).Seconds()), "blocks": t.blocks.Load(), "height": height, + "last10s": map[string]any{"blocks": len(bt), "over": over, + "blockP50ms": float64(pct(bt, .5)) / 1e6, "blockP90ms": float64(pct(bt, .9)) / 1e6, "blockMaxMs": float64(pct(bt, 1)) / 1e6, + "sealP50ms": float64(pct(st, .5)) / 1e6, "sealP90ms": float64(pct(st, .9)) / 1e6, "sealMaxMs": float64(pct(st, 1)) / 1e6, + "storeSealP90ms": storeP90}, + "maintenanceInFlight": t.inFlight.Load(), "mismatches": t.mismatches.Load(), + "heapHoleMB": float64(holes) / 1e6, "heapLiveMB": float64(live) / 1e6, + "heapScannedMB": float64(scanned) / 1e6, "heapMovedMB": float64(moved) / 1e6, + "heapSyncs": syncs, "heapSyncKBAvg": float64(syncBytes) / 1e3 / float64(max(syncs, 1)), + "permMerges": permMerges, "permFolds": permFolds, "permPacks": permPacks, "permIndexMB": float64(permIndexBytes) / 1e6, + "heapFsyncMsAvg": float64(heapFsync) / 1e6 / float64(max(syncs, 1)), "heapDeltaMsAvg": float64(deltaSync) / 1e6 / float64(max(syncs, 1)), + "heapReleases": hs.releases, "heapReleaseMs": hs.release.Milliseconds(), "heapSnapshots": hs.snapshots, "heapSnapshotMs": hs.snapshot.Milliseconds(), + }) + return b +} + +// heapSplit sums the heap layer's counters across the stores: the +// mover's and the sync's cost, cumulative since the start. +type heapSplit struct { + holes, live int64 + scanned, moved, syncs, bytes uint64 + fsync, delta time.Duration + releases, snapshots uint64 + release, snapshot time.Duration +} + +func (a heapSplit) minus(b heapSplit) heapSplit { + return heapSplit{holes: a.holes, live: a.live, + scanned: a.scanned - b.scanned, moved: a.moved - b.moved, syncs: a.syncs - b.syncs, bytes: a.bytes - b.bytes, + fsync: a.fsync - b.fsync, delta: a.delta - b.delta, releases: a.releases - b.releases, snapshots: a.snapshots - b.snapshots, + release: a.release - b.release, snapshot: a.snapshot - b.snapshot} +} + +func sumHeap(stores []*store) (hs heapSplit) { + for _, s := range stores { + for _, sh := range s.kv.Shards { + if sh.Heap == nil { + continue + } + h, l := sh.Heap.HoleRatio() + hs.holes, hs.live = hs.holes+h, hs.live+l + sc, mv := sh.Heap.Cleaned() + hs.scanned, hs.moved = hs.scanned+sc, hs.moved+mv + n, b, hf, ds := sh.Heap.SyncCost() + hs.syncs, hs.bytes, hs.fsync, hs.delta = hs.syncs+n, hs.bytes+b, hs.fsync+hf, hs.delta+ds + r, rt, sn, st := sh.Heap.MoverCost() + hs.releases, hs.release, hs.snapshots, hs.snapshot = hs.releases+r, hs.release+rt, hs.snapshots+sn, hs.snapshot+st + } + } + return hs +} + +// permValue is the value a permanent key carries: derived from the +// key, so that any read of the key can be checked without remembering +// what was written. Its length spreads over [min, max] by the key. +func permValue(k [32]byte, min, max uint) []byte { + n := int(min) + if max > min { + n += int(k[31]) * int(max-min) / 255 + } + v := make([]byte, n) + for i := 0; i < n; i += len(k) { + copy(v[i:], k[:]) + } + return v +} + func fail(what string, err error) { fmt.Fprintln(os.Stderr, "bdbench:", what+":", err) os.Exit(1) @@ -406,7 +602,7 @@ func main() { "dir": c.dir, "stores": c.stores, "duration": c.duration.String(), "interval": c.interval.String(), "shards": c.shards, "sealLimit": c.sealLimit, "window": c.window, "compressEvery": c.compressEvery, "packEvery": c.packEvery, "dynaPuts": c.dynaPuts, "permPuts": c.permPuts, "reads": c.reads, "hotKeys": c.hotKeys, - "valueMin": c.valueMin, "valueMax": c.valueMax, "seed": c.seed, "started": time.Now().UTC().Format(time.RFC3339), + "valueMin": c.valueMin, "valueMax": c.valueMax, "seed": c.seed, "dynaHeap": c.dynaHeap || c.permFiles, "permFiles": c.permFiles, "maintenancePhase": c.phase, "started": time.Now().UTC().Format(time.RFC3339), }, "", " ") if err := os.WriteFile(filepath.Join(c.dir, "run.json"), runJSON, 0o644); err != nil { fail("run.json", err) @@ -439,7 +635,9 @@ func main() { _ = csvw.Write([]string{"minute", "blocks", "over_budget", "block_p50_ms", "block_p90_ms", "block_max_ms", "seal_p50_ms", "seal_p90_ms", "seal_max_ms", "dyna_put_p99_us", "perm_put_p99_us", "read_p99_us", "compress_passes", "compress_s", "merge_passes", "merge_s", "pack_passes", "pack_s", "skipped", - "read_MBps", "write_MBps", "store_MB", "files", "perm_history", "dyna_history", "bloom_MB", "mismatches"}) + "read_MBps", "write_MBps", "store_MB", "files", "perm_history", "dyna_history", "bloom_MB", "mismatches", + "heap_syncs", "heap_fsync_ms_avg", "heap_sync_MB_avg", "heap_moved_MB", "heap_hole_MB", "heap_live_MB", + "heap_releases", "heap_release_ms", "heap_snapshots", "heap_snapshot_ms"}) csvw.Flush() fmt.Printf("bdbench: %d store(s) x %d shards, seal limit %d, window %d, maintenance every %d blocks, pack every %d; per block per store %d dyna + %d perm puts, %d reads; %s blocks for %s; %s\n", @@ -450,6 +648,7 @@ func main() { deadline := start.Add(c.duration) ioR0, ioW0 := procIO() period := start + var heap0 heapSplit report := func() { elapsed := time.Since(period) @@ -483,11 +682,20 @@ func main() { } minute := int(time.Since(start) / time.Minute) mism := t.mismatches.Load() - fmt.Printf("%3dm blocks %4d | block p50/p90/max %s/%s/%s ms | seal p50/p90/max %s/%s/%s ms | put p99 dyna %s perm %s us | read p99 %s us | maint compress %d (%.1fs) merge %d (%.1fs) pack %d (%.1fs) skipped %d | disk r %.0f w %.0f MB/s | store %.0f MB %d files | history perm %d dyna %d | bloom %.0f MB | mismatches %d%s\n", + heap1 := sumHeap(stores) + hs := heap1.minus(heap0) + heap0 = heap1 + heapNote := "" + if hs.syncs > 0 { + heapNote = fmt.Sprintf(" | heap fsync %.1f ms x %d (%.1f MB) moved %.0f MB hole %.0f MB live %.0f MB releases %d (%d ms) snapshots %d (%d ms)", + float64(hs.fsync)/1e6/float64(hs.syncs), hs.syncs, float64(hs.bytes)/1e6/float64(hs.syncs), float64(hs.moved)/1e6, float64(hs.holes)/1e6, float64(hs.live)/1e6, + hs.releases, hs.release.Milliseconds(), hs.snapshots, hs.snapshot.Milliseconds()) + } + fmt.Printf("%3dm blocks %4d | block p50/p90/max %s/%s/%s ms | seal p50/p90/max %s/%s/%s ms | put p99 dyna %s perm %s us | read p99 %s us | maint compress %d (%.1fs) merge %d (%.1fs) pack %d (%.1fs) skipped %d | disk r %.0f w %.0f MB/s | store %.0f MB %d files | history perm %d dyna %d | bloom %.0f MB | mismatches %d%s%s\n", minute, len(bt), ms(pct(bt, .5)), ms(pct(bt, .9)), ms(pct(bt, 1)), ms(pct(st, .5)), ms(pct(st, .9)), ms(pct(st, 1)), us(pct(dp, .99)), us(pct(pp, .99)), us(pct(rt, .99)), passes["compress"], spent["compress"].Seconds(), passes["merge"], spent["merge"].Seconds(), passes["pack"], spent["pack"].Seconds(), skipped, - rMB, wMB, float64(bytes)/1e6, files, permHist, dynaHist, float64(bloom)/1e6, mism, note) + rMB, wMB, float64(bytes)/1e6, files, permHist, dynaHist, float64(bloom)/1e6, mism, heapNote, note) _ = csvw.Write([]string{strconv.Itoa(minute), strconv.Itoa(len(bt)), strconv.FormatUint(over, 10), ms(pct(bt, .5)), ms(pct(bt, .9)), ms(pct(bt, 1)), ms(pct(st, .5)), ms(pct(st, .9)), ms(pct(st, 1)), us(pct(dp, .99)), us(pct(pp, .99)), us(pct(rt, .99)), @@ -497,10 +705,31 @@ func main() { strconv.FormatFloat(rMB, 'f', 1, 64), strconv.FormatFloat(wMB, 'f', 1, 64), strconv.FormatFloat(float64(bytes)/1e6, 'f', 0, 64), strconv.Itoa(files), strconv.Itoa(permHist), strconv.Itoa(dynaHist), strconv.FormatFloat(float64(bloom)/1e6, 'f', 1, 64), - strconv.FormatUint(mism, 10)}) + strconv.FormatUint(mism, 10), + strconv.FormatUint(hs.syncs, 10), strconv.FormatFloat(float64(hs.fsync)/1e6/float64(max(hs.syncs, 1)), 'f', 1, 64), + strconv.FormatFloat(float64(hs.bytes)/1e6/float64(max(hs.syncs, 1)), 'f', 1, 64), strconv.FormatFloat(float64(hs.moved)/1e6, 'f', 0, 64), + strconv.FormatFloat(float64(hs.holes)/1e6, 'f', 0, 64), strconv.FormatFloat(float64(hs.live)/1e6, 'f', 0, 64), + strconv.FormatUint(hs.releases, 10), strconv.FormatInt(hs.release.Milliseconds(), 10), + strconv.FormatUint(hs.snapshots, 10), strconv.FormatInt(hs.snapshot.Milliseconds(), 10)}) csvw.Flush() } + // The live state, every two seconds, beside the per-minute rows + liveStop := make(chan struct{}) + go func() { + tick := time.NewTicker(2 * time.Second) + defer tick.Stop() + for { + select { + case <-tick.C: + _ = os.WriteFile(filepath.Join(c.dir, "live.json"), t.liveState(c, stores, start), 0o644) + case <-liveStop: + return + } + } + }() + defer close(liveStop) + // Every store drives its own blocks; the first error stops the run. stop := make(chan struct{}) var stopOnce sync.Once @@ -551,6 +780,39 @@ wait: fail("close", err) } } + // Every store is reopened and read back: every sampled permanent + // key with its derived value, every checked hot key with its last + // value. What a seal made durable must be there after a close + // and an open, and a run that loses one fails. + var wrong int + for _, s := range stores { + re, err := blockchainDB.OpenKVShard(s.kv.Directory) + if err != nil { + fail(fmt.Sprintf("store %d reopen", s.id), err) + } + bad := 0 + for i, k := range s.permKeys { + if v, err := re.GetDeep(k); err != nil || !bytes.Equal(v, permValue(k, c.valueMin, c.valueMax)) { + if bad < 3 { + fmt.Printf(" sampled key %d of %d: err=%v got %d bytes want %d\n", i, len(s.permKeys), err, len(v), len(permValue(k, c.valueMin, c.valueMax))) + } + bad++ + } + } + for k, want := range s.last { + if v, err := re.GetDyna(k); err != nil || !bytes.Equal(v, want) { + bad++ + } + } + fmt.Printf("reopen store %d: %d permanent + %d dynamic keys read back, %d wrong\n", s.id, len(s.permKeys), len(s.last), bad) + wrong += bad + if err := re.Close(); err != nil { + fail("close after reopen", err) + } + } + if wrong > 0 { + fail("reopen", fmt.Errorf("%d keys lost or wrong after close and reopen", wrong)) + } if runErr != nil { fail("run", runErr) } diff --git a/cmd/bdbench/tools/README.md b/cmd/bdbench/tools/README.md new file mode 100644 index 0000000..3acbfef --- /dev/null +++ b/cmd/bdbench/tools/README.md @@ -0,0 +1,36 @@ +# bdbench tools + +Samplers that run beside a bdbench run and write one line every two +seconds, so that a minute's row can be taken apart afterwards. + +- `timeline.sh ` samples the run's `live.json` into + `.timeline`: the last ten seconds' seal p50/p90/max, the heap's + cumulative fsync average and bytes per sync, snapshots, releases, + moved bytes, hole and live bytes, perm merges and folds, and each + store's seal p90 (`ps`). +- `sys-timeline.sh ` samples the machine into + `.sys`: dirty and free memory, the NVMe device's write count, + write ticks and in-flight requests from `/proc/diskstats`, and the + load average. + +Both wait for `//live.json` to appear and stop when it +goes away (the run directory is removed). Start them detached before +the run: + + setsid nohup tools/timeline.sh $S heap-2a >/dev/null 2>&1 & + setsid nohup tools/sys-timeline.sh $S heap-2a >/dev/null 2>&1 & + +## Reading them + +Difference the cumulative fields between samples. The device's write +ticks per 2-second window is the queue depth times the latency: 1-3 s +on this disk when healthy, 100-500 s when the drive is stalling. + +A disk stall and a store tail look alike in the per-minute row (seal +p50 unchanged, p90 200-500 ms) and different here: a stall raises the +heap's fsync average for every store at once, the device's write ticks +with it, and the store's own counters (moved, snapshots, releases) show +no step. A store tail lines up with a step in one of those counters +and the device stays quiet. The disk this was learned on, and how it +was fixed, are in the operator's machine notes rather than here: on this +machine `~/infrastructure/disk-trim.md`. diff --git a/cmd/bdbench/tools/sys-timeline.sh b/cmd/bdbench/tools/sys-timeline.sh new file mode 100755 index 0000000..646951f --- /dev/null +++ b/cmd/bdbench/tools/sys-timeline.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# System counters every 2 s while a run's live.json exists: memory and the device +S=$1; run=$2 +until [ -f $S/$run/live.json ]; do sleep 2; done +while [ -f $S/$run/live.json ]; do + t=$(jq -r .elapsedSec $S/$run/live.json 2>/dev/null) + m=$(awk '/^(Dirty|Writeback|MemFree|MemAvailable|Cached):/{printf "%s=%d ", $1, $2/1024}' /proc/meminfo) + d=$(grep -w nvme0n1 /proc/diskstats | awk '{printf "wr=%d wr_ms=%d inflight=%d io_ms=%d", $8, $11, $12, $13}') + l=$(cut -d' ' -f1 /proc/loadavg) + echo "t=$t $m $d load=$l" >> $S/$run.sys + sleep 2 +done diff --git a/cmd/bdbench/tools/timeline.sh b/cmd/bdbench/tools/timeline.sh new file mode 100755 index 0000000..11ba052 --- /dev/null +++ b/cmd/bdbench/tools/timeline.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Samples a run's live.json every 2 s into .timeline (one JSON line each) +S=$1; run=$2 +until [ -f $S/$run/live.json ]; do sleep 2; done +while [ -f $S/$run/live.json ]; do + jq -c '{t:.elapsedSec, s50:.last10s.sealP50ms, s90:.last10s.sealP90ms, smax:.last10s.sealMaxMs, fs:.heapFsyncMsAvg, kb:.heapSyncKBAvg, syncs:.heapSyncs, snaps:.heapSnapshots, snapms:.heapSnapshotMs, rel:.heapReleases, relms:.heapReleaseMs, mv:.heapMovedMB, hole:.heapHoleMB, live:.heapLiveMB, merges:.permMerges, folds:.permFolds, inflight:.maintenanceInFlight, ps:((.last10s.storeSealP90ms // [])|map(floor))}' $S/$run/live.json 2>/dev/null >> $S/$run.timeline + sleep 2 +done diff --git a/database/heap.go b/database/heap.go new file mode 100644 index 0000000..166ac19 --- /dev/null +++ b/database/heap.go @@ -0,0 +1,1577 @@ +package blockchainDB + +import ( + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// HeapStore is the dynamic layer as files of entries and a map of keys +// (proposal docs/proposals/2026-09-16-entries-written-once.md): an +// entry is written once at the end of a file, a key is (file, offset, +// length), and space comes back by deleting a file once nothing in it +// is live -- never by rewriting an index or a filter. +// +// Files, in one directory: +// +// heap-N.dat entries, [len u32][height u64][key 32][value][crc32 +// of height+key+value], 8-byte aligned, appended in the +// order written. A block appends to the current data +// file; the mover appends to a file of its own, so the +// two never share a barrier. A file is rolled at +// HeapFileBytes and deleted once nothing in it is live. +// index-G.log generation G of the key map: a snapshot record of the +// whole map and the point in the data files from which +// the deltas after it are replayed. A snapshot starts a +// new generation in a file of its own, switched to by +// rename. +// +// A block's delta is appended to the block's data file right behind +// the block's entries, as an entry under a reserved key, so a block is +// ONE fsync per shard (entries and delta together) rather than two +// (measured: the second barrier was a third of the sync's cost, and +// the barrier count is what the device queue charges for). The delta +// names no key: a block's entries are contiguous in its file and each +// carries its key, so the delta is the RANGES the block wrote -- its +// own appends, and the mover's copies named this block -- and the few +// copies that arrived dead, a few dozen bytes a block (a delta of +// 44-byte records was a fifth of the heap's writes, all of it dead at +// the next snapshot and churned by the mover). Replay scans the +// ranges in order, later naming winning, and the last delta is +// trusted only if every entry in its ranges checks; an earlier delta +// was followed by a later block's fsync, which covers it. +// +// The key map is in memory for the live key set (spec 1.2: memory that +// scales with the working set). Open loads the newest whole +// generation's snapshot, replays the deltas in the data files after +// its replay point, and derives every file's live and dead bytes; a +// data file nothing names is deleted then, and one that is missing +// while named is an error. +// +// Durability (spec 1.8). The block sync fsyncs the data files the +// block wrote and then appends and fsyncs the block's delta, so an +// index entry is durable only after the entry it names is. A slot the +// last durable index names is never overwritten: a key rewritten in a +// later block takes a new slot, and its old slot is dead where it lies +// until its file is deleted -- and a file is deleted only after the +// delta naming the mover's copies out of it is durable, one sync late, +// the way spec 2.6 defers deletion (never unlink what a durable index +// names). A key rewritten again within the same block reuses the +// slot it took this block, since nothing durable names it yet. A +// crash therefore leaves every durable entry intact and every torn +// slot unnamed; the checksum catches a torn slot a stale index could +// name; and RepairHeapStore rebuilds the map from the data alone, +// since every entry carries its key and the height that wrote it. +type HeapStore struct { + Directory string + + mu sync.RWMutex + files map[uint32]*heapFile + cur *heapFile // The file the block appends to + mov *heapFile // The file the mover appends to + nextID uint32 + index map[[32]byte]slot + height uint64 // The block being written; slots taken in it may be rewritten in place + + // The block's delta in the making: the ranges the block appended + // (the mover's named copies first, then the block's own) and the + // copies that arrived dead; dirty the files written since the last + // sync; release the files the mover emptied, deleted once the + // delta naming their copies is durable. + ranges []heapRange + excluded []heapRange + dirty map[uint32]*heapFile + release []uint32 + // deltaAt is where the last delta ended: the replay point a + // snapshot records, and what a file is cut back to on open + deltaAt struct { + file uint32 + off int64 + } + + // syncMu serializes block syncs with each other and with a + // snapshot, so the map a snapshot writes is exactly the state of + // the last finished delta and no delta is in flight into a log + // about to be retired. + syncMu sync.Mutex + log *os.File + gen uint64 + snapAt uint64 // The block the key map was last snapshotted at + snapPhase uint64 // Blocks this shard's snapshot cadence is offset by + + closed bool + liveBytes int64 + deadBytes int64 + bound bool // The size bound engaged: the mover takes files below the ratio + + putTotal, putInPlace, putAppend atomic.Uint64 + lookups, hits atomic.Uint64 + cleanedBytes, movedBytes atomic.Uint64 + // The sync's cost, split: nanoseconds in the data files' fsyncs + // and in the delta's write and fsync, and the syncs and bytes + syncs, syncHeapNs, syncLogNs, syncBytes atomic.Uint64 + // The mover's cost on and off the block's path: files unlinked by + // a block's finish and the time in those unlinks; snapshots + // written and the time in them + releases, releaseNs, snapshotsN, snapshotNs atomic.Uint64 + unlink []uint32 // Released files not yet deleted +} + +// heapFile is one data file and its accounting. +type heapFile struct { + id uint32 + f *os.File + size int64 + live, dead int64 + deltas int64 // Bytes of delta entries not yet superseded by a snapshot: live until then + firstBlock uint64 // The block that first appended to it + cleaning bool // Taken by the pass in progress + inflight int // Copies reserved in it and not yet written: its size runs ahead of its bytes + releasing bool // Emptied by a pass; deleted by the next sync + // Bytes of ranges in it that the deltas of the current generation + // name, and where the last of them ends. A durable delta names + // the file, so the file stays until a snapshot supersedes the + // delta, whatever is live in it (spec 1.7): never unlink what a + // durable index names. + named, namedEnd int64 +} + +// nameRanges records that a delta of the current generation names +// these ranges. The caller holds the lock. +func (h *HeapStore) nameRanges(ranges []heapRange) { + for _, r := range ranges { + hf := h.files[r.file] + if hf == nil || r.to <= r.from { + continue + } + hf.named += int64(r.to - r.from) + if int64(r.to) > hf.namedEnd { + hf.namedEnd = int64(r.to) + } + } +} + +// heapRange is a stretch of one data file: what a delta names. +type heapRange struct { + file uint32 + from, to uint32 +} + +// slot is where an entry lives: its file, its offset there, the value +// length, and the block that took it, which decides whether a rewrite +// may reuse it in place. +type slot struct { + file uint32 + off uint32 + n uint32 + block uint64 +} + +const ( + heapHeader = 4 + 8 + 32 // len, height, key + heapTrailer = 4 // crc32 of height+key+value + heapAlign = 8 // Entries start on an 8-byte boundary + heapMagic = 0x48454150 // "HEAP", an index record's marker + heapSnapshot = 0x50414E53 // "SNAP", the record that starts a generation + heapIndexHdr = 4 + 8 + 4 + 4 + 4 // magic, height, count, replay file, replay offset + heapIndexRec = 32 + 4 + 4 + 4 +) + +// heapDeltaKey is the reserved key a delta entry is written under in +// a data file. A real key is a hash; the odds of one being all ones +// are those of a hash collision. +var heapDeltaKey = [32]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + +// HeapFileBytes is the size a data file is rolled at, and +// HeapFileBlocks the age: a shard that appends little per block (many +// shards, or a quiet one) would otherwise keep one file current for +// long stretches, and the current file is never the mover's, so its +// dead bytes could not be reclaimed (measured at 64 shards: 34 KB a +// block, no file rolled in three minutes, the store growing 2.3 GB a +// minute with the mover idle). +var HeapFileBytes int64 = 16 << 20 + +var HeapFileBlocks uint64 = 64 + +// HeapCleanBytes bounds one mover pass by the bytes it copies. The +// pass syncs its own copies, so the bound is not about a block's +// barrier but about the device queue the barrier shares: 16 MB +// passes put the seal's p90 at 150-250 ms in the minutes they ran +// (run 7), 4 MB releases more than the soak appends per shard per +// cadence (~4 MB, of which a quarter to a third is live). +var HeapCleanBytes int64 = 4 << 20 + +// HeapStoreCleanBytes is a store's mover budget per pass, shared out +// among its shards: a store sharded eight ways moves the same bytes a +// pass as one sharded once. Measured: with one shard and the +// per-shard budget alone the store grew to 6 GB in five minutes +// where eight shards held it under 4. +var HeapStoreCleanBytes int64 = 32 << 20 + +// HeapCleanFiles and HeapScanBytes bound a pass by the files it takes +// and the bytes it reads: with hot keys most of a file is dead and +// costs nothing to copy, so what limits the mover's pace is how much +// it looks at. At the soak's rate a shard makes ~50 MB of dead bytes +// per cadence; a pass that scans 128 MB releases more than that with +// room to spare (four 16 MB files did not, and the store floated with +// the size bound engaged). +var ( + HeapCleanFiles = 8 + HeapScanBytes int64 = 128 << 20 +) + +// HeapCleanRatio is the dead fraction a file must reach before the +// mover takes it -- unless the heap is over its size bound, when the +// deadest file is taken at HeapCleanFloor or more. The bound has +// hysteresis: it engages when dead bytes exceed live by +// HeapBoundOn and releases when they fall below live again, so the +// movers of nine stores in lockstep do not all engage on the same +// block and then all disengage (measured: p90 461-704 ms in the +// minutes the bound flipped, 70-90 ms otherwise). +var ( + HeapCleanRatio = 0.5 + HeapCleanFloor = 0.25 + HeapBoundOn = 1.5 +) + +// HeapSnapshotBlocks is how many blocks pass between key-map +// snapshots; between them the generation's deltas are what open +// replays. Counted in blocks, not maintenance calls, so the cadence +// of the caller does not set the cadence of the snapshots. +var HeapSnapshotBlocks uint64 = 100 + +// HeapSnapshotPinnedFiles is how many files' worth of bytes may be +// held only by their deltas before a snapshot is taken early, +// whatever the block count. +var HeapSnapshotPinnedFiles int64 = 2 + +// HeapCleanPeriod is the blocks over which a store spends +// HeapStoreCleanBytes: the store's mover rate is the one divided by +// the other, and a maintenance call moves what the blocks since the +// last call earned (KVShard.Compress). +var HeapCleanPeriod uint64 = 20 + +// entrySize is the bytes an entry of n value bytes takes, aligned. +func entrySize(n int) int64 { + return (int64(heapHeader+n+heapTrailer) + heapAlign - 1) &^ (heapAlign - 1) +} + +func dataName(id uint32) string { return fmt.Sprintf("heap-%06d.dat", id) } +func indexName(gen uint64) string { return fmt.Sprintf("index-%06d.log", gen) } + +// NewHeapStore creates an empty heap in directory, replacing anything +// there. +func NewHeapStore(directory string) (*HeapStore, error) { + os.RemoveAll(directory) + if err := os.MkdirAll(directory, 0o755); err != nil { + return nil, err + } + h := &HeapStore{Directory: directory} + return h, h.Open() +} + +// OpenHeapStore opens the heap in directory as it was left. A heap +// with data files but no index generation is refused: that is what +// RepairHeapStore is for, and it needs the committed height, which +// only the store above knows. +func OpenHeapStore(directory string) (*HeapStore, error) { + dataIDs, gens, err := listHeap(directory) + if err != nil { + return nil, fmt.Errorf("open heap at %s: %w", directory, err) + } + if len(dataIDs) > 0 && len(gens) == 0 { + return nil, fmt.Errorf("open heap at %s: %w", directory, ErrHeapNeedsRepair) + } + h := &HeapStore{Directory: directory} + return h, h.Open() +} + +// ErrHeapNeedsRepair says the heap's data is there and its index is +// not: RepairHeapStore rebuilds the index from the data. +var ErrHeapNeedsRepair = errors.New("heap has data but no index; repair it") + +// listHeap names the data files and the whole index generations in a +// directory, in order. A generation still being written (.tmp) is +// not whole and is removed. +func listHeap(directory string) (dataIDs []uint32, gens []uint64, err error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, nil, err + } + for _, e := range entries { + name := e.Name() + switch { + case strings.HasPrefix(name, "heap-") && strings.HasSuffix(name, ".dat"): + id, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(name, "heap-"), ".dat"), 10, 32) + if err == nil { + dataIDs = append(dataIDs, uint32(id)) + } + case strings.HasPrefix(name, "index-") && strings.HasSuffix(name, ".log"): + g, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(name, "index-"), ".log"), 10, 64) + if err == nil { + gens = append(gens, g) + } + case strings.HasSuffix(name, segTmpSuffix): + os.Remove(filepath.Join(directory, name)) + } + } + sort.Slice(dataIDs, func(i, j int) bool { return dataIDs[i] < dataIDs[j] }) + sort.Slice(gens, func(i, j int) bool { return gens[i] < gens[j] }) + return dataIDs, gens, nil +} + +// Open loads the key map from the newest generation, opens the data +// files and derives their accounting. Idempotent. +func (h *HeapStore) Open() (err error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.files != nil { + return nil + } + dataIDs, gens, err := listHeap(h.Directory) + if err != nil { + return err + } + h.index = map[[32]byte]slot{} + h.dirty = map[uint32]*heapFile{} + h.files = map[uint32]*heapFile{} + h.release, h.ranges, h.excluded = nil, nil, nil + h.closed = false + h.liveBytes, h.deadBytes = 0, 0 + // The newest whole generation is the index; older ones are what a + // snapshot left behind when it could not delete them + if len(gens) > 0 { + h.gen = gens[len(gens)-1] + for _, g := range gens[:len(gens)-1] { + os.Remove(filepath.Join(h.Directory, indexName(g))) + } + if err = h.replayGeneration(); err != nil { + return err + } + } else { + h.gen = 1 + // A fresh store: the first generation, written with the lock + // released since startGeneration takes it itself + h.mu.Unlock() + err = h.startGeneration() + h.mu.Lock() + if err != nil { + return err + } + } + for _, id := range dataIDs { + if id >= h.nextID { + h.nextID = id + 1 + } + } + if err = h.openFiles(dataIDs); err != nil { + return err + } + if err = h.replayDeltas(dataIDs); err != nil { + return err + } + return h.deriveFiles(dataIDs) +} + +// openFiles opens every data file present. The caller holds the lock. +func (h *HeapStore) openFiles(dataIDs []uint32) error { + for _, id := range dataIDs { + f, err := os.OpenFile(filepath.Join(h.Directory, dataName(id)), os.O_RDWR, 0o644) + if err != nil { + return err + } + st, err := f.Stat() + if err != nil { + f.Close() + return err + } + h.files[id] = &heapFile{id: id, f: f, size: st.Size()} + } + return nil +} + +// replayGeneration loads the current generation's snapshot; the +// deltas after it are in the data files and replayDeltas applies +// them. The caller holds the lock. +func (h *HeapStore) replayGeneration() (err error) { + path := filepath.Join(h.Directory, indexName(h.gen)) + if h.log, err = os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0o644); err != nil { + return err + } + buf, err := os.ReadFile(path) + if err != nil { + return err + } + n, err := h.applyRecord(buf) + if err != nil { + return fmt.Errorf("heap: generation %d: %w", h.gen, err) + } + if n != len(buf) { + return fmt.Errorf("heap: generation %d has %d trailing bytes", h.gen, len(buf)-n) + } + return nil +} + +// replayDeltas applies every delta entry in the data files after the +// snapshot's replay point, in file and offset order. The last delta +// is trusted only if every entry it names checks: one fsync covered +// it and its entries together, and a crash inside that fsync can +// leave the delta durable and an entry torn. An earlier delta was +// followed by a later block's fsync. The caller holds the lock; the +// files are open. +func (h *HeapStore) replayDeltas(dataIDs []uint32) error { + type found struct { + file uint32 + off int64 + end int64 + recs []byte + } + var deltas []found + for _, id := range dataIDs { + if id < h.deltaAt.file { + continue + } + hf := h.files[id] + if hf == nil { + continue + } + buf := make([]byte, hf.size) + if _, err := hf.f.ReadAt(buf, 0); err != nil && !errors.Is(err, io.EOF) { + return err + } + at := int64(0) + if id == h.deltaAt.file { + at = h.deltaAt.off + } + for at < int64(len(buf)) { + size, _, key, value, ok := decodeEntry(buf[at:]) + if size == 0 { + break + } + if ok && key == heapDeltaKey { + deltas = append(deltas, found{file: id, off: at, end: at + size, recs: append([]byte(nil), value...)}) + } + at += size // A damaged entry is stepped over; if it is named, Get reports it + } + } + for i, d := range deltas { + if !h.applyDelta(d.recs, i == len(deltas)-1) { + break // Torn, or the last block's entries do not all check: it did not commit + } + h.deltaAt.file, h.deltaAt.off = d.file, d.end + h.files[d.file].deltas += d.end - d.off + } + return nil +} + +// startGeneration begins an index generation: a snapshot of the map +// with the replay point -- where the last delta ended -- written +// aside and fsynced, renamed into place, the directory fsynced, and +// the previous generation's file removed once the new one is durable. +// The map and the point are taken together under the lock; a delta +// the seal appends after that lies past the point and is replayed +// over the snapshot on open, which is idempotent. The snapshot's +// write holds no lock (measured: a 3 MB snapshot under syncMu put +// the seal's p90 at 276 ms every hundredth block). A delta's bytes +// before the point are dead once the generation is durable. +func (h *HeapStore) startGeneration() error { + next := h.gen + if h.log != nil { + next = h.gen + 1 // A snapshot starts the generation after the current one + } + path := filepath.Join(h.Directory, indexName(next)) + tmp := path + segTmpSuffix + h.mu.Lock() + all := func(emit func(key [32]byte)) { + for key := range h.index { + emit(key) + } + } + snap := h.encodeIndexOf(heapSnapshot, all, len(h.index)) + superseded := map[uint32]int64{} + namedThen := map[uint32]int64{} + for id, hf := range h.files { + superseded[id] = hf.deltas + namedThen[id] = hf.named + } + h.mu.Unlock() + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + if _, err = f.Write(snap); err != nil { + f.Close() + return err + } + if err = fsync(f); err != nil { + f.Close() + return err + } + if err = f.Close(); err != nil { + return err + } + if err = os.Rename(tmp, path); err != nil { + return err + } + if err = fsyncDir(h.Directory); err != nil { + return err + } + h.mu.Lock() + defer h.mu.Unlock() + old := h.log + if h.log, err = os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0o644); err != nil { + return err + } + if old != nil { + old.Close() + os.Remove(filepath.Join(h.Directory, indexName(h.gen))) + } + h.gen = next + // The deltas the snapshot covers are dead where they lie + for id, n := range superseded { + if hf := h.files[id]; hf != nil && n > 0 { + hf.deltas -= n + hf.live -= n + hf.dead += n + h.liveBytes -= n + h.deadBytes += n + } + } + // The ranges those deltas named are no longer named by anything + // durable but the snapshot's own slots + for id, n := range namedThen { + if hf := h.files[id]; hf != nil { + hf.named -= n + if hf.named == 0 { + hf.namedEnd = 0 + } + } + } + return nil +} + +// fsyncDir makes a directory's entries durable: a rename or an unlink +// is not on disk until the directory is. +func fsyncDir(directory string) error { + d, err := os.Open(directory) + if err != nil { + return err + } + defer d.Close() + return fsync(d) +} + +// encodeDelta is the block's delta: marker, height, the ranges the +// block wrote (the mover's named copies first, then the block's own +// appends, so a later write of a key wins on replay) and the copies +// that arrived dead, checksummed. The caller holds the lock. +func (h *HeapStore) encodeDelta() []byte { + const rangeRec = 4 + 4 + 4 + buf := make([]byte, 4+8+4+4+len(h.ranges)*rangeRec+len(h.excluded)*rangeRec+4) + binary.LittleEndian.PutUint32(buf, heapMagic) + binary.LittleEndian.PutUint64(buf[4:], h.height) + binary.LittleEndian.PutUint32(buf[12:], uint32(len(h.ranges))) + binary.LittleEndian.PutUint32(buf[16:], uint32(len(h.excluded))) + at := 20 + for _, r := range append(append([]heapRange(nil), h.ranges...), h.excluded...) { + binary.LittleEndian.PutUint32(buf[at:], r.file) + binary.LittleEndian.PutUint32(buf[at+4:], r.from) + binary.LittleEndian.PutUint32(buf[at+8:], r.to) + at += rangeRec + } + binary.LittleEndian.PutUint32(buf[at:], crc32.ChecksumIEEE(buf[:at])) + return buf +} + +// decodeDelta reads a delta's ranges and exclusions. +func decodeDelta(buf []byte) (height uint64, ranges, excluded []heapRange, ok bool) { + const rangeRec = 4 + 4 + 4 + if len(buf) < 20 || binary.LittleEndian.Uint32(buf) != heapMagic { + return + } + nr, ne := int(binary.LittleEndian.Uint32(buf[12:])), int(binary.LittleEndian.Uint32(buf[16:])) + end := 20 + (nr+ne)*rangeRec + if len(buf) < end+4 || crc32.ChecksumIEEE(buf[:end]) != binary.LittleEndian.Uint32(buf[end:]) { + return + } + height = binary.LittleEndian.Uint64(buf[4:]) + at := 20 + for i := 0; i < nr+ne; i++ { + r := heapRange{file: binary.LittleEndian.Uint32(buf[at:]), from: binary.LittleEndian.Uint32(buf[at+4:]), to: binary.LittleEndian.Uint32(buf[at+8:])} + if i < nr { + ranges = append(ranges, r) + } else { + excluded = append(excluded, r) + } + at += rangeRec + } + return height, ranges, excluded, true +} + +// applyDelta names every entry in the delta's ranges, in order, later +// naming winning, except the excluded copies; verify makes it refuse +// a delta any of whose entries does not check. The caller holds the +// lock; the files are open. +func (h *HeapStore) applyDelta(recs []byte, verify bool) bool { + height, ranges, excluded, ok := decodeDelta(recs) + if !ok { + return false + } + skip := map[heapRange]bool{} + for _, e := range excluded { + skip[heapRange{file: e.file, from: e.from}] = true + } + type naming struct { + key [32]byte + s slot + } + var names []naming + for _, r := range ranges { + hf := h.files[r.file] + if hf == nil || int64(r.to) > hf.size { + return false + } + buf := make([]byte, r.to-r.from) + if _, err := hf.f.ReadAt(buf, int64(r.from)); err != nil { + return false + } + at := int64(0) + for at < int64(len(buf)) { + size, _, key, value, ok := decodeEntry(buf[at:]) + if size == 0 || !ok && verify { + return false // Torn; or the last block's entries do not all check + } + off := int64(r.from) + at + // A damaged entry of a committed block is named all the + // same, so a read of it reports the damage rather than + // answering that the key is absent + n := uint32(len(value)) + if !ok { + n = binary.LittleEndian.Uint32(buf[at:]) + } + if key != heapDeltaKey && !skip[heapRange{file: r.file, from: uint32(off)}] { + names = append(names, naming{key: key, s: slot{file: r.file, off: uint32(off), n: n}}) + } + at += size + } + } + for _, n := range names { + h.index[n.key] = n.s + } + h.nameRanges(ranges) + if height > h.height { + h.height = height + } + return true +} + +func (h *HeapStore) encodeIndexOf(magic uint32, each func(emit func(key [32]byte)), count int) []byte { + buf := make([]byte, heapIndexHdr+count*heapIndexRec+4) + binary.LittleEndian.PutUint32(buf, magic) + binary.LittleEndian.PutUint64(buf[4:], h.height) + binary.LittleEndian.PutUint32(buf[12:], uint32(count)) + binary.LittleEndian.PutUint32(buf[16:], h.deltaAt.file) + binary.LittleEndian.PutUint32(buf[20:], uint32(h.deltaAt.off)) + at := heapIndexHdr + each(func(key [32]byte) { + s := h.index[key] + copy(buf[at:], key[:]) + binary.LittleEndian.PutUint32(buf[at+32:], s.file) + binary.LittleEndian.PutUint32(buf[at+36:], s.off) + binary.LittleEndian.PutUint32(buf[at+40:], s.n) + at += heapIndexRec + }) + binary.LittleEndian.PutUint32(buf[at:], crc32.ChecksumIEEE(buf[:at])) + return buf +} + +var errHeapTorn = errors.New("heap: torn index record") + +// applyRecord applies one index record and returns its length. +func (h *HeapStore) applyRecord(buf []byte) (int, error) { + if len(buf) < heapIndexHdr { + return 0, errHeapTorn + } + magic := binary.LittleEndian.Uint32(buf) + if magic != heapSnapshot { + return 0, errHeapTorn + } + count := int(binary.LittleEndian.Uint32(buf[12:])) + end := heapIndexHdr + count*heapIndexRec + if len(buf) < end+4 || crc32.ChecksumIEEE(buf[:end]) != binary.LittleEndian.Uint32(buf[end:]) { + return 0, errHeapTorn + } + if height := binary.LittleEndian.Uint64(buf[4:]); height > h.height { + h.height = height + } + if magic == heapSnapshot { + h.deltaAt.file = binary.LittleEndian.Uint32(buf[16:]) + h.deltaAt.off = int64(binary.LittleEndian.Uint32(buf[20:])) + } + for at := heapIndexHdr; at < end; at += heapIndexRec { + var key [32]byte + copy(key[:], buf[at:]) + h.index[key] = slot{file: binary.LittleEndian.Uint32(buf[at+32:]), off: binary.LittleEndian.Uint32(buf[at+36:]), n: binary.LittleEndian.Uint32(buf[at+40:])} + } + return end + 4, nil +} + +// deriveFiles derives every open file's live and dead bytes, cuts +// the newest back to its last named slot or its last delta, and +// deletes any file the map does not name and no delta needed: +// nothing durable names it, and its bytes are a crash's or the +// mover's leftovers. The caller holds the lock. +func (h *HeapStore) deriveFiles(dataIDs []uint32) error { + named := map[uint32]int64{} // file -> end of its last named slot + for _, s := range h.index { + if e := int64(s.off) + entrySize(int(s.n)); e > named[s.file] { + named[s.file] = e + } + h.liveBytes += entrySize(int(s.n)) + } + for id := range named { + if h.files[id] == nil { + return fmt.Errorf("heap: the index names %s: missing", dataName(id)) + } + } + for _, id := range dataIDs { + hf := h.files[id] + end, live := named[id] + if id == h.deltaAt.file && h.deltaAt.off > end { + end, live = h.deltaAt.off, true + } + if hf.namedEnd > end { + end, live = hf.namedEnd, true // A delta of this generation names it + } + if !live && hf.deltas == 0 { + hf.f.Close() + delete(h.files, id) + os.Remove(filepath.Join(h.Directory, dataName(id))) + continue + } + if err := hf.f.Truncate(end); err != nil { + return err + } + hf.size = end + } + for _, s := range h.index { + h.files[s.file].live += entrySize(int(s.n)) + } + for _, hf := range h.files { + hf.live += hf.deltas + h.liveBytes += hf.deltas + hf.dead = hf.size - hf.live + h.deadBytes += hf.dead + } + return nil +} + +// Close syncs what is pending and closes the files. Reopen with Open. +func (h *HeapStore) Close() error { + p, err := h.beginBlockSync() + if err != nil { + if errors.Is(err, errStoreClosed) { + return nil + } + return err + } + if err := p.finish(); err != nil { + return err + } + if err := h.unlinkReleased(); err != nil { + return err + } + h.mu.Lock() + defer h.mu.Unlock() + h.closed = true + for _, hf := range h.files { + if cerr := hf.f.Close(); err == nil { + err = cerr + } + } + if cerr := h.log.Close(); err == nil { + err = cerr + } + h.files, h.cur, h.mov, h.log = nil, nil, nil, nil + return err +} + +// encodeEntry lays out one entry: the block that wrote it is in the +// header so that, with the index gone, a scan can tell the current +// copy of a key (the highest height) from stale ones and a committed +// entry from one whose block never synced. +func encodeEntry(height uint64, key [32]byte, value []byte) []byte { + buf := make([]byte, entrySize(len(value))) + binary.LittleEndian.PutUint32(buf, uint32(len(value))) + binary.LittleEndian.PutUint64(buf[4:], height) + copy(buf[12:], key[:]) + copy(buf[heapHeader:], value) + binary.LittleEndian.PutUint32(buf[heapHeader+len(value):], crc32.ChecksumIEEE(buf[4:heapHeader+len(value)])) + return buf +} + +// decodeEntry checks the entry at the start of buf and returns its +// fields and its aligned size; ok is false for an unwritten, torn or +// damaged slot. +func decodeEntry(buf []byte) (size int64, height uint64, key [32]byte, value []byte, ok bool) { + if len(buf) < heapHeader { + return + } + n := int(binary.LittleEndian.Uint32(buf)) + size = entrySize(n) + if n == 0 && binary.LittleEndian.Uint64(buf[4:]) == 0 || size > int64(len(buf)) { + return 0, 0, key, nil, false + } + copy(key[:], buf[12:]) + if crc32.ChecksumIEEE(buf[4:heapHeader+n]) != binary.LittleEndian.Uint32(buf[heapHeader+n:]) { + return size, 0, key, nil, false // Damaged: its size lets a scan step over it, its key lets replay name it + } + height = binary.LittleEndian.Uint64(buf[4:]) + copy(key[:], buf[12:]) + return size, height, key, buf[heapHeader : heapHeader+n], true +} + +// newFile opens the next data file. The caller holds the lock. +func (h *HeapStore) newFile() (*heapFile, error) { + id := h.nextID + h.nextID++ + f, err := os.OpenFile(filepath.Join(h.Directory, dataName(id)), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return nil, err + } + hf := &heapFile{id: id, f: f} + h.files[id] = hf + return hf, nil +} + +// reserve takes size bytes at the end of the block's or the mover's +// file, rolling to a new file at HeapFileBytes. The caller holds the +// lock. +func (h *HeapStore) reserve(mover bool, size int64) (hf *heapFile, off int64, err error) { + at := &h.cur + if mover { + at = &h.mov + } + if *at == nil || (*at).size+size > HeapFileBytes || (!mover && h.height >= (*at).firstBlock+HeapFileBlocks) { + if *at, err = h.newFile(); err != nil { + return nil, 0, err + } + (*at).firstBlock = h.height + } + hf = *at + off = hf.size + hf.size += size + hf.live += size + h.liveBytes += size + if mover { + // The pass syncs its own file; the block's sync must never + // capture it as dirty, or the block waits for the copies + hf.inflight++ + } else { + h.dirty[hf.id] = hf + h.extendRange(hf.id, off, off+size) + } + return hf, off, nil +} + +// extendRange adds [from, to) of a file to the block's delta, +// extending the last range when it abuts. The caller holds the lock. +func (h *HeapStore) extendRange(file uint32, from, to int64) { + if n := len(h.ranges); n > 0 && h.ranges[n-1].file == file && int64(h.ranges[n-1].to) == from { + h.ranges[n-1].to = uint32(to) + return + } + h.ranges = append(h.ranges, heapRange{file: file, from: uint32(from), to: uint32(to)}) +} + +// kill accounts a slot its key stopped naming. The caller holds the +// lock. +func (h *HeapStore) kill(s slot) { + size := entrySize(int(s.n)) + h.liveBytes -= size + h.deadBytes += size + hf := h.files[s.file] + hf.live -= size + hf.dead += size +} + +// Put writes value under key: in place if the key took its slot this +// block and the entry keeps its size, else appended to the block's +// file. The slot a durable index names is never rewritten. +func (h *HeapStore) Put(key [32]byte, value []byte) error { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return errStoreClosed + } + h.putTotal.Add(1) + old, had := h.index[key] + var s slot + var hf *heapFile + if had && old.block == h.height && entrySize(len(value)) == entrySize(int(old.n)) { + // Taken this block, nothing durable names it, and the same + // aligned size, so the file stays a contiguous sequence of + // entries a scan can walk: rewrite in place + s = old + s.n = uint32(len(value)) + hf = h.files[s.file] + h.dirty[hf.id] = hf + h.putInPlace.Add(1) + } else { + var off int64 + var err error + if hf, off, err = h.reserve(false, entrySize(len(value))); err != nil { + return err + } + s = slot{file: hf.id, off: uint32(off), n: uint32(len(value)), block: h.height} + if had { + h.kill(old) // Dead where it lies + } + h.putAppend.Add(1) + } + if _, err := hf.f.WriteAt(encodeEntry(h.height, key, value), int64(s.off)); err != nil { + return err + } + h.index[key] = s + return nil +} + +// Get answers from the key map and one read of the slot, taken +// outside the lock. A slot whose checksum fails is reported as +// corrupt, never as a value. A file deleted between the lookup and +// the read -- the mover moved the entry and a sync released the file +// meanwhile -- is looked up again. +func (h *HeapStore) Get(key [32]byte) ([]byte, error) { + for attempt := 0; ; attempt++ { + h.mu.RLock() + if h.closed { + h.mu.RUnlock() + return nil, errStoreClosed + } + h.lookups.Add(1) + s, ok := h.index[key] + var f *os.File + if ok { + f = h.files[s.file].f + } + h.mu.RUnlock() + if !ok { + return nil, errNotFound + } + h.hits.Add(1) + buf := make([]byte, heapHeader+int(s.n)+heapTrailer) + if _, err := f.ReadAt(buf, int64(s.off)); err != nil { + if attempt == 0 && errors.Is(err, os.ErrClosed) { + continue + } + return nil, err + } + return heapEntryValue(buf, key) + } +} + +// heapEntryValue checks an entry read from a slot and returns its +// value. +func heapEntryValue(buf []byte, key [32]byte) ([]byte, error) { + n := int(binary.LittleEndian.Uint32(buf)) + if len(buf) != heapHeader+n+heapTrailer || [32]byte(buf[12:heapHeader]) != key { + return nil, fmt.Errorf("heap: slot does not hold the key it is named for") + } + if crc32.ChecksumIEEE(buf[4:heapHeader+n]) != binary.LittleEndian.Uint32(buf[heapHeader+n:]) { + return nil, fmt.Errorf("heap: entry checksum failed") + } + return append([]byte(nil), buf[heapHeader:heapHeader+n]...), nil +} + +// GetDeep is Get: a heap has no history to reach into. +func (h *HeapStore) GetDeep(key [32]byte) ([]byte, error) { return h.Get(key) } + +// AdvanceBlock sets the block new writes belong to. +func (h *HeapStore) AdvanceBlock(height uint64) { + h.mu.Lock() + h.height = height + h.mu.Unlock() +} + +// heapSync is a block sync in flight: the files to make durable, the +// delta, and the files to delete once it is. It holds syncMu until +// finished. +type heapSync struct { + h *HeapStore + dirty []*heapFile + delta []byte + release []uint32 + bytes int64 +} + +// beginBlockSync takes the block's delta under the lock; finish makes +// it durable outside it. +func (h *HeapStore) beginBlockSync() (blockSync, error) { + h.syncMu.Lock() + h.mu.Lock() + defer h.mu.Unlock() + if h.closed || h.files == nil { + h.syncMu.Unlock() + return nil, errStoreClosed + } + p := &heapSync{h: h, release: h.release} + h.release = nil + if len(h.ranges) > 0 || len(p.release) > 0 { + // The delta goes into the block's data file behind its entries, + // under the reserved key, so the block's one fsync covers both + p.delta = h.encodeDelta() + h.nameRanges(h.ranges) + h.ranges, h.excluded = nil, nil + hf, off, err := h.reserve(false, entrySize(len(p.delta))) + h.ranges = nil // The delta entry itself is not part of the block's range + if err != nil { + h.syncMu.Unlock() + return nil, err + } + if _, err := hf.f.WriteAt(encodeEntry(h.height, heapDeltaKey, p.delta), off); err != nil { + h.syncMu.Unlock() + return nil, err + } + hf.deltas += entrySize(len(p.delta)) + h.deltaAt.file, h.deltaAt.off = hf.id, off+entrySize(len(p.delta)) + } + for _, hf := range h.dirty { + p.dirty = append(p.dirty, hf) + p.bytes += hf.size + } + h.dirty = map[uint32]*heapFile{} + return p, nil +} + +// finish: entries durable, then the delta durable, then the files the +// delta's copies emptied are deleted. Releases syncMu. +func (p *heapSync) finish() (err error) { + h := p.h + defer h.syncMu.Unlock() + if p.delta == nil { + return nil + } + t := time.Now() + for _, hf := range p.dirty { + if err = fsync(hf.f); err != nil { + return err + } + } + h.syncHeapNs.Add(uint64(time.Since(t))) + h.syncs.Add(1) + h.syncBytes.Add(uint64(p.bytes)) + h.mu.Lock() + defer h.mu.Unlock() + // The files the delta no longer names leave the map now; their + // unlinks are the mover's, off the block's path. Until then they + // are unnamed files on disk, which an open deletes as such. + for _, id := range p.release { + hf := h.files[id] + hf.f.Close() + delete(h.files, id) + h.deadBytes -= hf.dead + h.unlink = append(h.unlink, id) + } + return nil +} + +// SetSnapshotPhase offsets this shard's snapshot cadence by blocks, so +// that a store's shards do not all snapshot on the same block. +func (h *HeapStore) SetSnapshotPhase(blocks uint64) { + h.mu.Lock() + h.snapPhase = blocks + h.mu.Unlock() +} + +// unlinkReleased deletes the files the block syncs have released. +// Called without the lock. +func (h *HeapStore) unlinkReleased() error { + h.mu.Lock() + ids := h.unlink + h.unlink = nil + h.mu.Unlock() + if len(ids) == 0 { + return nil + } + t := time.Now() + for _, id := range ids { + if err := os.Remove(filepath.Join(h.Directory, dataName(id))); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + h.releases.Add(uint64(len(ids))) + h.releaseNs.Add(uint64(time.Since(t))) + return nil +} + +// compact is one slice of the heap's maintenance: the released +// files unlinked, a mover pass bounded by budget bytes copied, and +// every HeapSnapshotBlocks a new index generation, which bounds the +// replay on open. +func (h *HeapStore) compact(budget int64) (bool, error) { + if err := h.unlinkReleased(); err != nil { + return false, err + } + moved, err := h.clean(budget) + if err != nil { + return moved, err + } + // A snapshot is due by age, or when the files it would free + // outweigh it: a file whose only live bytes are deltas cannot go + // until a snapshot supersedes them, so the space those files hold + // is what a snapshot buys, and it is taken when that exceeds + // HeapSnapshotPinned + h.mu.Lock() + var pinned int64 + for _, hf := range h.files { + if hf != h.cur && hf != h.mov && hf.live == hf.deltas && (hf.deltas > 0 || hf.named > 0) { + pinned += hf.size // Nothing live in it but what the generation's deltas need + } + } + due := h.height+h.snapPhase-h.snapAt >= HeapSnapshotBlocks || pinned >= HeapSnapshotPinnedFiles*HeapFileBytes + if due { + h.snapAt = h.height + } + h.mu.Unlock() + if due { + t := time.Now() + err = h.Snapshot() + h.snapshotsN.Add(1) + h.snapshotNs.Add(uint64(time.Since(t))) + } + return moved, err +} + +// Snapshot starts a new index generation: the whole map, then the +// deltas to come. Serialized with block syncs, so the map is exactly +// the state of the last finished delta and no delta lands in the +// generation being retired. Off the protocol path. +func (h *HeapStore) Snapshot() error { + h.mu.RLock() + closed := h.closed + h.mu.RUnlock() + if closed { + return errStoreClosed + } + return h.startGeneration() +} + +// moverHook, when set, runs between the mover's copy and its naming: +// the window in which a put can make a copy dead on arrival. +var moverHook func() + +// heapMove is one live entry the mover copies: where it was, where +// it goes, and the bytes to write there. +type heapMove struct { + key [32]byte + from, to slot + hf *heapFile + height uint64 // The block the copy is written as + value []byte + entry []byte +} + +// heapPlanChunk is how many entries a pass plans per lock hold. +const heapPlanChunk = 4096 + +// clean is the mover: it takes the files with the most dead bytes -- +// once HeapCleanRatio of each is dead, or the deadest whatever its +// ratio while dead bytes exceed live -- copies their live entries to +// the mover's file, at most budget bytes per pass, and marks a file +// left with nothing live for deletion once the delta naming the +// copies is durable. The pass syncs its own copies, so they never +// land in a block's barrier, and it holds the shard's lock only to +// pick, to plan, and to name: never across a read, a write or an +// fsync (spec 1.6). A byte released never costs more than a byte +// copied unless the size bound forces it (spec 1.2). +func (h *HeapStore) clean(budget int64) (bool, error) { + // 1. Under the lock: pick the files + h.mu.Lock() + if h.closed { + h.mu.Unlock() + return false, errStoreClosed + } + if len(h.release) > 0 { + h.mu.Unlock() + return false, nil // The last pass's deletions are still waiting on a sync + } + // A file with nothing live in it needs no scan and no copy: it is + // released outright, any number of them a pass, one sync late as + // every release is. Without this the pass's file count paced the + // mover at about the rate dead bytes appeared, and the store + // floated with the size bound engaged. + emptied := 0 + for _, hf := range h.files { + if hf.live == 0 && hf.named == 0 && hf != h.cur && hf != h.mov && !hf.cleaning && hf.inflight == 0 && !hf.releasing && hf.size > 0 { + hf.releasing = true + h.release = append(h.release, hf.id) + emptied++ + } + } + var taken []*heapFile + var scan int64 + for len(taken) < HeapCleanFiles && scan < HeapScanBytes { + hf := h.pickFile() + if hf == nil { + break + } + hf.cleaning = true + taken = append(taken, hf) + scan += hf.size + } + h.mu.Unlock() + if len(taken) == 0 { + return emptied > 0, nil + } + // 2. Under the lock, briefly: the picked files' live slots, from + // the index -- one pass over the map, not a decode of the files. + // Scanning a file to find its live entries read and checksummed + // up to 128 MB a shard per pass, and nine stores' passes together + // starved the block loops for CPU (load 27 on 24 cores, seal p50 + // 200 ms for the minute). Then without the lock: read those + // entries, and only those + type liveSlot struct { + key [32]byte + s slot + } + h.mu.Lock() + picked := map[uint32]int{} + for i, hf := range taken { + picked[hf.id] = i + } + slots := make([][]liveSlot, len(taken)) + for key, s := range h.index { + if i, ok := picked[s.file]; ok { + slots[i] = append(slots[i], liveSlot{key: key, s: s}) + } + } + h.mu.Unlock() + entries := make([][]heapEntry, len(taken)) + for i, hf := range taken { + sort.Slice(slots[i], func(a, b int) bool { return slots[i][a].s.off < slots[i][b].s.off }) + for _, ls := range slots[i] { + buf := make([]byte, entrySize(int(ls.s.n))) + if _, err := hf.f.ReadAt(buf, int64(ls.s.off)); err != nil { + h.mu.Lock() + for _, hf := range taken { + hf.cleaning = false + } + h.mu.Unlock() + return false, err + } + size, _, key, value, ok := decodeEntry(buf) + if !ok || key != ls.key { + continue // Damaged or stale: left where it is, and a read of it reports the damage + } + entries[i] = append(entries[i], heapEntry{off: ls.s.off, size: size, key: key, value: value}) + } + h.cleanedBytes.Add(uint64(hf.size)) + } + // 3. Under the lock, a chunk of entries at a time: confirm each is + // still named and reserve its copy's slot in the mover's file + var moves []heapMove + var copied int64 + for i, hf := range taken { + for at := 0; at < len(entries[i]) && copied < budget; at += heapPlanChunk { + end := at + heapPlanChunk + if end > len(entries[i]) { + end = len(entries[i]) + } + h.mu.Lock() + m, n, err := h.planFile(hf, entries[i][at:end], budget-copied) + h.mu.Unlock() + if err != nil { + h.mu.Lock() + for _, hf := range taken { + hf.cleaning = false + } + h.mu.Unlock() + return false, err + } + moves = append(moves, m...) + copied += n + } + } + h.mu.Lock() + for _, hf := range taken { + hf.cleaning = false + } + h.mu.Unlock() + var movFiles []*heapFile + for _, m := range moves { + if len(movFiles) == 0 || movFiles[len(movFiles)-1] != m.hf { + movFiles = append(movFiles, m.hf) + } + } + // The copies' bytes are laid out outside the lock + for i := range moves { + m := &moves[i] + m.entry = encodeEntry(m.height, m.key, m.value) + } + // 4. Without the lock: write the copies and make them durable + fail := func(err error) (bool, error) { + h.mu.Lock() + for _, m := range moves { + m.hf.inflight-- // Reserved, never written: dead, and read no further + h.kill(m.to) + } + h.mu.Unlock() + return false, err + } + for _, m := range moves { + if _, err := m.hf.f.WriteAt(m.entry, int64(m.to.off)); err != nil { + return fail(err) + } + } + for _, hf := range movFiles { + if err := fsync(hf.f); err != nil { + return fail(err) + } + } + if moverHook != nil { + moverHook() // Tests: a put between the copy and its naming + } + // 5. Under the lock: name the copies, unless the key was rewritten + // meanwhile, in which case the copy is dead on arrival; and mark + // the files the pass emptied + h.mu.Lock() + defer h.mu.Unlock() + // The copies were reserved in order, so they form ranges of the + // mover's files; a copy that arrived dead is excluded from them. + // The mover's ranges go before the block's own in the delta, so + // that a put of the same key later in the block wins on replay + var mine []heapRange + for _, m := range moves { + m.hf.inflight-- + size := entrySize(int(m.to.n)) + if n := len(mine); n > 0 && mine[n-1].file == m.to.file && int64(mine[n-1].to) == int64(m.to.off) { + mine[n-1].to = uint32(int64(m.to.off) + size) + } else { + mine = append(mine, heapRange{file: m.to.file, from: m.to.off, to: uint32(int64(m.to.off) + size)}) + } + s, live := h.index[m.key] + if live && s == m.from { + h.index[m.key] = m.to + h.kill(m.from) + } else { + h.kill(m.to) // Reserved and written, but no longer wanted + h.excluded = append(h.excluded, heapRange{file: m.to.file, from: m.to.off}) + } + } + h.ranges = append(mine, h.ranges...) + for _, hf := range taken { + if hf.live == 0 && hf.named == 0 && hf != h.cur && hf != h.mov { + hf.releasing = true + h.release = append(h.release, hf.id) + } + } + h.movedBytes.Add(uint64(copied)) + return true, nil +} + +// pickFile is the file the mover takes next: the deadest by fraction +// among those not open for append, if it is dead enough or the heap +// is over its size bound. The caller holds the lock. +func (h *HeapStore) pickFile() *heapFile { + var pick *heapFile + best := 0.0 + for _, hf := range h.files { + if hf == h.cur || hf == h.mov || hf.cleaning || hf.inflight > 0 || hf.releasing || hf.dead == 0 { + continue + } + if f := float64(hf.dead) / float64(hf.size); f > best { + pick, best = hf, f + } + } + switch { + case !h.bound && float64(h.deadBytes) > HeapBoundOn*float64(h.liveBytes): + h.bound = true + case h.bound && h.deadBytes < h.liveBytes: + h.bound = false + } + if pick == nil || best < HeapCleanRatio && (!h.bound || best < HeapCleanFloor) { + return nil + } + return pick +} + +// heapEntry is one entry of a picked file as decoded outside the +// lock: where it is, and its bytes ready to be copied. +type heapEntry struct { + off uint32 + size int64 + key [32]byte + value []byte +} + +// planFile reserves, in the mover's file, a slot for each entry of a +// picked file the index still names, up to budget bytes; the rest +// wait for the next pass. The caller holds the lock; the entries +// were decoded outside it. +func (h *HeapStore) planFile(hf *heapFile, entries []heapEntry, budget int64) (moves []heapMove, copied int64, err error) { + for _, e := range entries { + s, live := h.index[e.key] + if !live || s.file != hf.id || s.off != e.off { + continue + } + if copied >= budget { + break // The rest next pass + } + to, off, err := h.reserve(true, e.size) + if err != nil { + return nil, 0, err + } + ns := slot{file: to.id, off: uint32(off), n: uint32(len(e.value)), block: h.height} + // The copy is this block's write of the key: it carries this + // height, so a repair scan prefers it to the original + moves = append(moves, heapMove{key: e.key, from: s, to: ns, hf: to, height: h.height, value: e.value}) + copied += e.size + } + return moves, copied, nil +} + +// RepairHeapStore rebuilds the key map by reading the keys from the +// data: every entry carries its key, its height and a checksum, so a +// scan of the data files in order recovers the map without any index. +// For each key the copy with the highest height wins, later in the +// scan on a tie; an entry above the committed height -- a block whose +// sync never finished -- is dropped, as is any torn or damaged slot. +// The rebuilt map starts a new generation, so the next open is an +// ordinary one. +func RepairHeapStore(directory string, committed uint64) (*HeapStore, error) { + dataIDs, gens, err := listHeap(directory) + if err != nil { + return nil, err + } + for _, g := range gens { + os.Remove(filepath.Join(directory, indexName(g))) + } + h := &HeapStore{Directory: directory, index: map[[32]byte]slot{}, dirty: map[uint32]*heapFile{}, files: map[uint32]*heapFile{}, height: committed} + heights := map[[32]byte]uint64{} + for _, id := range dataIDs { + data, err := os.ReadFile(filepath.Join(directory, dataName(id))) + if err != nil { + return nil, err + } + var off int64 + for off < int64(len(data)) { + size, height, key, value, ok := decodeEntry(data[off:]) + if size == 0 { + break + } + if !ok || key == heapDeltaKey { + off += size + continue + } + if height <= committed { + if prev, seen := heights[key]; !seen || height >= prev { + heights[key] = height + h.index[key] = slot{file: id, off: uint32(off), n: uint32(len(value))} + } + } + off += size + } + if id >= h.nextID { + h.nextID = id + 1 + } + } + // A repair trusts the data alone: the snapshot's replay point is + // the end of the newest file, so no old delta is replayed over it + if n := len(dataIDs); n > 0 { + if st, err := os.Stat(filepath.Join(directory, dataName(dataIDs[n-1]))); err == nil { + h.deltaAt.file, h.deltaAt.off = dataIDs[n-1], st.Size() + } + } + h.gen = 1 + if err = h.startGeneration(); err != nil { + return nil, err + } + if err = h.openFiles(dataIDs); err != nil { + return nil, err + } + if err = h.deriveFiles(dataIDs); err != nil { + return nil, err + } + return h, nil +} + +// Stats is the heap's report in the store's terms: puts and lookups +// as the segment layer counts them (every hit is answered from the +// key map, so they are all LiveHit), and the heap's own figures. +func (h *HeapStore) Stats() StoreStats { + h.mu.RLock() + defer h.mu.RUnlock() + return StoreStats{ + PutTotal: h.putTotal.Load(), + PutNew: h.putAppend.Load(), + LookupTotal: h.lookups.Load(), + LiveHit: h.hits.Load(), + ResidentIndexBytes: uint64(len(h.index)) * (32 + 24), + HeapFiles: len(h.files), + HeapLiveBytes: uint64(h.liveBytes), + HeapDeadBytes: uint64(h.deadBytes), + HeapScannedBytes: h.cleanedBytes.Load(), + HeapMovedBytes: h.movedBytes.Load(), + } +} + +// HoleRatio reports the dead bytes in the files against the live +// bytes: what the mover has yet to reclaim. Live includes the delta +// entries a snapshot has not yet superseded. +func (h *HeapStore) HoleRatio() (dead, live int64) { + h.mu.RLock() + defer h.mu.RUnlock() + return h.deadBytes, h.liveBytes +} + +// deltaBytes is the bytes of delta entries not yet superseded. +func (h *HeapStore) deltaBytes() (n int64) { + h.mu.RLock() + defer h.mu.RUnlock() + for _, hf := range h.files { + n += hf.deltas + } + return n +} + +// SyncCost reports the block syncs so far: how many, the bytes their +// data fsyncs covered, and the time spent in the data files' fsyncs +// and in the delta's write and fsync. +func (h *HeapStore) SyncCost() (syncs, bytes uint64, heapFsync, delta time.Duration) { + return h.syncs.Load(), h.syncBytes.Load(), time.Duration(h.syncHeapNs.Load()), time.Duration(h.syncLogNs.Load()) +} + +// Cleaned reports what the mover has scanned and what it had to +// copy: the ratio is the heap's write amplification. +func (h *HeapStore) Cleaned() (scanned, moved uint64) { + return h.cleanedBytes.Load(), h.movedBytes.Load() +} + +// MoverCost reports the files a block's finish has unlinked and the +// time in those unlinks, and the snapshots written and their time. +func (h *HeapStore) MoverCost() (releases uint64, release time.Duration, snapshots uint64, snapshot time.Duration) { + return h.releases.Load(), time.Duration(h.releaseNs.Load()), h.snapshotsN.Load(), time.Duration(h.snapshotNs.Load()) +} + +// SetFilterBlocks and SetSealLimit are the segment layer's knobs; a +// heap has neither a window nor a tail. +func (h *HeapStore) SetFilterBlocks(uint64) error { return nil } +func (h *HeapStore) SetSealLimit(uint64) error { return nil } + +// LiveRecords is the live key count; a heap is never sealed on it. +func (h *HeapStore) LiveRecords() uint64 { + h.mu.RLock() + defer h.mu.RUnlock() + return uint64(len(h.index)) +} diff --git a/database/heap_crash_test.go b/database/heap_crash_test.go new file mode 100644 index 0000000..1a469bb --- /dev/null +++ b/database/heap_crash_test.go @@ -0,0 +1,123 @@ +package blockchainDB + +import ( + "bufio" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const heapCrashDirEnv = "BDB_HEAP_CRASH_DIR" + +// The child: blocks of the same keys, every block rewriting every key +// with a value that says which block, a sync per block and a mover +// pass every few, and a line on stdout at each durability point. +// Killed by the parent at a moment of its choosing. +func TestHeapCrashChild(t *testing.T) { + dir := os.Getenv(heapCrashDirEnv) + if dir == "" { + t.Skip("helper process for TestHeapCrashRecovery") + } + HeapFileBytes = 64 << 10 + h, err := NewHeapStore(dir) + require.NoError(t, err, "child: new") + for b := uint64(1); ; b++ { + h.AdvanceBlock(b) + for i := 0; i < 300; i++ { + require.NoError(t, h.Put(crashKey(i), crashValue(b, i)), "child: put") + } + p, err := h.beginBlockSync() + require.NoError(t, err, "child: sync") + require.NoError(t, p.finish(), "child: finish") + fmt.Printf("CHECKPOINT %d\n", b) + if b%4 == 0 { + _, err := h.compact(HeapCleanBytes) + require.NoError(t, err, "child: compact") + } + } +} + +func crashKey(i int) (k [32]byte) { + k[0], k[1] = byte(i>>8), byte(i) + return +} + +func crashValue(b uint64, i int) []byte { + v := make([]byte, 40+i%60) + v[0], v[1], v[2] = byte(b>>16), byte(b>>8), byte(b) + v[3] = byte(i) + return v +} + +// The parent: run the child, let it reach some durability points, +// kill it dead, and check that the heap opens to exactly the last +// durable block -- every key at that block's value, nothing torn, +// nothing from the block in flight -- and that a repair from the data +// alone agrees. +func TestHeapCrashRecovery(t *testing.T) { + if testing.Short() { + t.Skip("spawns processes") + } + for round := 0; round < 3; round++ { + dir := filepath.Join(t.TempDir(), fmt.Sprintf("crash-%d", round)) + cmd := exec.Command(os.Args[0], "-test.run", "TestHeapCrashChild$") + cmd.Env = append(os.Environ(), heapCrashDirEnv+"="+dir) + out, err := cmd.StdoutPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + var last uint64 + sc := bufio.NewScanner(out) + want := uint64(6 + round*5) + for sc.Scan() { + line := sc.Text() + if !strings.HasPrefix(line, "CHECKPOINT ") { + continue + } + n, err := strconv.ParseUint(strings.TrimPrefix(line, "CHECKPOINT "), 10, 64) + require.NoError(t, err) + last = n + if last >= want { + break + } + } + // Somewhere inside the next block: the child is mid-write or + // mid-sync when it dies + time.Sleep(time.Duration(round*7) * time.Millisecond) + require.NoError(t, cmd.Process.Kill()) + _ = cmd.Wait() + + check := func(h *HeapStore, what string) { + t.Helper() + for i := 0; i < 300; i++ { + v, err := h.Get(crashKey(i)) + require.NoError(t, err, "%s: key %d", what, i) + require.Equal(t, crashValue(h.height, i), v, "%s: key %d is at the durable block", what, i) + } + } + h, err := OpenHeapStore(dir) + require.NoError(t, err) + require.GreaterOrEqual(t, h.height, last, "the durable block is at least the last checkpoint the parent saw") + check(h, "open") + durable := h.height + require.NoError(t, h.Close()) + + // The index thrown away: the data alone must say the same + _, gens, err := listHeap(dir) + require.NoError(t, err) + for _, g := range gens { + require.NoError(t, os.Remove(filepath.Join(dir, indexName(g)))) + } + r, err := RepairHeapStore(dir, durable) + require.NoError(t, err) + require.Equal(t, durable, r.height) + check(r, "repair") + require.NoError(t, r.Close()) + } +} diff --git a/database/heap_test.go b/database/heap_test.go new file mode 100644 index 0000000..4577d88 --- /dev/null +++ b/database/heap_test.go @@ -0,0 +1,451 @@ +package blockchainDB + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func heapDir(t *testing.T) string { + t.Helper() + return filepath.Join(t.TempDir(), "heap") +} + +func key(b byte) (k [32]byte) { k[0] = b; return } + +func syncHeap(t *testing.T, h *HeapStore) { + t.Helper() + p, err := h.beginBlockSync() + require.NoError(t, err) + require.NoError(t, p.finish()) +} + +// Small files, so that rolling and deletion happen within a test. +func smallFiles(t *testing.T) { + t.Helper() + was := HeapFileBytes + HeapFileBytes = 1024 + t.Cleanup(func() { HeapFileBytes = was }) +} + +// A key rewritten within the block reuses its slot; rewritten in a +// later block it is appended, the old slot dead where it lies until +// the mover copies the file's live entries out and the next sync +// deletes the file. +func TestHeapRewriteReusesWithinTheBlockAndMovesOneSyncLate(t *testing.T) { + smallFiles(t) + h, err := NewHeapStore(heapDir(t)) + require.NoError(t, err) + defer h.Close() + h.AdvanceBlock(1) + require.NoError(t, h.Put(key(1), []byte("one"))) + first := h.index[key(1)] + require.NoError(t, h.Put(key(1), []byte("uno"))) + require.Equal(t, first, h.index[key(1)], "same block, fits: in place") + require.EqualValues(t, 1, h.putInPlace.Load()) + v, err := h.Get(key(1)) + require.NoError(t, err) + require.Equal(t, "uno", string(v)) + + syncHeap(t, h) + h.AdvanceBlock(2) + require.NoError(t, h.Put(key(1), []byte("two"))) + require.NotEqual(t, first, h.index[key(1)], "a durable slot is never rewritten") + dead, live := h.HoleRatio() + require.EqualValues(t, entrySize(3), dead, "the old slot is dead where it lies") + require.EqualValues(t, entrySize(3), live-h.deltaBytes(), "one live entry besides the deltas") + // Fill the block's file past its size so it rolls, then kill most + // of what the first file holds + for i := byte(10); i < 40; i++ { + require.NoError(t, h.Put(key(i), make([]byte, 64))) + } + syncHeap(t, h) + require.Greater(t, len(h.files), 1, "the block's file rolled") + h.AdvanceBlock(3) + for i := byte(10); i < 39; i++ { + require.NoError(t, h.Put(key(i), make([]byte, 64))) + } + syncHeap(t, h) + require.NoError(t, h.Snapshot()) // The deltas so far are superseded: dead where they lie + + h.AdvanceBlock(4) + moved, err := h.clean(1 << 20) + require.NoError(t, err) + require.True(t, moved, "a mostly dead file is taken") + require.NotEmpty(t, h.release, "a file emptied by the pass waits for the sync") + for _, id := range h.release { + _, err := os.Stat(filepath.Join(h.Directory, dataName(id))) + require.NoError(t, err, "not deleted yet: the copies are not durable") + } + released := append([]uint32(nil), h.release...) + syncHeap(t, h) + for _, id := range released { + require.Nil(t, h.files[id], "out of the map after the sync") + _, err := os.Stat(filepath.Join(h.Directory, dataName(id))) + require.NoError(t, err, "the unlink is the mover's, off the block's path") + } + require.NoError(t, h.unlinkReleased()) + for _, id := range released { + _, err := os.Stat(filepath.Join(h.Directory, dataName(id))) + require.ErrorIs(t, err, os.ErrNotExist, "deleted by the mover") + } + _, copied := h.Cleaned() + require.Greater(t, copied, uint64(0), "the live entries left in it were copied out") + v, err = h.Get(key(1)) + require.NoError(t, err) + require.Equal(t, "two", string(v)) + for i := byte(10); i < 40; i++ { + _, err := h.Get(key(i)) + require.NoError(t, err) + } +} + +// Reopening replays the generation: every synced value is back, the +// files' accounting is derived, and a block that was never synced is +// gone -- its slots unnamed and its bytes cut from the file. +func TestHeapReopenKeepsTheDurableAndDropsTheRest(t *testing.T) { + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + h.AdvanceBlock(1) + for i := byte(1); i <= 50; i++ { + require.NoError(t, h.Put(key(i), []byte{i})) + } + syncHeap(t, h) + h.AdvanceBlock(2) + require.NoError(t, h.Put(key(1), []byte("rewritten in block 2"))) + syncHeap(t, h) + size := h.deltaAt.off // Where block 2's delta ended: the durable append point + // Block 3: written, never synced -- the crash + h.AdvanceBlock(3) + require.NoError(t, h.Put(key(2), []byte("lost"))) + require.NoError(t, h.Put(key(99), []byte("lost too"))) + for _, hf := range h.files { + hf.f.Close() + } + h.log.Close() + + r, err := OpenHeapStore(dir) + require.NoError(t, err) + defer r.Close() + v, err := r.Get(key(1)) + require.NoError(t, err) + require.Equal(t, "rewritten in block 2", string(v)) + v, err = r.Get(key(2)) + require.NoError(t, err) + require.Equal(t, []byte{2}, v, "block 3's rewrite was never durable") + _, err = r.Get(key(99)) + require.ErrorIs(t, err, errNotFound) + require.Len(t, r.files, 1) + for _, hf := range r.files { + require.Equal(t, size, hf.size, "the file is cut back to the durable append point") + } + dead, _ := r.HoleRatio() + require.EqualValues(t, entrySize(1), dead, "key 1's block-1 slot is dead where it lies") + require.EqualValues(t, 2, r.height, "the durable height: block 3 never synced") +} + +// A torn entry at the end of the data file -- a crash mid-write -- +// is cut, and the deltas before it stand. +func TestHeapTornTailIsCut(t *testing.T) { + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + h.AdvanceBlock(1) + require.NoError(t, h.Put(key(1), []byte("one"))) + syncHeap(t, h) + end := h.deltaAt.off + require.NoError(t, h.Close()) + f, err := os.OpenFile(filepath.Join(dir, dataName(0)), os.O_WRONLY|os.O_APPEND, 0o644) + require.NoError(t, err) + _, err = f.Write([]byte{9, 0, 0, 0, 1, 2, 3, 4, 5, 6}) // A length and a few bytes of nothing + require.NoError(t, err) + require.NoError(t, f.Close()) + + r, err := OpenHeapStore(dir) + require.NoError(t, err) + defer r.Close() + v, err := r.Get(key(1)) + require.NoError(t, err) + require.Equal(t, "one", string(v)) + st, err := os.Stat(filepath.Join(dir, dataName(0))) + require.NoError(t, err) + require.Equal(t, end, st.Size(), "cut back to the last delta") +} + +// A snapshot starts a new generation in its own file and retires the +// old one; the replay lands on the new one; and a generation whose +// write was interrupted is ignored. +func TestHeapSnapshotStartsAGenerationSafely(t *testing.T) { + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + every := HeapSnapshotBlocks + HeapSnapshotBlocks = 1 + defer func() { HeapSnapshotBlocks = every }() + for b := uint64(1); b <= 30; b++ { + h.AdvanceBlock(b) + for i := byte(1); i <= 20; i++ { + require.NoError(t, h.Put(key(i), []byte{byte(b), i})) + } + syncHeap(t, h) + if b == 20 { + _, err := h.compact(HeapCleanBytes) + require.NoError(t, err) + _, err = os.Stat(filepath.Join(dir, indexName(1))) + require.ErrorIs(t, err, os.ErrNotExist, "generation 1 retired") + _, err = os.Stat(filepath.Join(dir, indexName(2))) + require.NoError(t, err) + } + } + require.NoError(t, h.Close()) + // A crash mid-snapshot leaves a .tmp; it is not a generation + require.NoError(t, os.WriteFile(filepath.Join(dir, indexName(3)+".tmp"), []byte("half"), 0o644)) + r, err := OpenHeapStore(dir) + require.NoError(t, err) + defer r.Close() + require.EqualValues(t, 2, r.gen) + _, err = os.Stat(filepath.Join(dir, indexName(3)+".tmp")) + require.ErrorIs(t, err, os.ErrNotExist) + for i := byte(1); i <= 20; i++ { + v, err := r.Get(key(i)) + require.NoError(t, err) + require.Equal(t, []byte{30, i}, v) + } +} + +// A slot whose bytes were damaged is an error, never a value. +func TestHeapChecksumCatchesADamagedSlot(t *testing.T) { + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + h.AdvanceBlock(1) + require.NoError(t, h.Put(key(1), []byte("intact"))) + s := h.index[key(1)] + syncHeap(t, h) + h.AdvanceBlock(2) + require.NoError(t, h.Put(key(2), []byte("later"))) // A later block's sync covers block 1 + require.NoError(t, h.Close()) + f, err := os.OpenFile(filepath.Join(dir, dataName(s.file)), os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteAt([]byte("damaged"), int64(s.off)+heapHeader) + require.NoError(t, err) + require.NoError(t, f.Close()) + r, err := OpenHeapStore(dir) + require.NoError(t, err) + defer r.Close() + _, err = r.Get(key(1)) + require.ErrorContains(t, err, "checksum") +} + +// With the index gone, the map is rebuilt from the data: the highest +// committed copy of each key wins, an entry from the block that never +// synced is dropped. +func TestHeapRepairReadsTheKeysFromTheData(t *testing.T) { + smallFiles(t) + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + h.AdvanceBlock(1) + for i := byte(1); i <= 10; i++ { + require.NoError(t, h.Put(key(i), make([]byte, 100))) + } + syncHeap(t, h) + h.AdvanceBlock(2) + require.NoError(t, h.Put(key(1), []byte{2, 1})) // Newer copy, in a later file + syncHeap(t, h) + h.AdvanceBlock(3) + require.NoError(t, h.Put(key(2), []byte{3, 2})) // Never synced: not committed + for _, hf := range h.files { + hf.f.Close() + } + h.log.Close() + require.NoError(t, os.Remove(filepath.Join(dir, indexName(1)))) + + _, err = OpenHeapStore(dir) + require.ErrorIs(t, err, ErrHeapNeedsRepair) + r, err := RepairHeapStore(dir, 2) + require.NoError(t, err) + defer r.Close() + v, err := r.Get(key(1)) + require.NoError(t, err) + require.Equal(t, []byte{2, 1}, v, "the block-2 copy wins") + v, err = r.Get(key(2)) + require.NoError(t, err) + require.Len(t, v, 100, "block 3 never committed: its copy is dropped") + require.EqualValues(t, 10, r.LiveRecords()) + require.NoError(t, r.Close()) + re, err := OpenHeapStore(dir) + require.NoError(t, err, "the repair left a generation: an ordinary open") + defer re.Close() + v, err = re.Get(key(1)) + require.NoError(t, err) + require.Equal(t, []byte{2, 1}, v) +} + +// A shard built with the heap seals, compacts, closes and reopens as +// a heap, through the same KV2 and KVShard surface, with files rolled +// and deleted along the way. +func TestHeapShardRoundTrip(t *testing.T) { + was := HeapFileBytes + HeapFileBytes = 256 << 10 + defer func() { HeapFileBytes = was }() + dir := filepath.Join(t.TempDir(), "shards") + kvs, err := NewKVShardHeapN(dir, 2, 1000) + require.NoError(t, err) + require.NoError(t, kvs.SetFilterBlocks(MinFilterBlocks)) + fr := NewFastRandom([]byte{7}) + hot := make([][32]byte, 200) + for i := range hot { + hot[i] = fr.NextHash() + } + for b := uint64(1); b <= 60; b++ { + for _, k := range hot { + require.NoError(t, kvs.PutDyna(k, append([]byte{byte(b)}, fr.RandBuff(100, 300)...))) + } + require.NoError(t, kvs.PutPerm(fr.NextHash(), []byte("perm"))) + require.NoError(t, kvs.SealBlock(b)) + if b%10 == 0 { + require.NoError(t, kvs.Compress()) + if b > MinFilterBlocks { + _, err := kvs.MergeFinalized(b - MinFilterBlocks) + require.NoError(t, err) + } + } + } + for _, k := range hot { + v, err := kvs.GetDyna(k) + require.NoError(t, err) + require.Equal(t, byte(60), v[0]) + } + _, dyna := kvs.Stats() + require.EqualValues(t, 60*200, dyna.PutTotal) + // Maintenance is a slice per call, sized by the blocks since the + // last: a call a block over a few blocks visits every shard with + // budget to spare for what the last ten blocks left dead, and each + // seal releases what the pass before it emptied + for b := uint64(61); b <= 64; b++ { + require.NoError(t, kvs.SealBlock(b)) + require.NoError(t, kvs.Compress()) + } + require.NoError(t, kvs.SealBlock(65)) + dead, live := kvs.Shards[0].Heap.HoleRatio() + require.Less(t, dead, 2*live+HeapFileBytes, "dead bytes are bounded: at most the current file, which the mover never takes, beyond the live set") + require.NoError(t, kvs.Close()) + + re, err := OpenKVShard(dir) + require.NoError(t, err) + defer re.Close() + require.NotNil(t, re.Shards[0].Heap, "reopened as a heap") + require.Nil(t, re.Shards[0].DynaKV) + for _, k := range hot { + v, err := re.GetDyna(k) + require.NoError(t, err) + require.Equal(t, byte(60), v[0]) + } +} + +// A key rewritten between the mover's copy and its naming leaves the +// copy dead on arrival: the put's value stands, and the copy's bytes +// are accounted dead in the mover's file. +func TestHeapMoveIsDeadOnArrivalIfTheKeyWasRewritten(t *testing.T) { + smallFiles(t) // Nine 112-byte entries to a file + h, err := NewHeapStore(heapDir(t)) + require.NoError(t, err) + defer h.Close() + h.AdvanceBlock(1) + for i := byte(1); i <= 20; i++ { + require.NoError(t, h.Put(key(i), make([]byte, 64))) + } + syncHeap(t, h) + // Two more blocks rewriting every key but 20 leave key 20 the one + // live entry in a file otherwise dead: the mover's next pick + for b := uint64(2); b <= 3; b++ { + h.AdvanceBlock(b) + for i := byte(1); i <= 19; i++ { + require.NoError(t, h.Put(key(i), make([]byte, 64))) + } + syncHeap(t, h) + } + require.NoError(t, h.Snapshot()) // The deltas so far are superseded: the file is mostly dead + files := HeapCleanFiles + HeapCleanFiles = 64 // One pass takes every eligible file, key 20's half-dead one included + defer func() { HeapCleanFiles = files }() + h.AdvanceBlock(4) + moverHook = func() { + require.NoError(t, h.Put(key(20), []byte("rewritten while moving"))) + } + defer func() { moverHook = nil }() + deadBefore, _ := h.HoleRatio() + moved, err := h.clean(1 << 20) + require.NoError(t, err) + require.True(t, moved) + _, copied := h.Cleaned() + require.GreaterOrEqual(t, copied, uint64(entrySize(64)), "key 20 was among the entries copied") + v, err := h.Get(key(20)) + require.NoError(t, err) + require.Equal(t, "rewritten while moving", string(v), "the put wins") + // Every copied entry left its old slot dead; key 20's copy is dead + // as well, since the put took the key elsewhere before it was named + dead, _ := h.HoleRatio() + require.EqualValues(t, deadBefore+int64(copied)+entrySize(64), dead, "the old slots and the unwanted copy are dead") + syncHeap(t, h) + v, err = h.Get(key(20)) + require.NoError(t, err) + require.Equal(t, "rewritten while moving", string(v)) +} + +// A file the mover copied into is named by the deltas of the current +// generation, as the destination of their ranges. When every copy in +// it dies it must not be unlinked until a snapshot supersedes those +// deltas: never unlink what a durable index names (spec 1.7). Found +// by the platform's reopen check: a store closed cleanly would not +// open, "the index names heap-000006.dat: missing". +func TestHeapMoverFileNamedByDeltasOutlivesItsEntries(t *testing.T) { + was, blocks, pinned := HeapFileBytes, HeapSnapshotBlocks, HeapSnapshotPinnedFiles + HeapFileBytes, HeapSnapshotBlocks, HeapSnapshotPinnedFiles = 64<<10, 1<<20, 1<<30 // No snapshot at all + defer func() { HeapFileBytes, HeapSnapshotBlocks, HeapSnapshotPinnedFiles = was, blocks, pinned }() + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + value := make([]byte, 400) + last := map[byte]byte{} + // Half the keys are rewritten every block, half every seventh: a + // file is soon half dead with live entries the mover copies out, + // and the copies die within seven blocks, emptying the mover's file + put := func(b uint64) { + h.AdvanceBlock(b) + for i := byte(1); i <= 100; i++ { + if i > 50 && b%7 != 0 { + continue + } + value[0] = byte(b) + require.NoError(t, h.Put(key(i), value)) + last[i] = byte(b) + } + syncHeap(t, h) + } + for b := uint64(1); b <= 70; b++ { + put(b) + if b%3 == 0 { + _, err := h.compact(HeapCleanBytes) // Copies live entries into mover files, releases what emptied + require.NoError(t, err) + } + } + _, moved := h.Cleaned() + require.Greater(t, moved, uint64(0), "the mover must have copied") + releases, _, snapshots, _ := h.MoverCost() + t.Logf("moved %d bytes, %d files unlinked, %d snapshots, %d files open, gen %d", moved, releases, snapshots, len(h.files), h.gen) + require.NoError(t, h.Close()) + h, err = OpenHeapStore(dir) + require.NoError(t, err, "a store closed cleanly reopens") + t.Logf("after open: %d files, gen %d, replay point %v", len(h.files), h.gen, h.deltaAt) + for i := byte(1); i <= 100; i++ { + v, err := h.Get(key(i)) + require.NoError(t, err, "key %d", i) + require.Equal(t, last[i], v[0], "key %d", i) + } + require.NoError(t, h.Close()) +} diff --git a/database/kv_2.go b/database/kv_2.go index f5008cd..37c0d7e 100644 --- a/database/kv_2.go +++ b/database/kv_2.go @@ -13,6 +13,8 @@ import ( const PermDirName = "perm" const DynaDirName = "dyna" +const HeapDirName = "dyna-heap" // The dynamic layer as a heap with holes (heap.go) +const PermFilesDirName = "perm-files" // The permanent layer as files of entries and buckets of keys (perm.go) // KV2 // Maintains 2 layers of key value pairs with different immutability characteristics: @@ -58,8 +60,10 @@ type KV2 struct { // every commit and read of the shard for the whole copy (issue #57). Mutex sync.RWMutex Directory string // Directory where the PermKV and DynaKV directories are - PermKV *SegmentStore // The Perm layer: sealed, immutable segments - DynaKV *SegmentStore // The Dyna layer: sealed, mutable segments + PermKV *SegmentStore // The Perm layer: sealed, immutable segments; nil when Perm is set + Perm *PermStore // The Perm layer as files of entries and buckets of keys (perm.go); nil when PermKV is set + DynaKV *SegmentStore // The Dyna layer: sealed, mutable segments; nil when Heap is set + Heap *HeapStore // The Dyna layer as a heap with holes (heap.go); nil when DynaKV is set // dWrites and pWrites count writes to each layer since the last // Compress. Atomic, because the writes that bump them run // concurrently: KV2.Put takes the lock SHARED (see Put). @@ -150,20 +154,123 @@ func (k *KV2) Writes() (dyna, perm int) { func (k *KV2) SetFilterBlocks(n uint64) (err error) { k.Mutex.Lock() defer k.Mutex.Unlock() - if err = k.PermKV.SetFilterBlocks(n); err != nil { + if err = k.perm().SetFilterBlocks(n); err != nil { return err } + if k.DynaKV == nil { + return nil // A heap has no window + } return k.DynaKV.SetFilterBlocks(n) } +// dynaLayer is what KV2 asks of its dynamic layer, whichever of the +// two it opened: sealed mutable segments (SegmentStore) or the heap +// with holes (HeapStore). +type dynaLayer interface { + Open() error + Close() error + Get(key [32]byte) ([]byte, error) + GetDeep(key [32]byte) ([]byte, error) + Put(key [32]byte, value []byte) error + AdvanceBlock(height uint64) + LiveRecords() uint64 + beginBlockSync() (blockSync, error) + compact(budget int64) (bool, error) // A heap moves at most budget bytes; a segment store ignores it + Stats() StoreStats +} + +// blockSync is the second half of a block sync, finished outside the +// shard's lock. +type blockSync interface{ finish() error } + +// permLayer is what KV2 asks of its permanent layer, whichever of the +// two it opened: sealed immutable segments (SegmentStore) or files of +// entries and buckets of keys (PermStore). +type permLayer interface { + Open() error + Close() error + Get(key [32]byte) ([]byte, error) + GetDeep(key [32]byte) ([]byte, error) + Put(key [32]byte, value []byte) error + PutIfAbsent(key [32]byte, value []byte) (existing []byte, existed bool, err error) + LiveCount() int + AdvanceBlock(height uint64) + SetFilterBlocks(n uint64) error + beginPermSeal(height uint64) (blockSync, error) + mergeBelow(height uint64) (bool, error) + Stats() StoreStats +} + +// perm is the permanent layer this shard opened. +func (k *KV2) perm() permLayer { + if k.Perm != nil { + return k.Perm + } + return k.PermKV +} + +// NewKV2Files is NewKV2 with both layers as files of entries: the +// dynamic layer a heap (heap.go), the permanent layer files with +// buckets of keys (perm.go). +func NewKV2Files(directory string, sealLimit uint64) (kv2 *KV2, err error) { + if kv2, err = NewKV2Heap(directory, sealLimit); err != nil { + return nil, err + } + if err = kv2.PermKV.Close(); err != nil { + return nil, err + } + os.RemoveAll(filepath.Join(directory, PermDirName)) + kv2.PermKV = nil + if kv2.Perm, err = NewPermStore(filepath.Join(directory, PermFilesDirName), MinFilterBlocks); err != nil { + return nil, err + } + return kv2, nil +} + +// dyna is the dynamic layer this shard opened. +func (k *KV2) dyna() dynaLayer { + if k.Heap != nil { + return k.Heap + } + return k.DynaKV +} + +// NewKV2Heap is NewKV2 with the dynamic layer as a heap with holes. +func NewKV2Heap(directory string, sealLimit uint64) (kv2 *KV2, err error) { + if kv2, err = NewKV2(directory, sealLimit); err != nil { + return nil, err + } + if err = kv2.DynaKV.Close(); err != nil { + return nil, err + } + os.RemoveAll(filepath.Join(directory, DynaDirName)) + kv2.DynaKV = nil + if kv2.Heap, err = NewHeapStore(filepath.Join(directory, HeapDirName)); err != nil { + return nil, err + } + return kv2, nil +} + func OpenKV2(directory string) (kv2 *KV2, err error) { kv2 = new(KV2) kv2.Directory = directory permDirName := filepath.Join(directory, PermDirName) // Add directory names dynaDirName := filepath.Join(directory, DynaDirName) // Add directory names - if kv2.PermKV, err = OpenSegmentStore(permDirName); err != nil { + // The permanent layer is whichever the store was built with + if _, statErr := os.Stat(filepath.Join(directory, PermFilesDirName, "perm.json")); statErr == nil { + if kv2.Perm, err = OpenPermStore(filepath.Join(directory, PermFilesDirName)); err != nil { + return nil, err + } + } else if kv2.PermKV, err = OpenSegmentStore(permDirName); err != nil { return nil, err } + // The dynamic layer is whichever the store was built with + if _, statErr := os.Stat(filepath.Join(directory, HeapDirName)); statErr == nil { + if kv2.Heap, err = OpenHeapStore(filepath.Join(directory, HeapDirName)); err != nil { + return nil, err + } + return kv2, nil + } if kv2.DynaKV, err = OpenSegmentStore(dynaDirName); err != nil { return nil, err } @@ -179,19 +286,21 @@ func (k *KV2) Open() error { } k.Mutex.Lock() defer k.Mutex.Unlock() - if err := k.PermKV.Open(); err != nil { + if err := k.perm().Open(); err != nil { return err } if k.SealLimit == 0 { // Restore what the database was built with; only a store // predating the persisted field falls back to a default - if limit := k.PermKV.SealLimit; limit > 0 { + if k.PermKV == nil { + k.SealLimit = int(DefaultBloomCapacity) + } else if limit := k.PermKV.SealLimit; limit > 0 { k.SealLimit = int(limit) } else { k.SealLimit = int(DefaultBloomCapacity) } } - if err := k.DynaKV.Open(); err != nil { + if err := k.dyna().Open(); err != nil { return err } k.opened.Store(true) @@ -212,8 +321,8 @@ func (k *KV2) Close() error { k.Mutex.Lock() defer k.Mutex.Unlock() k.opened.Store(false) - err := k.PermKV.Close() - if dynaErr := k.DynaKV.Close(); err == nil { + err := k.perm().Close() + if dynaErr := k.dyna().Close(); err == nil { err = dynaErr } return err @@ -225,7 +334,7 @@ func (k *KV2) GetDyna(key [32]byte) (value []byte, err error) { k.Mutex.RLock() defer k.Mutex.RUnlock() - if value, err = k.DynaKV.Get(key); err != nil { // Not in DynaKV, then return whatever + if value, err = k.dyna().Get(key); err != nil { // Not in DynaKV, then return whatever return nil, err } return value, nil @@ -244,7 +353,7 @@ func (k *KV2) GetPerm(key [32]byte) (value []byte, err error) { k.Mutex.RLock() defer k.Mutex.RUnlock() - if value, err = k.PermKV.Get(key); err != nil { // Not in PermKV, then return whatever + if value, err = k.perm().Get(key); err != nil { // Not in PermKV, then return whatever return nil, err } return value, nil @@ -258,7 +367,7 @@ func (k *KV2) GetPerm(key [32]byte) (value []byte, err error) { func (k *KV2) GetPermDeep(key [32]byte) (value []byte, err error) { k.Mutex.RLock() defer k.Mutex.RUnlock() - return k.PermKV.GetDeep(key) + return k.perm().GetDeep(key) } // Get @@ -284,7 +393,7 @@ func (k *KV2) Get(key [32]byte) (value []byte, err error) { defer k.Mutex.RUnlock() // Check and see if this is a key that has been changed - value, err = k.DynaKV.Get(key) + value, err = k.dyna().Get(key) switch { case err == nil: return value, nil @@ -297,7 +406,7 @@ func (k *KV2) Get(key [32]byte) (value []byte, err error) { // is how it says so. return nil, err } - return k.PermKV.Get(key) // Not in DynaKV; return whatever PermKV has + return k.perm().Get(key) // Not in DynaKV; return whatever PermKV has } // GetDeep @@ -308,14 +417,14 @@ func (k *KV2) GetDeep(key [32]byte) (value []byte, err error) { k.Mutex.RLock() defer k.Mutex.RUnlock() - value, err = k.DynaKV.GetDeep(key) + value, err = k.dyna().GetDeep(key) switch { case err == nil: return value, nil case !errors.Is(err, errNotFound): return nil, err // See Get: a failure is not an absence } - return k.PermKV.GetDeep(key) + return k.perm().GetDeep(key) } // PutDyna @@ -326,7 +435,7 @@ func (k *KV2) PutDyna(key [32]byte, value []byte) (writes int, err error) { k.Mutex.RLock() // Shared: see Put (issue #66) defer k.Mutex.RUnlock() k.dWrites.Add(1) - if err = k.DynaKV.Put(key, value); err != nil { + if err = k.dyna().Put(key, value); err != nil { return int(k.dWrites.Load()), err } autoSeal, err = k.sealDynaIfFull() @@ -341,7 +450,7 @@ func (k *KV2) PutPerm(key [32]byte, value []byte) (writes int, err error) { k.Mutex.RLock() // Shared: see Put (issue #66) defer k.Mutex.RUnlock() k.pWrites.Add(1) - if err = k.PermKV.Put(key, value); err != nil { + if err = k.perm().Put(key, value); err != nil { return int(k.pWrites.Load()), err } autoSeal, err = k.sealPermIfFull() @@ -374,7 +483,7 @@ func finishAutoSeal(p *pendingSeal, err error) error { // published and under the exclusive lock in Open, and the layer // serializes its own seal. func (k *KV2) sealPermIfFull() (p *pendingSeal, err error) { - if k.SealLimit <= 0 || k.PermKV.LiveCount() < k.SealLimit { + if k.Perm != nil || k.SealLimit <= 0 || k.PermKV.LiveCount() < k.SealLimit { return nil, nil } return k.PermKV.beginSealNext() @@ -388,8 +497,8 @@ func (k *KV2) sealPermIfFull() (p *pendingSeal, err error) { // replayed in full on every open. The caller must hold the KV2 lock, // shared or exclusive; see sealPermIfFull. func (k *KV2) sealDynaIfFull() (p *pendingSeal, err error) { - if k.SealLimit <= 0 || k.DynaKV.LiveRecords() < uint64(k.SealLimit) { - return nil, nil + if k.Heap != nil || k.SealLimit <= 0 || k.DynaKV.LiveRecords() < uint64(k.SealLimit) { + return nil, nil // A heap is never sealed: it has no tail to fill } return k.DynaKV.beginSealNext() } @@ -436,9 +545,9 @@ func (k *KV2) sealDynaIfFull() (p *pendingSeal, err error) { // out (issue #84; the two halves are seal.go's). func (k *KV2) Seal(height uint64) (meta SegmentMeta, err error) { k.Mutex.Lock() - perm, err := k.PermKV.beginSeal(height) - dyna, dynaErr := k.DynaKV.beginSync() - k.DynaKV.AdvanceBlock(height + 1) + perm, err := k.perm().beginPermSeal(height) + dyna, dynaErr := k.dyna().beginBlockSync() + k.dyna().AdvanceBlock(height + 1) k.Mutex.Unlock() // Both halves finish regardless of the other: a failure to sync @@ -452,7 +561,7 @@ func (k *KV2) Seal(height uint64) (meta SegmentMeta, err error) { }() } if perm != nil { - meta, err = perm.finish() + err = perm.finish() } wg.Wait() if err == nil { @@ -474,6 +583,10 @@ func (k *KV2) Seal(height uint64) (meta SegmentMeta, err error) { // KV2 lock here held every Put, Get and Seal of the shard for the // whole copy, which is the pause issue #57 measures. func (k *KV2) MergeBelow(height uint64) (meta SegmentMeta, merged bool, err error) { + if k.Perm != nil { + merged, err = k.Perm.mergeBelow(height) + return meta, merged, err + } return k.PermKV.MergeBelow(height) } @@ -517,12 +630,12 @@ func (k *KV2) Put(key [32]byte, value []byte) (writes int, err error) { k.Mutex.RLock() defer k.Mutex.RUnlock() - if value2, err2 := k.DynaKV.Get(key); err2 == nil { // Check. Is this a DynaKV key? + if value2, err2 := k.dyna().Get(key); err2 == nil { // Check. Is this a DynaKV key? if bytes.Equal(value, value2) { // If the key is in DynaKV, it stays there. return int(k.dWrites.Load()), nil // If the value is not changed, do nothing } k.dWrites.Add(1) - if err = k.DynaKV.Put(key, value); err != nil { // If the value DID change, update + if err = k.dyna().Put(key, value); err != nil { // If the value DID change, update return int(k.dWrites.Load()), err } autoSeal, err = k.sealDynaIfFull() @@ -534,7 +647,7 @@ func (k *KV2) Put(key [32]byte, value []byte) (writes int, err error) { // whether the key was there and then calling Put, which asked again // to enforce immutability, made a new key -- the common case, and a // miss by definition -- pay for the answer twice. - existing, existed, err := k.PermKV.PutIfAbsent(key, value) + existing, existed, err := k.perm().PutIfAbsent(key, value) if err != nil { return int(k.dWrites.Load()), err } @@ -553,7 +666,7 @@ func (k *KV2) Put(key [32]byte, value []byte) (writes int, err error) { return int(k.dWrites.Load()), nil } k.dWrites.Add(1) - if err = k.DynaKV.Put(key, value); err != nil { // If the perm value changed, it is now a DynaKV + if err = k.dyna().Put(key, value); err != nil { // If the perm value changed, it is now a DynaKV return int(k.dWrites.Load()), err } autoSeal, err = k.sealDynaIfFull() @@ -593,8 +706,11 @@ func (k *KV2) Put(key [32]byte, value []byte) (writes int, err error) { // weight rather than a wrong answer. // // TODO: Cleanse PermKV of keys in DynaKV -func (k *KV2) Compress() error { - if _, err := k.DynaKV.CompactHistory(); err != nil { +func (k *KV2) Compress() error { return k.compress(HeapCleanBytes) } + +// compress is Compress with the mover's budget for this call. +func (k *KV2) compress(budget int64) error { + if _, err := k.dyna().compact(budget); err != nil { return err } k.Mutex.Lock() diff --git a/database/kv_shard.go b/database/kv_shard.go index 9a31c74..504e237 100644 --- a/database/kv_shard.go +++ b/database/kv_shard.go @@ -82,6 +82,13 @@ type KVShard struct { // found" for data that is on disk. Shards []*KV2 + // Maintenance in rotation (Compress): the height last sealed, the + // height at the last call, and the shard next in line + maintMu sync.Mutex + sealed uint64 + maintAt uint64 + maintNext int + // Sets holds the finalized Perm data that has left the shards: one // block-set file per completed set of blocks, packed from every // shard's merged segment (blockset.go). Each shard's Perm layer @@ -103,6 +110,9 @@ func (k *KVShard) attachSets() (err error) { if shard == nil || shard.PermKV == nil { continue } + if shard.PermKV == nil { + continue // A file-backed permanent layer keeps its own deep history + } if err = shard.PermKV.attachCold(shardSets{k.Sets, i}); err != nil { return fmt.Errorf("shard %d: %w", i, err) } @@ -169,6 +179,7 @@ func OpenKVShard(directory string) (kVShard *KVShard, err error) { return nil, err } } + kVShard.phaseSnapshots() kVShard.useSharedBlockRecord() if err = kVShard.adoptBlockHeight(); err != nil { return nil, err @@ -195,6 +206,9 @@ func OpenKVShard(directory string) (kVShard *KVShard, err error) { // again, which commits and retires them if newest, ok := kVShard.Sets.Newest(); ok { for i, shard := range kVShard.Shards { + if shard.PermKV == nil { + continue + } if _, err = shard.PermKV.DropBelow(newest.Last + 1); err != nil { return nil, fmt.Errorf("shard %d: %w", i, err) } @@ -227,7 +241,34 @@ func NewKVShard(directory string, sealLimit uint64) (kvs *KVShard, err error) { // // How many to ask for is a question about rate, not about size; see // DefaultNumShards. +// NewKVShardHeapN is NewKVShardN with every shard's dynamic layer a +// heap with holes (heap.go). +func NewKVShardHeapN(directory string, shards int, sealLimit uint64) (kvs *KVShard, err error) { + return newKVShardN(directory, shards, sealLimit, NewKV2Heap) +} + +// NewKVShardFilesN is NewKVShardN with both layers as files of entries +// (heap.go, perm.go). +func NewKVShardFilesN(directory string, shards int, sealLimit uint64) (kvs *KVShard, err error) { + return newKVShardN(directory, shards, sealLimit, NewKV2Files) +} + func NewKVShardN(directory string, shards int, sealLimit uint64) (kvs *KVShard, err error) { + return newKVShardN(directory, shards, sealLimit, NewKV2) +} + +// phaseSnapshots spreads the heap shards' key-map snapshots over the +// snapshot period, so that a store's shards do not all write theirs +// on the same block. +func (k *KVShard) phaseSnapshots() { + for i, shard := range k.Shards { + if shard.Heap != nil { + shard.Heap.SetSnapshotPhase(uint64(i) * HeapSnapshotBlocks / uint64(len(k.Shards))) + } + } +} + +func newKVShardN(directory string, shards int, sealLimit uint64, newShard func(string, uint64) (*KV2, error)) (kvs *KVShard, err error) { if shards < 1 { return nil, fmt.Errorf("a database needs at least one shard, asked for %d", shards) } @@ -241,10 +282,11 @@ func NewKVShardN(directory string, shards int, sealLimit uint64) (kvs *KVShard, kvs.Shards = make([]*KV2, shards) for i := range kvs.Shards { // Then create all the shards shardDir := kvs.ShardDir(i) - if kvs.Shards[i], err = NewKV2(shardDir, sealLimit); err != nil { // Create the KV2 for each shard + if kvs.Shards[i], err = newShard(shardDir, sealLimit); err != nil { // Create the KV2 for each shard return nil, err } } + kvs.phaseSnapshots() kvs.useSharedBlockRecord() if kvs.Sets, err = NewSetStore(kvs.setDir()); err != nil { return nil, err @@ -446,6 +488,9 @@ func (k *KVShard) writeBlockHeight(height uint64) (err error) { // with the block it belongs to func (k *KVShard) adoptBlockHeight() error { height, err := k.readBlockHeight() + k.maintMu.Lock() + k.sealed = height + k.maintMu.Unlock() if err != nil { return err } @@ -457,10 +502,10 @@ func (k *KVShard) adoptBlockHeight() error { continue } if shard.PermKV != nil { - shard.PermKV.AdvanceBlock(height) + shard.perm().AdvanceBlock(height) } if shard.DynaKV != nil { // So that its window is where the set's is - shard.DynaKV.AdvanceBlock(height) + shard.dyna().AdvanceBlock(height) } } return nil @@ -471,7 +516,9 @@ func (k *KVShard) adoptBlockHeight() error { func (k *KVShard) useSharedBlockRecord() { for _, shard := range k.Shards { if shard != nil && shard.PermKV != nil { - shard.PermKV.ExternalBlockRecord = true + if shard.PermKV != nil { + shard.PermKV.ExternalBlockRecord = true + } } } } @@ -519,7 +566,13 @@ func (k *KVShard) SealBlock(height uint64) (err error) { return err } } - return k.writeBlockHeight(height + 1) + if err = k.writeBlockHeight(height + 1); err != nil { + return err + } + k.maintMu.Lock() + k.sealed = height + 1 + k.maintMu.Unlock() + return nil } // MergeFinalized @@ -592,6 +645,19 @@ func (k *KVShard) MergeFinalized(height uint64) (mergedShards int, err error) { // pack is done, and whichever of the two lands first, the drop that // follows removes what the set holds. func (k *KVShard) PackFinalized(height uint64) (meta SetMeta, packed bool, err error) { + // A file-backed permanent layer packs within the shard: its + // buckets retire into one run of keys, no bodies, no set file + if len(k.Shards) > 0 && k.Shards[0].Perm != nil { + for i, shard := range k.Shards { + if err = shard.Open(); err != nil { + return meta, false, fmt.Errorf("shard %d: %w", i, err) + } + if err = shard.Perm.Pack(); err != nil { + return meta, false, fmt.Errorf("shard %d: %w", i, err) + } + } + return meta, true, nil + } // One pack at a time. Every guard in here and below is a // check-then-act on state another pack can change: two calls read // the same watermark and pack the same segments into overlapping @@ -704,16 +770,22 @@ func (k *KVShard) Stats() (perm, dyna StoreStats) { dst.HistorySegments += s.HistorySegments dst.ActiveSegments += s.ActiveSegments dst.ResidentBloomBytes += s.ResidentBloomBytes + dst.ResidentIndexBytes += s.ResidentIndexBytes + dst.HeapFiles += s.HeapFiles + dst.HeapLiveBytes += s.HeapLiveBytes + dst.HeapDeadBytes += s.HeapDeadBytes + dst.HeapScannedBytes += s.HeapScannedBytes + dst.HeapMovedBytes += s.HeapMovedBytes } for _, shard := range k.Shards { if shard == nil { continue } if shard.PermKV != nil { - add(&perm, shard.PermKV.Stats()) + add(&perm, shard.perm().Stats()) } - if shard.DynaKV != nil { - add(&dyna, shard.DynaKV.Stats()) + if shard.Heap != nil || shard.DynaKV != nil { + add(&dyna, shard.dyna().Stats()) } } return perm, dyna @@ -722,11 +794,59 @@ func (k *KVShard) Stats() (perm, dyna StoreStats) { // Compress // Compress all the shards func (k *KVShard) Compress() (err error) { - for i, kvs := range k.Shards { - if err = kvs.Open(); err != nil { + if len(k.Shards) > 0 { + if err = k.Shards[0].Open(); err != nil { + return fmt.Errorf("shard 0: %w", err) + } + } + if len(k.Shards) == 0 || k.Shards[0].Heap == nil { + // Segment-store shards: a bounded pass on every shard, as before + for i, kvs := range k.Shards { + if err = kvs.Open(); err != nil { + return fmt.Errorf("shard %d: %w", i, err) + } + if err = kvs.Compress(); err != nil { + return fmt.Errorf("shard %d: %w", i, err) + } + } + return nil + } + // Heap shards: one slice of the store's maintenance, sized by the + // blocks since the last call. The store's mover rate is + // HeapStoreCleanBytes per HeapCleanPeriod blocks; a call takes the + // shards next in rotation, as many as the elapsed blocks earn, each + // with its share of what those blocks earned. Called every + // HeapCleanPeriod blocks it moves the whole budget over every + // shard; called every block it moves a little on the next shard, + // and the copies reach the device as a trickle rather than every + // shard's pass at once -- which, at nine stores in lockstep, was + // 300 MB and 72 barriers in one second, and queued the seals' + // fsyncs behind them for the next four. A hundred shards spread + // the same rate a hundred ways, and each call locks one. + k.maintMu.Lock() + n := len(k.Shards) + elapsed := k.sealed - k.maintAt + if k.maintAt == 0 || elapsed > HeapCleanPeriod { + elapsed = HeapCleanPeriod + } + k.maintAt = k.sealed + due := int(uint64(n) * elapsed / HeapCleanPeriod) + if due < 1 { + due = 1 + } + if due > n { + due = n + } + budget := HeapStoreCleanBytes * int64(elapsed) / int64(HeapCleanPeriod) / int64(due) + start := k.maintNext + k.maintNext = (start + due) % n + k.maintMu.Unlock() + for j := 0; j < due; j++ { + i := (start + j) % n + if err = k.Shards[i].Open(); err != nil { return fmt.Errorf("shard %d: %w", i, err) } - if err = kvs.Compress(); err != nil { + if err = k.Shards[i].compress(budget); err != nil { return fmt.Errorf("shard %d: %w", i, err) } } diff --git a/database/perm.go b/database/perm.go new file mode 100644 index 0000000..7831ebf --- /dev/null +++ b/database/perm.go @@ -0,0 +1,1324 @@ +package blockchainDB + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "sync" + "sync/atomic" +) + +// PermStore is the permanent layer as files of entries and buckets of +// keys (proposal docs/proposals/2026-09-16-entries-written-once.md, +// step 2). An entry is written once at the end of a data file and +// never moves; everything the store does to stay fast is done on +// 44-byte records that name entries (permindex.go). +// +// Files, in one directory: +// +// perm-N.dat entries, [len][height][key][value][crc] (heap.go's +// layout), appended in the order written, rolled at +// PermFileBytes; never rewritten +// runs-N.dat maintenance's run file: a bucket's run at every merge, +// a retired run at every pack; rolled at PermFileBytes +// and deleted once no run in it is referenced +// +// A block's delta -- its records' run, with its filter -- is appended +// to the block's data file right behind the block's entries, as an +// entry under the reserved key heap.go's deltas use, so a block is +// ONE fsync per shard for the permanent layer: entries and delta +// together. Open replays the deltas in the data files after the +// point the manifest records, verifies the last one's entries by +// their checksums before trusting it, and cuts the newest data file +// back to the last delta. +// +// perm.json the manifest: which runs are the window, the buckets +// and the retired history, and where in the run files +// the deltas not yet in the manifest begin +// +// Tiers, newest to oldest: +// +// - the live map: this block's records, in memory until the seal; +// - the window: the last FilterBlocks deltas, each a run with its +// filter resident. The protocol path reads the live map and the +// window and nothing older (spec 1.3): an immutable key the window +// does not hold is absent; +// - the buckets: PermBuckets buckets by the key's first byte, each a +// few sorted runs. A delta that leaves the window feeds each +// bucket's recent records; every bucket is merged every +// PermMergeEvery blocks, in rotation, its recent records written +// as a run and its runs folded by the store's ratio. The buckets +// hold only the history above the pack watermark, so a bucket is +// bounded by the pack period and hashed keys keep it even; +// - retired runs: at every pack, every bucket's runs and every +// delta below the watermark fold into one sorted run, and the +// buckets start again. Retired runs are the deep history. +// +// GetDeep walks the tiers in that order; Get stops after the window. +// +// Durability (spec 1.8): a seal fsyncs the data files the block wrote +// and then appends and fsyncs the block's delta, so a delta names +// only durable entries. Open reads the manifest, then every whole +// delta after the offset it records; a torn delta is dropped whole. +// Maintenance writes new runs, fsyncs them, and commits the manifest +// (written aside and renamed) before any run file is deleted: never +// unlink what a durable manifest names. +type PermStore struct { + Directory string + + mu sync.RWMutex + files map[uint32]*heapFile // Data files, by id + cur *heapFile // The data file the block appends to + nextID uint32 + dirty map[uint32]*heapFile + + runs map[uint32]*runFile // Run files, by id + maintRun *runFile // The run file maintenance appends to: never the seal's, so their barriers never share an inode + nextRun uint32 + live map[[32]byte]permRecord // This block's records + window []*permDelta // The last FilterBlocks deltas, oldest first + pending []*permDelta // Deltas below the window not yet in every bucket + buckets [PermBuckets]permBucket // The history above the watermark + retired []*permRun // The history below it, newest last + height uint64 + window_n uint64 // FilterBlocks + rotation int // The next bucket to merge + lastMerge uint64 // The height Merge last ran at: what decides how many buckets are due + + // The manifest's view: the run file and offset after which deltas + // are replayed on open + deltasFrom struct { + file uint32 + off int64 + } + syncMu sync.Mutex // Serializes seals with each other and with manifest commits + manifestAt uint64 // The height the manifest was last committed at + manifestDirty bool // Merges since then + closed bool + + putTotal, putDuplicate, lookups, windowHits, deepHits atomic.Uint64 + mergeRuns, foldRuns, packRuns atomic.Uint64 + indexBytes atomic.Uint64 +} + +// PermManifestBlocks is how many blocks may pass between manifest +// commits when no fold made one necessary. Counted in blocks so the +// caller's cadence does not set the commit cadence. +var PermManifestBlocks uint64 = 160 + +// PermBuckets is how many buckets a shard's history above the +// watermark is kept in, by the key's first byte. +const PermBuckets = 256 + +// PermMergeEvery is how many blocks pass between merges of one +// bucket: PermBuckets/PermMergeEvery buckets are merged each block. +// It bounds how many deltas wait unmerged, which every deep read +// probes and every merge reads a slice of: at 256, read p99 climbed +// from 7 to 47 us over five minutes and merge work from 157 to 305 s +// a minute. One window's worth keeps both flat; the extra folds +// cost a fraction of the index bytes. +var PermMergeEvery uint64 = MinFilterBlocks + +// PermFileBytes is the size data files and run files are rolled at. +var PermFileBytes int64 = 64 << 20 + +// PermFoldRatio is the ratio a bucket's runs fold by: a suffix of runs +// folds while each older run is no larger than 1/ratio of what has +// gathered behind it. +var PermFoldRatio = 0.25 + +type permDelta struct { + height uint64 + run *permRun + f *os.File // The data file the delta lies in + data uint32 // Its id +} + +type permBucket struct { + runs []*permRun // Oldest first + files []*runFile + merged uint64 // The height of the newest delta merged in +} + +// runFile is a file of runs and how many runs still reference it. +type runFile struct { + id uint32 + f *os.File + size int64 + refs int + unsynced bool // Runs written since the file was last fsynced +} + +func permDataName(id uint32) string { return fmt.Sprintf("perm-%06d.dat", id) } +func permRunName(id uint32) string { return fmt.Sprintf("runs-%06d.dat", id) } +func (rf *runFile) name() string { return permRunName(rf.id) } + +// NewPermStore creates an empty store in directory, replacing anything +// there. +func NewPermStore(directory string, filterBlocks uint64) (*PermStore, error) { + os.RemoveAll(directory) + if err := os.MkdirAll(directory, 0o755); err != nil { + return nil, err + } + p := &PermStore{Directory: directory, window_n: filterBlocks} + return p, p.Open() +} + +// OpenPermStore opens the store in directory as it was left. +func OpenPermStore(directory string) (*PermStore, error) { + if _, err := os.Stat(filepath.Join(directory, "perm.json")); err != nil { + return nil, fmt.Errorf("open perm at %s: %w", directory, err) + } + p := &PermStore{Directory: directory} + return p, p.Open() +} + +// permManifest is perm.json. +type permManifest struct { + Version int `json:"version"` + Height uint64 `json:"height"` + FilterBlocks uint64 `json:"filterBlocks"` + Rotation int `json:"rotation"` + NextData uint32 `json:"nextData"` + NextRun uint32 `json:"nextRun"` + DeltasFile uint32 `json:"deltasFile"` // Deltas after this file:offset are replayed + DeltasOff int64 `json:"deltasOff"` + Window []permRunRef `json:"window"` + Pending []permRunRef `json:"pending"` + Buckets []permBucketM `json:"buckets"` + Retired []permRunRef `json:"retired"` + DataFiles []uint32 `json:"dataFiles"` + RunFiles []uint32 `json:"runFiles"` +} + +type permRunRef struct { + File uint32 `json:"file"` // A data file for a delta, a run file otherwise + Off int64 `json:"off"` + Height uint64 `json:"height,omitempty"` +} + +type permBucketM struct { + Runs []permRunRef `json:"runs"` + Merged uint64 `json:"merged"` +} + +// Open loads the manifest and replays the deltas after it. +func (p *PermStore) Open() (err error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.files != nil { + return nil + } + p.files = map[uint32]*heapFile{} + p.dirty = map[uint32]*heapFile{} + p.runs = map[uint32]*runFile{} + p.live = map[[32]byte]permRecord{} + p.closed = false + var m permManifest + buf, err := os.ReadFile(filepath.Join(p.Directory, "perm.json")) + switch { + case errors.Is(err, os.ErrNotExist): + // A new store: the first data file and run file + if p.window_n == 0 { + p.window_n = MinFilterBlocks + } + if p.cur, err = p.newDataFile(); err != nil { + return err + } + return p.commitManifest() + case err != nil: + return err + } + if err = json.Unmarshal(buf, &m); err != nil { + return fmt.Errorf("perm.json: %w", err) + } + if m.Version != 1 { + return fmt.Errorf("perm.json: version %d, want 1", m.Version) + } + p.height, p.window_n, p.rotation, p.nextID, p.nextRun = m.Height, m.FilterBlocks, m.Rotation, m.NextData, m.NextRun + for _, id := range m.DataFiles { + f, err := os.OpenFile(filepath.Join(p.Directory, permDataName(id)), os.O_RDWR, 0o644) + if err != nil { + return fmt.Errorf("perm: the manifest names %s: %w", permDataName(id), err) + } + st, _ := f.Stat() + p.files[id] = &heapFile{id: id, f: f, size: st.Size()} + } + for _, id := range m.RunFiles { + f, err := os.OpenFile(filepath.Join(p.Directory, permRunName(id)), os.O_RDWR, 0o644) + if err != nil { + return fmt.Errorf("perm: the manifest names %s: %w", permRunName(id), err) + } + st, _ := f.Stat() + p.runs[id] = &runFile{id: id, f: f, size: st.Size()} + } + // Data files the seals rolled after the manifest's commit are not + // named by it, but they hold committed deltas: they are taken in + // id order from the manifest's next id for as long as they exist, + // and the replay below covers them. Run files past the manifest's + // next id are maintenance output that was never named; nothing + // durable refers to them, so they go, and their ids are free for + // O_EXCL creation again. + for { + f, err := os.OpenFile(filepath.Join(p.Directory, permDataName(p.nextID)), os.O_RDWR, 0o644) + if errors.Is(err, os.ErrNotExist) { + break + } + if err != nil { + return err + } + st, _ := f.Stat() + p.files[p.nextID] = &heapFile{id: p.nextID, f: f, size: st.Size()} + p.nextID++ + } + for id := p.nextRun; ; id++ { + err := os.Remove(filepath.Join(p.Directory, (&runFile{id: id}).name())) + if errors.Is(err, os.ErrNotExist) { + break + } + if err != nil { + return err + } + } + load := func(ref permRunRef, resident bool) (*permRun, *runFile, error) { + rf := p.runs[ref.File] + if rf == nil { + return nil, nil, fmt.Errorf("perm: run file %d not open", ref.File) + } + r, err := openPermRun(rf.f, ref.File, ref.Off, resident) + if err != nil { + return nil, nil, err + } + rf.refs++ + return r, rf, nil + } + loadDelta := func(ref permRunRef) (*permDelta, error) { + hf := p.files[ref.File] + if hf == nil { + return nil, fmt.Errorf("perm: data file %d not open", ref.File) + } + r, err := openPermRun(hf.f, ref.File, ref.Off, true) + if err != nil { + return nil, err + } + return &permDelta{height: ref.Height, run: r, f: hf.f, data: ref.File}, nil + } + for _, ref := range m.Window { + d, err := loadDelta(ref) + if err != nil { + return err + } + p.window = append(p.window, d) + } + for _, ref := range m.Pending { + d, err := loadDelta(ref) + if err != nil { + return err + } + p.pending = append(p.pending, d) + } + for i, bm := range m.Buckets { + if i >= PermBuckets { + break + } + p.buckets[i].merged = bm.Merged + for _, ref := range bm.Runs { + r, rf, err := load(ref, true) + if err != nil { + return err + } + p.buckets[i].runs = append(p.buckets[i].runs, r) + p.buckets[i].files = append(p.buckets[i].files, rf) + } + } + for _, ref := range m.Retired { + r, _, err := load(ref, false) + if err != nil { + return err + } + p.retired = append(p.retired, r) + } + // The current data file is the newest; the seal's run file is the + // manifest's; maintenance opens a file of its own when it runs + for _, hf := range p.files { + if p.cur == nil || hf.id > p.cur.id { + p.cur = hf + } + } + p.deltasFrom.file, p.deltasFrom.off = m.DeltasFile, m.DeltasOff + return p.replayDeltas() +} + +// replayDeltas applies every delta entry in the data files after the +// manifest's point, oldest first, into the window and pending. The +// last delta is trusted only if every entry it names checks -- one +// fsync covered it and its entries together -- and the newest data +// file is cut back to the last delta admitted. The caller holds the +// lock. +func (p *PermStore) replayDeltas() error { + ids := make([]uint32, 0, len(p.files)) + for id := range p.files { + if id >= p.deltasFrom.file { + ids = append(ids, id) + } + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + type found struct { + hf *heapFile + off, end int64 + run *permRun + } + var deltas []found + for _, id := range ids { + hf := p.files[id] + buf := make([]byte, hf.size) + if _, err := hf.f.ReadAt(buf, 0); err != nil && !errors.Is(err, io.EOF) { + return err + } + at := int64(0) + if id == p.deltasFrom.file { + at = p.deltasFrom.off + } + for at < int64(len(buf)) { + size, _, key, _, ok := decodeEntry(buf[at:]) + if size == 0 { + break + } + if ok && key == heapDeltaKey { + r, err := openPermRun(hf.f, id, at+heapHeader, true) + if err == nil { + deltas = append(deltas, found{hf: hf, off: at, end: at + size, run: r}) + } + } + at += size + } + } + var last *found + for i := range deltas { + d := &deltas[i] + if i == len(deltas)-1 && !p.deltaEntriesCheck(d.run, d.hf.f) { + break // The block did not commit + } + p.admit(&permDelta{height: d.run.height, run: d.run, f: d.hf.f, data: d.hf.id}) + if d.run.height >= p.height { + p.height = d.run.height + 1 + } + last = d + } + // The newest data file is cut back to what is committed: the last + // delta admitted, or the manifest's point + newest := p.files[ids[len(ids)-1]] + end := int64(0) + if newest.id == p.deltasFrom.file { + end = p.deltasFrom.off + } + if last != nil && last.hf == newest { + end = last.end + } + if newest.size > end { + if err := newest.f.Truncate(end); err != nil { + return err + } + newest.size = end + } + p.cur = newest + return nil +} + +// deltaEntriesCheck reads every entry a delta names and checks it. +func (p *PermStore) deltaEntriesCheck(r *permRun, f *os.File) bool { + recs, err := r.records(f) + if err != nil { + return false + } + for _, rec := range recs { + if _, err := p.readEntry(rec, rec.key); err != nil { + return false + } + } + return true +} + +// admit puts a delta in the window and moves what falls out of it to +// pending. The caller holds the lock. +func (p *PermStore) admit(d *permDelta) { + p.window = append(p.window, d) + for len(p.window) > int(p.window_n) { + p.pending = append(p.pending, p.window[0]) + p.window = p.window[1:] + } +} + +func (p *PermStore) newDataFile() (*heapFile, error) { + id := p.nextID + p.nextID++ + f, err := os.OpenFile(filepath.Join(p.Directory, permDataName(id)), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return nil, err + } + hf := &heapFile{id: id, f: f} + p.files[id] = hf + return hf, nil +} + +func (p *PermStore) newRunFile() (*runFile, error) { + id := p.nextRun + p.nextRun++ + rf := &runFile{id: id} + f, err := os.OpenFile(filepath.Join(p.Directory, rf.name()), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return nil, err + } + rf.f = f + p.runs[id] = rf + return rf, nil +} + +// Close seals what is pending and closes the files. +func (p *PermStore) Close() error { + sync, err := p.beginSeal(p.height) + if err != nil { + return err + } + if err = sync.finish(); err != nil { + return err + } + p.mu.Lock() + defer p.mu.Unlock() + if p.manifestDirty { + if err = p.syncRunFiles(); err != nil { + return err + } + if err = p.commitManifest(); err != nil { + return err + } + if err = p.dropUnreferencedRunFiles(); err != nil { + return err + } + } + p.closed = true + for _, hf := range p.files { + if cerr := hf.f.Close(); err == nil { + err = cerr + } + } + for _, rf := range p.runs { + if cerr := rf.f.Close(); err == nil { + err = cerr + } + } + p.files, p.runs, p.cur, p.maintRun = nil, nil, nil, nil + return err +} + +// PutIfAbsent writes value under key unless the window already holds +// the key, in which case the existing value is returned instead. The +// window is the store's immutability horizon (spec 1.3). +func (p *PermStore) PutIfAbsent(key [32]byte, value []byte) (existing []byte, existed bool, err error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { + return nil, false, errStoreClosed + } + p.putTotal.Add(1) + if rec, ok := p.live[key]; ok { + p.putDuplicate.Add(1) + v, err := p.readEntry(rec, key) + return v, true, err + } + for i := len(p.window) - 1; i >= 0; i-- { + d := p.window[i] + rec, found, err := d.run.lookup(d.f, key) + if err != nil { + return nil, false, err + } + if found { + p.putDuplicate.Add(1) + v, err := p.readEntry(rec, key) + return v, true, err + } + } + size := entrySize(len(value)) + if p.cur.size+size > PermFileBytes { + if p.cur, err = p.newDataFile(); err != nil { + return nil, false, err + } + } + rec := permRecord{key: key, file: p.cur.id, off: uint32(p.cur.size), n: uint32(len(value))} + if _, err = p.cur.f.WriteAt(encodeEntry(p.height, key, value), p.cur.size); err != nil { + return nil, false, err + } + p.cur.size += size + p.dirty[p.cur.id] = p.cur + p.live[key] = rec + return nil, false, nil +} + +// Put is PutIfAbsent for callers that only need the error; a key +// already held is not an error, the value stands. +func (p *PermStore) Put(key [32]byte, value []byte) error { + _, _, err := p.PutIfAbsent(key, value) + return err +} + +func (p *PermStore) readEntry(rec permRecord, key [32]byte) ([]byte, error) { + hf := p.files[rec.file] + if hf == nil { + return nil, fmt.Errorf("perm: data file %d not open", rec.file) + } + buf := make([]byte, heapHeader+int(rec.n)+heapTrailer) + if _, err := hf.f.ReadAt(buf, int64(rec.off)); err != nil { + return nil, err + } + return heapEntryValue(buf, key) +} + +// Get answers from the live map and the window; a key older than the +// window is absent here (spec 1.3), GetDeep reaches it. +func (p *PermStore) Get(key [32]byte) ([]byte, error) { + p.mu.RLock() + defer p.mu.RUnlock() + if p.closed { + return nil, errStoreClosed + } + p.lookups.Add(1) + if rec, ok := p.live[key]; ok { + p.windowHits.Add(1) + return p.readEntry(rec, key) + } + for i := len(p.window) - 1; i >= 0; i-- { + d := p.window[i] + rec, found, err := d.run.lookup(d.f, key) + if err != nil { + return nil, err + } + if found { + p.windowHits.Add(1) + return p.readEntry(rec, key) + } + } + return nil, errNotFound +} + +// GetDeep is Get, then the deltas not yet merged, the key's bucket +// newest run first, and the retired runs newest first. +func (p *PermStore) GetDeep(key [32]byte) ([]byte, error) { + v, err := p.Get(key) + if err == nil || !errors.Is(err, errNotFound) { + return v, err + } + p.mu.RLock() + defer p.mu.RUnlock() + for i := len(p.pending) - 1; i >= 0; i-- { + d := p.pending[i] + rec, found, err := d.run.lookup(d.f, key) + if err != nil { + return nil, err + } + if found { + p.deepHits.Add(1) + return p.readEntry(rec, key) + } + } + b := &p.buckets[key[0]] + for i := len(b.runs) - 1; i >= 0; i-- { + rec, found, err := b.runs[i].lookup(b.files[i].f, key) + if err != nil { + return nil, err + } + if found { + p.deepHits.Add(1) + return p.readEntry(rec, key) + } + } + for i := len(p.retired) - 1; i >= 0; i-- { + r := p.retired[i] + rf := p.runs[r.file] + rec, found, err := r.lookup(rf.f, key) + if err != nil { + return nil, err + } + if found { + p.deepHits.Add(1) + return p.readEntry(rec, key) + } + } + return nil, errNotFound +} + +// permSeal is a seal in flight: the data files to fsync, the delta +// to append. Holds syncMu until finished. +type permSeal struct { + p *PermStore + dirty []*heapFile + recs []permRecord + height uint64 + run *permRun // The delta as written, admitted once durable + hf *heapFile // The data file it was written to +} + +// beginSeal takes the block's records under the lock; finish makes +// them durable outside it. +func (p *PermStore) beginSeal(height uint64) (*permSeal, error) { + p.syncMu.Lock() + p.mu.Lock() + defer p.mu.Unlock() + if p.closed || p.files == nil { + p.syncMu.Unlock() + return nil, errStoreClosed + } + s := &permSeal{p: p, height: height} + for _, hf := range p.dirty { + s.dirty = append(s.dirty, hf) + } + p.dirty = map[uint32]*heapFile{} + s.recs = make([]permRecord, 0, len(p.live)) + for _, rec := range p.live { + s.recs = append(s.recs, rec) + } + p.live = map[[32]byte]permRecord{} // A put from here on is the next block's + sortPermRecords(s.recs) + if len(s.recs) == 0 { + return s, nil + } + // The delta's run goes into the block's data file behind the + // block's entries, under the reserved key: one fsync of that file + // covers both, and replay trusts the last delta only if every + // entry it names checks (spec 1.7) + var w bufWriterAt + run, err := writePermRun(&w, 0, p.cur.id, s.recs, s.height) + if err != nil { + p.syncMu.Unlock() + return nil, err + } + entry := encodeEntry(s.height, heapDeltaKey, w.buf) + if p.cur.size+int64(len(entry)) > PermFileBytes { + if p.cur, err = p.newDataFile(); err != nil { + p.syncMu.Unlock() + return nil, err + } + } + hf, at := p.cur, p.cur.size + if _, err = hf.f.WriteAt(entry, at); err != nil { + p.syncMu.Unlock() + return nil, err + } + hf.size += int64(len(entry)) + run.file, run.off, run.bloomAt = hf.id, at+heapHeader, at+heapHeader+run.bloomAt + s.run, s.hf = run, hf + if _, dirty := p.dirty[hf.id]; !dirty { + for _, d := range s.dirty { + if d == hf { + dirty = true + } + } + if !dirty { + s.dirty = append(s.dirty, hf) + } + } + return s, nil +} + +// finish: the block's entries and its delta durable, one fsync per +// file touched, with the lock released; then the delta admitted to +// the window. Releases syncMu. +func (s *permSeal) finish() error { + p := s.p + defer p.syncMu.Unlock() + for _, hf := range s.dirty { + if err := fsync(hf.f); err != nil { + return err + } + } + p.mu.Lock() + defer p.mu.Unlock() + if s.run != nil { + p.indexBytes.Add(uint64(s.run.bytes)) + p.admit(&permDelta{height: s.height, run: s.run, f: s.hf.f, data: s.hf.id}) + } + if s.height >= p.height { + p.height = s.height + 1 + } + return nil +} + +// bufWriterAt collects what a run writer writes at offset 0. +type bufWriterAt struct{ buf []byte } + +func (w *bufWriterAt) WriteAt(b []byte, off int64) (int, error) { + if need := int(off) + len(b); need > len(w.buf) { + w.buf = append(w.buf, make([]byte, need-len(w.buf))...) + } + copy(w.buf[off:], b) + return len(b), nil +} + +// AdvanceBlock sets the block new writes belong to. +func (p *PermStore) AdvanceBlock(height uint64) { + p.mu.Lock() + p.height = height + p.mu.Unlock() +} + +// BlockHeight is the block being written. +func (p *PermStore) BlockHeight() uint64 { + p.mu.RLock() + defer p.mu.RUnlock() + return p.height +} + +// LiveCount is the records this block has written so far. +func (p *PermStore) LiveCount() int { + p.mu.RLock() + defer p.mu.RUnlock() + return len(p.live) +} + +// maintFile is the run file maintenance appends to, rolled at +// PermFileBytes. The caller holds the lock. +func (p *PermStore) maintFile() (*runFile, error) { + if p.maintRun == nil || p.maintRun.size > PermFileBytes { + rf, err := p.newRunFile() + if err != nil { + return nil, err + } + p.maintRun = rf + } + return p.maintRun, nil +} + +// Merge is the maintenance step: the buckets due in rotation take +// their records from the pending deltas, fold, and the manifest is +// committed; deltas every bucket has absorbed are released. Runs +// are written and fsynced outside the lock; the lock is held to +// choose and to swap. +func (p *PermStore) Merge() error { + p.mu.Lock() + if p.closed { + p.mu.Unlock() + return errStoreClosed + } + if len(p.pending) == 0 { + p.mu.Unlock() + return nil + } + // Which buckets: every bucket is due once per PermMergeEvery + // blocks, so a call that comes after n blocks takes n/PermMergeEvery + // of them, whatever the caller's cadence + elapsed := p.height - p.lastMerge + if p.lastMerge == 0 || elapsed > PermMergeEvery { + elapsed = PermMergeEvery + } + due := int(uint64(PermBuckets) * elapsed / PermMergeEvery) + if due < 1 { + due = 1 + } + if due > PermBuckets { + due = PermBuckets + } + p.lastMerge = p.height + newest := p.pending[len(p.pending)-1].height + type job struct{ b int } + var jobs []job + for i := 0; i < due; i++ { + jobs = append(jobs, job{b: (p.rotation + i) % PermBuckets}) + } + rotation := (p.rotation + due) % PermBuckets + // Read the deltas' records for these buckets outside the lock: + // runs are immutable. The due buckets are a run of the rotation, + // so their records are one or two slices of each sorted delta + oldest := ^uint64(0) + for _, j := range jobs { + if p.buckets[j.b].merged < oldest { + oldest = p.buckets[j.b].merged + } + } + var ranges [][2]byte + if last := p.rotation + due - 1; last < PermBuckets { + ranges = append(ranges, [2]byte{byte(p.rotation), byte(last)}) + } else { + ranges = append(ranges, [2]byte{byte(p.rotation), PermBuckets - 1}, [2]byte{0, byte(last - PermBuckets)}) + } + pendingCopy := append([]*permDelta(nil), p.pending...) + p.mu.Unlock() + byBucket := map[int][][]permRecord{} + for _, d := range pendingCopy { + if d.height <= oldest { + continue // Every due bucket has this delta already + } + var recs []permRecord + for _, rg := range ranges { + part, err := d.run.recordsBucketRange(d.f, rg[0], rg[1]) + if err != nil { + return err + } + recs = append(recs, part...) + } + for _, j := range jobs { + if d.height <= p.buckets[j.b].merged { + continue + } + var mine []permRecord + for _, r := range recs { + if int(r.key[0]) == j.b { + mine = append(mine, r) + } + } + byBucket[j.b] = append(byBucket[j.b], mine) + } + } + // Write each due bucket's new run to the maintenance run file + p.mu.Lock() + rf, err := p.maintFile() + p.mu.Unlock() + if err != nil { + return err + } + var written []struct { + b int + run *permRun + } + for _, j := range jobs { + recs := mergePermRuns(byBucket[j.b]) + if len(recs) == 0 { + continue + } + p.mu.Lock() + at := rf.size + rf.size += 0 + p.mu.Unlock() + run, err := writePermRun(rf.f, at, rf.id, recs, 0) + if err != nil { + return err + } + p.mu.Lock() + rf.size = at + int64(run.bytes) + p.mu.Unlock() + p.indexBytes.Add(uint64(run.bytes)) + p.mergeRuns.Add(1) + written = append(written, struct { + b int + run *permRun + }{b: j.b, run: run}) + } + // No barrier here: a bucket's run is named by the manifest alone, + // so it needs to be durable before the manifest commit and not + // before. Every merge call once ended with an fsync of its run + // file, which at a call per shard per block was 72-144 barriers + // a second at nine stores and doubled every seal's fsync. + // Swap: the new runs join their buckets; then fold what the ratio + // says, one bucket at a time + p.mu.Lock() + rf.unsynced = true + for _, w := range written { + bk := &p.buckets[w.b] + bk.runs = append(bk.runs, w.run) + bk.files = append(bk.files, rf) + rf.refs++ + } + for _, j := range jobs { + p.buckets[j.b].merged = newest + } + p.rotation = rotation + // Deltas every bucket has absorbed are released + minMerged := ^uint64(0) + for i := range p.buckets { + if p.buckets[i].merged < minMerged { + minMerged = p.buckets[i].merged + } + } + var keep []*permDelta + for _, d := range p.pending { + if d.height > minMerged { + keep = append(keep, d) // A delta in a data file costs nothing to drop: the file stays + } + } + p.pending = keep + folds := p.planFolds() + p.mu.Unlock() + // The folds' runs are written without a barrier too; the manifest + // commit syncs every run file written since the last + for _, f := range folds { + if _, err := p.fold(f); err != nil { + return err + } + } + // The manifest names the new runs. It is committed when a fold + // left run files to drop, and otherwise every PermManifestEvery + // merges: the runs are durable in their file already, and a + // merge a crash loses is done again from the pending deltas, so a + // commit per merge -- two barriers a shard -- bought nothing but + // a shorter replay. Open removes the run files a lost merge left + // unnamed. + p.mu.Lock() + defer p.mu.Unlock() + p.manifestDirty = true + // A commit is due by age, or when a fold left a run file that + // nothing references, since only a commit can drop it. A fold + // alone is not a reason: with a merge every block a bucket folds + // on nearly every call, and a commit each time was two barriers + // a shard a block. + droppable := false + for _, rf := range p.runs { + if rf.refs == 0 && rf != p.maintRun { + droppable = true + break + } + } + if !droppable && p.height-p.manifestAt < PermManifestBlocks { + return nil + } + if err := p.syncRunFiles(); err != nil { + return err + } + if err := p.commitManifest(); err != nil { + return err + } + return p.dropUnreferencedRunFiles() +} + +type permFold struct { + b int + at int // The runs from this index on fold into one + count int +} + +// planFolds chooses, per bucket, the suffix of runs the ratio says to +// fold. The caller holds the lock. +func (p *PermStore) planFolds() (folds []permFold) { + for b := range p.buckets { + runs := p.buckets[b].runs + if len(runs) < 2 { + continue + } + var behind uint32 + i := len(runs) - 1 + for ; i >= 0; i-- { + if i < len(runs)-1 && float64(runs[i].count)*PermFoldRatio > float64(behind) { + break + } + behind += runs[i].count + } + if n := len(runs) - (i + 1); n >= 2 { + folds = append(folds, permFold{b: b, at: i + 1, count: n}) + } + } + return folds +} + +// fold merges a bucket's chosen runs into one, written to the +// maintenance run file (returned, for the caller to sync), and swaps +// it in under the lock. +func (p *PermStore) fold(f permFold) (*runFile, error) { + p.mu.RLock() + bk := &p.buckets[f.b] + if f.at+f.count > len(bk.runs) { + p.mu.RUnlock() + return nil, nil // The bucket changed under us; next time + } + runs := append([]*permRun(nil), bk.runs[f.at:f.at+f.count]...) + files := append([]*runFile(nil), bk.files[f.at:f.at+f.count]...) + p.mu.RUnlock() + inputs := make([][]permRecord, len(runs)) + for i, r := range runs { + recs, err := r.records(files[i].f) + if err != nil { + return nil, err + } + inputs[i] = recs + } + merged := mergePermRuns(inputs) + p.mu.Lock() + rf, err := p.maintFile() + if err != nil { + p.mu.Unlock() + return nil, err + } + at := rf.size + rf.size += 0 + p.mu.Unlock() + run, err := writePermRun(rf.f, at, rf.id, merged, 0) + if err != nil { + return nil, err + } + p.indexBytes.Add(uint64(run.bytes)) + p.foldRuns.Add(1) + p.mu.Lock() + defer p.mu.Unlock() + rf.size = at + int64(run.bytes) + rf.unsynced = true + bk = &p.buckets[f.b] + if f.at+f.count > len(bk.runs) { + return rf, nil + } + for _, old := range files { + old.refs-- + } + rest := append([]*permRun(nil), bk.runs[f.at+f.count:]...) + restF := append([]*runFile(nil), bk.files[f.at+f.count:]...) + bk.runs = append(append(bk.runs[:f.at], run), rest...) + bk.files = append(append(bk.files[:f.at], rf), restF...) + rf.refs++ + return rf, nil +} + +// Pack retires the history above the watermark: every bucket's runs +// and every pending delta fold into one sorted run, and the buckets +// start again. The retired run is the deep history's newest. +func (p *PermStore) Pack() error { + p.mu.RLock() + var inputs [][]permRecord + var release []*runFile + for _, d := range p.pending { + recs, err := d.run.records(d.f) + if err != nil { + p.mu.RUnlock() + return err + } + inputs = append(inputs, recs) + } + for b := range p.buckets { + for i, r := range p.buckets[b].runs { + recs, err := r.records(p.buckets[b].files[i].f) + if err != nil { + p.mu.RUnlock() + return err + } + inputs = append(inputs, recs) + release = append(release, p.buckets[b].files[i]) + } + } + height := p.height + p.mu.RUnlock() + if len(inputs) == 0 { + return nil + } + merged := mergePermRuns(inputs) + p.mu.Lock() + rf, err := p.maintFile() + if err != nil { + p.mu.Unlock() + return err + } + at := rf.size + p.mu.Unlock() + run, err := writePermRun(rf.f, at, rf.id, merged, height) + if err != nil { + return err + } + if err = fsync(rf.f); err != nil { + return err + } + p.indexBytes.Add(uint64(run.bytes)) + p.packRuns.Add(1) + p.mu.Lock() + defer p.mu.Unlock() + rf.size = at + int64(run.bytes) + run.bloom = nil // Retired runs are probed cold + p.retired = append(p.retired, run) + rf.refs++ + for _, r := range release { + r.refs-- + } + p.pending = nil + for b := range p.buckets { + p.buckets[b] = permBucket{merged: height} + } + if err := p.syncRunFiles(); err != nil { + return err + } + if err := p.commitManifest(); err != nil { + return err + } + return p.dropUnreferencedRunFiles() +} + +// syncRunFiles makes every run file written since its last fsync +// durable, with the lock released for the barriers. Maintenance is +// one pass at a time, so nothing writes a run file meanwhile. The +// caller holds the lock and gets it back. +func (p *PermStore) syncRunFiles() error { + var dirty []*runFile + for _, rf := range p.runs { + if rf.unsynced { + dirty = append(dirty, rf) + } + } + if len(dirty) == 0 { + return nil + } + p.mu.Unlock() + var err error + for _, rf := range dirty { + if err = fsync(rf.f); err != nil { + break + } + } + p.mu.Lock() + if err != nil { + return err + } + for _, rf := range dirty { + rf.unsynced = false + } + return nil +} + +// commitManifest writes perm.json: encoded under the lock, written, +// fsynced and renamed with the lock released, so that a merge's +// manifest commit -- every twenty blocks, per shard -- is not a +// barrier the shard's puts and seal wait behind (spec 1.6). A delta +// the seal appends meanwhile lies after the offset the manifest +// records, and open replays it. The caller holds the lock and gets +// it back. +func (p *PermStore) commitManifest() error { + buf, err := p.encodeManifest() + if err != nil { + return err + } + p.mu.Unlock() + err = p.writeManifest(buf) + p.mu.Lock() + if err == nil { + p.manifestAt, p.manifestDirty = p.height, false + } + return err +} + +// encodeManifest is the manifest as of now. The caller holds the lock. +func (p *PermStore) encodeManifest() ([]byte, error) { + m := permManifest{Version: 1, Height: p.height, FilterBlocks: p.window_n, Rotation: p.rotation, NextData: p.nextID, NextRun: p.nextRun} + ref := func(r *permRun, rf *runFile, height uint64) permRunRef { + return permRunRef{File: rf.id, Off: r.off, Height: height} + } + for _, d := range p.window { + m.Window = append(m.Window, permRunRef{File: d.data, Off: d.run.off, Height: d.height}) + } + for _, d := range p.pending { + m.Pending = append(m.Pending, permRunRef{File: d.data, Off: d.run.off, Height: d.height}) + } + for b := range p.buckets { + bm := permBucketM{Merged: p.buckets[b].merged} + for i, r := range p.buckets[b].runs { + bm.Runs = append(bm.Runs, ref(r, p.buckets[b].files[i], 0)) + } + m.Buckets = append(m.Buckets, bm) + } + for _, r := range p.retired { + m.Retired = append(m.Retired, permRunRef{File: r.file, Off: r.off, Height: r.height}) + } + for id := range p.files { + m.DataFiles = append(m.DataFiles, id) + } + // Only the run files kept: a file nothing references is dropped + // once this manifest is durable, and a durable manifest must never + // name a file that is gone (1.7, 1.8) + for id, rf := range p.runs { + if rf.refs > 0 || rf == p.maintRun { + m.RunFiles = append(m.RunFiles, id) + } + } + sort.Slice(m.DataFiles, func(i, j int) bool { return m.DataFiles[i] < m.DataFiles[j] }) + sort.Slice(m.RunFiles, func(i, j int) bool { return m.RunFiles[i] < m.RunFiles[j] }) + // Deltas sealed after this point lie in the data files from the + // current one's end; open replays from there + m.DeltasFile, m.DeltasOff = p.cur.id, p.cur.size + p.deltasFrom.file, p.deltasFrom.off = m.DeltasFile, m.DeltasOff + return json.MarshalIndent(m, "", " ") +} + +// writeManifest writes the encoded manifest aside, fsyncs it, renames +// it into place and fsyncs the directory. No lock is held. +func (p *PermStore) writeManifest(buf []byte) error { + path := filepath.Join(p.Directory, "perm.json") + tmp := path + segTmpSuffix + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + if _, err = f.Write(buf); err != nil { + f.Close() + return err + } + if err = fsync(f); err != nil { + f.Close() + return err + } + if err = f.Close(); err != nil { + return err + } + if err = os.Rename(tmp, path); err != nil { + return err + } + return fsyncDir(p.Directory) +} + +// dropUnreferencedRunFiles deletes run files no run references, after +// the manifest that no longer names them is durable. The caller +// holds the lock. +func (p *PermStore) dropUnreferencedRunFiles() error { + for id, rf := range p.runs { + if rf.refs > 0 || rf == p.maintRun { + continue + } + rf.f.Close() + delete(p.runs, id) + if err := os.Remove(filepath.Join(p.Directory, rf.name())); err != nil { + return err + } + } + return nil +} + +// Counters reports the maintenance so far: merge runs, folds, packs, +// and the index bytes written. +func (p *PermStore) Counters() (merges, folds, packs, indexBytes uint64) { + return p.mergeRuns.Load(), p.foldRuns.Load(), p.packRuns.Load(), p.indexBytes.Load() +} + +// Stats is the store's report. +func (p *PermStore) Stats() StoreStats { + p.mu.RLock() + defer p.mu.RUnlock() + var runs int + for b := range p.buckets { + runs += len(p.buckets[b].runs) + } + return StoreStats{ + PutTotal: p.putTotal.Load(), + PutNew: p.putTotal.Load() - p.putDuplicate.Load(), + PutDuplicate: p.putDuplicate.Load(), + LookupTotal: p.lookups.Load(), + LiveHit: p.windowHits.Load(), + ActiveSegments: len(p.window), + HistorySegments: runs + len(p.retired), + HeapFiles: len(p.files) + len(p.runs), + HeapMovedBytes: p.indexBytes.Load(), + } +} + +// beginPermSeal and mergeBelow are the permLayer surface (kv_2.go). +func (p *PermStore) beginPermSeal(height uint64) (blockSync, error) { + s, err := p.beginSeal(height) + if err != nil { + return nil, err + } + return s, nil +} + +func (p *PermStore) mergeBelow(uint64) (bool, error) { return true, p.Merge() } + +// SetFilterBlocks sets the window. +func (p *PermStore) SetFilterBlocks(n uint64) error { + if n < MinFilterBlocks { + return fmt.Errorf("perm: filter blocks %d below the minimum %d", n, MinFilterBlocks) + } + p.mu.Lock() + p.window_n = n + p.mu.Unlock() + return nil +} diff --git a/database/perm_roll_test.go b/database/perm_roll_test.go new file mode 100644 index 0000000..832ee06 --- /dev/null +++ b/database/perm_roll_test.go @@ -0,0 +1,116 @@ +package blockchainDB + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// A data file rolled by a seal between two manifest commits is still +// replayed on reopen: every block sealed before Close is found, and +// the store keeps writing after (spec 1.7: what a seal made durable +// stays durable). +func TestPermStoreReopenAcrossRolledFiles(t *testing.T) { + size := PermFileBytes + PermFileBytes = 64 << 10 + defer func() { PermFileBytes = size }() + dir := filepath.Join(t.TempDir(), "perm") + p, err := NewPermStore(dir, MinFilterBlocks) + require.NoError(t, err) + fr := NewFastRandom([]byte{7}) + var keys [][32]byte + value := make([]byte, 1000) + for b := uint64(1); b <= 8; b++ { // 8 blocks x 40 KB: several rolls, no merge + p.AdvanceBlock(b) + for i := 0; i < 40; i++ { + k := fr.NextHash() + keys = append(keys, k) + value[0] = byte(b) + require.NoError(t, p.Put(k, value)) + } + sealPerm(t, p, b) + } + require.Greater(t, len(p.files), 1, "the test must roll a data file") + require.NoError(t, p.Close()) + + p, err = OpenPermStore(dir) + require.NoError(t, err) + for i, k := range keys { + v, err := p.GetDeep(k) + require.NoError(t, err, "key %d after reopen", i) + require.Equal(t, byte(i/40+1), v[0]) + } + require.Equal(t, uint64(9), p.BlockHeight()) + // And the store keeps going: a new block, sealed, merged, reopened + p.AdvanceBlock(9) + k := fr.NextHash() + require.NoError(t, p.Put(k, value)) + sealPerm(t, p, 9) + require.NoError(t, p.Merge()) + require.NoError(t, p.Close()) + p, err = OpenPermStore(dir) + require.NoError(t, err) + _, err = p.GetDeep(k) + require.NoError(t, err) + _, err = p.GetDeep(keys[0]) + require.NoError(t, err) + require.NoError(t, p.Close()) +} + +// Merges between manifest commits survive a crash: the store reopened +// from the stale manifest finds every key, does the lost merges again +// from the pending deltas, and creates run files without colliding +// with the ones the lost merges left behind. +func TestPermStoreReopenWithoutManifestCommit(t *testing.T) { + every, size := PermMergeEvery, PermFileBytes + PermMergeEvery, PermFileBytes = 4, 64<<10 + defer func() { PermMergeEvery, PermFileBytes = every, size }() + dir := filepath.Join(t.TempDir(), "perm") + p, err := NewPermStore(dir, MinFilterBlocks) + require.NoError(t, err) + fr := NewFastRandom([]byte{9}) + var keys [][32]byte + value := make([]byte, 500) + for b := uint64(1); b <= 3*MinFilterBlocks; b++ { + p.AdvanceBlock(b) + for i := 0; i < 50; i++ { + k := fr.NextHash() + keys = append(keys, k) + require.NoError(t, p.Put(k, value)) + } + sealPerm(t, p, b) + if b%3 == 0 { + require.NoError(t, p.Merge()) + } + } + // Keep going until a merge is left uncommitted (a fold commits) + last := uint64(3 * MinFilterBlocks) + for !p.manifestDirty { + last++ + require.Less(t, last, uint64(6*MinFilterBlocks), "a merge without a fold must come") + p.AdvanceBlock(last) + k := fr.NextHash() + keys = append(keys, k) + require.NoError(t, p.Put(k, value)) + sealPerm(t, p, last) + require.NoError(t, p.Merge()) + } + // No Close: the manifest on disk is the last commit's + q, err := OpenPermStore(dir) + require.NoError(t, err) + for i, k := range keys { + _, err := q.GetDeep(k) + require.NoError(t, err, "key %d after reopen", i) + } + q.AdvanceBlock(last + 1) + require.NoError(t, q.Put(fr.NextHash(), value)) + sealPerm(t, q, last+1) + require.NoError(t, q.Merge(), "a new run file must not collide with a lost merge's") + require.NoError(t, q.Close()) + q, err = OpenPermStore(dir) + require.NoError(t, err) + _, err = q.GetDeep(keys[0]) + require.NoError(t, err) + require.NoError(t, q.Close()) +} diff --git a/database/perm_sizing_sim_test.go b/database/perm_sizing_sim_test.go new file mode 100644 index 0000000..b8d99b5 --- /dev/null +++ b/database/perm_sizing_sim_test.go @@ -0,0 +1,278 @@ +package blockchainDB + +import ( + "flag" + "fmt" + "sort" + "strings" + "testing" +) + +var simFlag = flag.Bool("sim", false, "run the permanent-index sizing simulation (TestPermSizingSim)") + +// A sizing simulation for the permanent layer's bucketed long search +// (docs/proposals/2026-09-16-entries-written-once.md, step 2). Not a +// test of the store: it models the index maintenance the proposal +// describes and prints, for a sweep of bucket counts, what a shard +// pays per block and what a lookup walks. Run with +// +// go test ./database/ -run TestPermSizingSim -v -sim +// +// The model, per shard: every block appends K records and seals a +// delta of K sorted 44-byte index records with a filter. A delta +// leaving the window (N blocks) feeds each record to its bucket's +// resident recent section. Every block the next bucket in rotation +// writes its recent section as a new sorted run and folds its runs by +// the store's ratio rule (compactionRunWithin: a suffix of runs is +// folded while each older run is no larger than 1/ratio of what has +// gathered behind it). Today's cost, for comparison: every +// MergeEvery blocks the shard folds its finished segments by copying +// bodies (~V bytes a record) and rebuilding the index, under the same +// ratio rule over merged segments; every PackEvery blocks the merged +// segments are copied again into a set. +func TestPermSizingSim(t *testing.T) { + if !*simFlag { + t.Skip("a sizing simulation; run with -sim") + } + const ( + K = 1040 // Permanent records per block per shard (soak: 8.3k per store, 8 shards) + V = 300 // Bytes a record's body costs to copy + rec = 44 // Bytes an index record costs + N = 20 // The window, blocks + ratio = 0.25 + mergeEvery = 20 + packEvery = 1000 + bloomBits = 12 // Bits per key in a filter + ) + horizons := []int{86_400, 7 * 86_400, 30 * 86_400} + buckets := []int{1, 64, 256, 1024, 4096} + splits := []int64{32 << 20, 64 << 20} // Bucket index bytes at which a bucket splits in two + const ( + M = 256 // Every bucket is merged every M blocks: B/M buckets a block + residentBudget = 64 << 20 // Filter bytes resident per shard, newest runs first; the rest are probed cold + runFile = 64 << 20 // Runs are appended to run files of this size; a run is (file, offset, length) + ) + + var out strings.Builder + fmt.Fprintf(&out, "\nPermanent index, one shard, K=%d records/block, ratio %.2f, window %d blocks\n", K, ratio, N) + for _, blocks := range horizons { + fmt.Fprintf(&out, "\n== %d blocks (%.0f days), %d M records ==\n", blocks, float64(blocks)/86_400, K*blocks/1_000_000) + // Today + today := simToday(blocks, K, V, rec, ratio, mergeEvery, packEvery) + fmt.Fprintf(&out, "today: index+body bytes written %.1f GB (%.1f MB/block avg), largest pass %.0f MB, merged segments at end %d, sets %d\n", + float64(today.written)/1e9, float64(today.written)/float64(blocks)/1e6, float64(today.largest)/1e6, today.segments, today.sets) + fmt.Fprintf(&out, "%-22s %12s %11s %11s %11s %9s %11s %11s %9s\n", "buckets", "idx KB/blk", "largest MB", "runs/bkt", "probes", "files", "resident MB", "cold probes", "recent MB") + for _, b := range buckets { + r := simBuckets(blocks, K, rec, N, ratio, b, 0, M, bloomBits, residentBudget, runFile) + fmt.Fprintf(&out, "%-22d %12.0f %11.1f %11.1f %11.1f %9d %11.1f %11.1f %9.1f\n", + b, float64(r.written)/float64(blocks)/1e3, float64(r.largest)/1e6, r.runsAvg, r.probes, r.files, float64(r.resident)/1e6, r.coldProbes, float64(r.recent)/1e6) + } + for _, sp := range splits { + for _, rt := range []float64{ratio, 0.1} { + r := simBuckets(blocks, K, rec, N, rt, 256, sp, M, bloomBits, residentBudget, runFile) + fmt.Fprintf(&out, "%-22s %12.0f %11.1f %11.1f %11.1f %9d %11.1f %11.1f %9.1f -> %d buckets\n", + fmt.Sprintf("256 split@%dMB r=%.2f", sp>>20, rt), float64(r.written)/float64(blocks)/1e3, float64(r.largest)/1e6, r.runsAvg, r.probes, r.files, float64(r.resident)/1e6, r.coldProbes, float64(r.recent)/1e6, r.buckets) + } + } + } + fmt.Fprintf(&out, "\nidx KB/blk: index bytes written per block, runs and folds; largest: the biggest single fold (or split);\n") + fmt.Fprintf(&out, "runs/bkt: sorted runs a lookup below the window probes (one filter each); probes: for a key not in the\n") + fmt.Fprintf(&out, "shard, window filter + recent section + runs, of which cold probes are on disk (K byte reads each);\n") + fmt.Fprintf(&out, "files: run files of %d MB; resident: filters within a %d MB budget, newest runs first; recent: the\n", runFile>>20, residentBudget>>20) + fmt.Fprintf(&out, "recent sections in memory (every bucket merged every %d blocks); split@S: a bucket whose runs exceed S\n", M) + fmt.Fprintf(&out, "splits in two; r: the fold ratio (a run folds into the older while the older is no larger than 1/r of it).\n") + t.Log(out.String()) +} + +type simResult struct { + written, largest int64 + runsAvg, probes float64 + coldProbes float64 + files, buckets int + resident, recent int64 +} + +// simBuckets models the bucketed long search. split > 0 splits a +// bucket in two once its runs hold more than split bytes (each half +// keeps half of every run: the keys are hashed, so a split is a +// rewrite of the bucket, counted as a fold). B/M buckets are merged +// each block so every bucket is merged every M blocks. +func simBuckets(blocks, K, rec, N int, ratio float64, B int, split int64, M, bloomBits int, residentBudget, runFile int64) simResult { + type bucket struct { + recent int64 // Records in the resident recent section + runs []int64 // Records per run, oldest first + } + bs := make([]bucket, B) + var r simResult + rotation := 0 + fold := func(b *bucket) { + if len(b.runs) < 2 { + return + } + var behind int64 + i := len(b.runs) - 1 + for ; i >= 0; i-- { + if i < len(b.runs)-1 && float64(b.runs[i])*ratio > float64(behind) { + break + } + behind += b.runs[i] + } + run := b.runs[i+1:] + if len(run) >= 2 { + var total int64 + for _, n := range run { + total += n + } + r.written += total * int64(rec) + if total*int64(rec) > r.largest { + r.largest = total * int64(rec) + } + b.runs = append(b.runs[:i+1], total) + } + } + for blk := 0; blk < blocks; blk++ { + if blk >= N { + perBucket := int64(K) / int64(len(bs)) + extra := int64(K) - perBucket*int64(len(bs)) + for i := range bs { + bs[i].recent += perBucket + } + for i := int64(0); i < extra; i++ { + bs[(blk*7+int(i))%len(bs)].recent++ + } + } + visits := (len(bs) + M - 1) / M + for v := 0; v < visits; v++ { + b := &bs[rotation%len(bs)] + rotation++ + if split > 0 { + var total int64 + for _, n := range b.runs { + total += n + } + if total*int64(rec) > split { + r.written += total * int64(rec) + if total*int64(rec) > r.largest { + r.largest = total * int64(rec) + } + half := bucket{recent: b.recent / 2} + b.recent -= half.recent + for i, n := range b.runs { + b.runs[i] = n / 2 + half.runs = append(half.runs, n-n/2) + } + bs = append(bs, half) + b = &bs[(rotation-1)%len(bs)] + } + } + if b.recent == 0 { + continue + } + r.written += b.recent * int64(rec) + b.runs = append(b.runs, b.recent) + b.recent = 0 + fold(b) + } + } + // Filters: newest runs first across the shard, within the budget + var runs, cold int + type runRef struct{ n int64 } + var newest, older []int64 // Per bucket: the newest run's records, then the rest + for _, b := range bs { + runs += len(b.runs) + r.recent += b.recent * int64(rec) + for i, n := range b.runs { + if i == len(b.runs)-1 { + newest = append(newest, n) + } else { + older = append(older, n) + } + } + } + budget := residentBudget + for _, n := range append(newest, older...) { + bytes := n * int64(bloomBits) / 8 + if budget >= bytes { + budget -= bytes + r.resident += bytes + } else { + cold++ + } + } + r.resident += r.recent + var indexBytes int64 + for _, b := range bs { + for _, n := range b.runs { + indexBytes += n * int64(rec) + } + } + r.files = int((indexBytes + runFile - 1) / runFile) + r.buckets = len(bs) + r.runsAvg = float64(runs) / float64(len(bs)) + r.coldProbes = float64(cold) / float64(len(bs)) + r.probes = 2 + r.runsAvg + _ = runRef{} + return r +} + +type simTodayResult struct { + written, largest int64 + segments, sets int +} + +// simToday models the store as it is: segments folded by copying +// bodies every mergeEvery blocks under the ratio rule, and packed +// every packEvery blocks by copying again. +func simToday(blocks, K, V, rec int, ratio float64, mergeEvery, packEvery int) simTodayResult { + perRecord := int64(V + rec) + var segs []int64 // Records per merged segment, oldest first + var r simTodayResult + pending := 0 // Finished blocks not yet merged + for blk := 1; blk <= blocks; blk++ { + pending++ + if blk%mergeEvery == 0 { + // The finished blocks become one merged segment, copied + n := int64(pending * K) + r.written += n * perRecord + segs = append(segs, n) + pending = 0 + var behind int64 + i := len(segs) - 1 + for ; i >= 0; i-- { + if i < len(segs)-1 && float64(segs[i])*ratio > float64(behind) { + break + } + behind += segs[i] + } + run := segs[i+1:] + if len(run) >= 2 { + var total int64 + for _, n := range run { + total += n + } + r.written += total * perRecord + if total*perRecord > r.largest { + r.largest = total * perRecord + } + segs = append(segs[:i+1], total) + } + } + if blk%packEvery == 0 { + // Everything merged so far below the watermark is copied + // into a set and dropped from the shard + var total int64 + for _, n := range segs { + total += n + } + r.written += total * perRecord + if total*perRecord > r.largest { + r.largest = total * perRecord + } + segs = segs[:0] + r.sets++ + } + } + r.segments = len(segs) + sort.Slice(segs, func(i, j int) bool { return segs[i] < segs[j] }) + return r +} diff --git a/database/perm_test.go b/database/perm_test.go new file mode 100644 index 0000000..bf409f4 --- /dev/null +++ b/database/perm_test.go @@ -0,0 +1,192 @@ +package blockchainDB + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func sealPerm(t *testing.T, p *PermStore, height uint64) { + t.Helper() + s, err := p.beginSeal(height) + require.NoError(t, err) + require.NoError(t, s.finish()) +} + +// Records are written once and found in the window; past the window +// Get says absent and GetDeep finds them through the buckets and, +// after a pack, the retired run; the manifest and the deltas replay +// on reopen; a torn delta is dropped whole. +func TestPermStoreTiersAndReopen(t *testing.T) { + every := PermMergeEvery + PermMergeEvery = 4 // 64 buckets a merge + defer func() { PermMergeEvery = every }() + dir := filepath.Join(t.TempDir(), "perm") + p, err := NewPermStore(dir, MinFilterBlocks) + require.NoError(t, err) + fr := NewFastRandom([]byte{21}) + const perBlock = 300 + var keys [][32]byte + value := func(b uint64, i int) []byte { return []byte{byte(b >> 8), byte(b), byte(i >> 8), byte(i), 'v'} } + for b := uint64(1); b <= 3*MinFilterBlocks; b++ { + p.AdvanceBlock(b) + for i := 0; i < perBlock; i++ { + k := fr.NextHash() + keys = append(keys, k) + existing, existed, err := p.PutIfAbsent(k, value(b, i)) + require.NoError(t, err) + require.False(t, existed) + require.Nil(t, existing) + } + // A key written again in its block is refused with its value + existing, existed, err := p.PutIfAbsent(keys[len(keys)-1], []byte("other")) + require.NoError(t, err) + require.True(t, existed) + require.Equal(t, value(b, perBlock-1), existing) + sealPerm(t, p, b) + if b%5 == 0 { + require.NoError(t, p.Merge()) + } + } + // The window holds the last MinFilterBlocks blocks; older keys are + // absent to Get and present to GetDeep + inWindow := keys[2*MinFilterBlocks*perBlock:] + older := keys[:2*MinFilterBlocks*perBlock] + for i, k := range inWindow[:perBlock] { + v, err := p.Get(k) + require.NoError(t, err) + require.Equal(t, value(2*MinFilterBlocks+1, i), v) + } + for _, k := range older[:100] { + _, err := p.Get(k) + require.ErrorIs(t, err, errNotFound, "past the window, absent on the protocol path") + _, err = p.GetDeep(k) + require.NoError(t, err, "and found deep") + } + _, err = p.Get(fr.NextHash()) + require.ErrorIs(t, err, errNotFound) + st := p.Stats() + require.Greater(t, st.HistorySegments, 0, "buckets hold runs") + + // A pack retires the buckets and the pending deltas into one run + require.NoError(t, p.Pack()) + require.Len(t, p.retired, 1) + for b := range p.buckets { + require.Empty(t, p.buckets[b].runs) + } + for _, k := range older[:200] { + _, err = p.GetDeep(k) + require.NoError(t, err, "found in the retired run") + } + + // More blocks, sealed but never in a manifest, then reopen + for b := uint64(3*MinFilterBlocks + 1); b <= 3*MinFilterBlocks+5; b++ { + p.AdvanceBlock(b) + for i := 0; i < perBlock; i++ { + k := fr.NextHash() + keys = append(keys, k) + require.NoError(t, p.Put(k, value(b, i))) + } + sealPerm(t, p, b) + } + // A torn entry after the last seal: the crash + _, err = p.cur.f.WriteAt([]byte{9, 0, 0, 0, 1, 2, 3, 4, 5, 6}, p.cur.size) + require.NoError(t, err) + last := keys[len(keys)-perBlock:] + for _, hf := range p.files { + hf.f.Close() + } + for _, rf := range p.runs { + rf.f.Close() + } + r, err := OpenPermStore(dir) + require.NoError(t, err) + defer r.Close() + require.EqualValues(t, 3*MinFilterBlocks+6, r.height, "the replayed deltas leave the height at the next block, as the seal does") + for i, k := range last { + v, err := r.Get(k) + require.NoError(t, err, "a delta replayed after the manifest") + require.Equal(t, value(3*MinFilterBlocks+5, i), v) + } + for _, k := range older[:100] { + _, err = r.GetDeep(k) + require.NoError(t, err, "the retired run came back through the manifest") + } + _, err = r.Get(fr.NextHash()) + require.ErrorIs(t, err, errNotFound) +} + +// A shard built with both layers as files seals, merges, packs, +// closes and reopens as files, through the same KV2 and KVShard +// surface; permanent keys are found in the window and, once past it, +// by GetDeep; dynamic keys keep their last value. +func TestFilesShardRoundTrip(t *testing.T) { + every := PermMergeEvery + PermMergeEvery = 8 + defer func() { PermMergeEvery = every }() + dir := filepath.Join(t.TempDir(), "shards") + kvs, err := NewKVShardFilesN(dir, 2, 1000) + require.NoError(t, err) + require.NoError(t, kvs.SetFilterBlocks(MinFilterBlocks)) + fr := NewFastRandom([]byte{9}) + hot := make([][32]byte, 100) + for i := range hot { + hot[i] = fr.NextHash() + } + var perm [][32]byte + for b := uint64(1); b <= 3*MinFilterBlocks; b++ { + for _, k := range hot { + require.NoError(t, kvs.PutDyna(k, append([]byte{byte(b)}, k[:4]...))) + } + for i := 0; i < 50; i++ { + k := fr.NextHash() + perm = append(perm, k) + require.NoError(t, kvs.PutPerm(k, append([]byte{byte(b)}, k[:4]...))) + } + require.NoError(t, kvs.SealBlock(b)) + if b%10 == 0 { + require.NoError(t, kvs.Compress()) + _, err := kvs.MergeFinalized(b) + require.NoError(t, err) + } + } + for _, k := range hot { + v, err := kvs.GetDyna(k) + require.NoError(t, err) + require.Equal(t, byte(3*MinFilterBlocks), v[0]) + } + recent := perm[len(perm)-50:] + old := perm[:50] + for _, k := range recent { + _, err := kvs.GetPerm(k) + require.NoError(t, err, "in the window") + } + for _, k := range old { + _, err := kvs.GetPerm(k) + require.ErrorIs(t, err, errNotFound, "past the window") + v, err := kvs.Shards[kvs.ShardIndex(k[:])].GetPermDeep(k) + require.NoError(t, err, "found deep") + require.Equal(t, byte(1), v[0]) + } + _, packed, err := kvs.PackFinalized(3 * MinFilterBlocks) + require.NoError(t, err) + require.True(t, packed) + require.NoError(t, kvs.Close()) + + re, err := OpenKVShard(dir) + require.NoError(t, err) + defer re.Close() + require.NotNil(t, re.Shards[0].Perm, "reopened as files") + require.NotNil(t, re.Shards[0].Heap) + for _, k := range hot { + v, err := re.GetDyna(k) + require.NoError(t, err) + require.Equal(t, byte(3*MinFilterBlocks), v[0]) + } + for _, k := range old { + v, err := re.Shards[re.ShardIndex(k[:])].GetPermDeep(k) + require.NoError(t, err, "the retired run came back") + require.Equal(t, byte(1), v[0]) + } +} diff --git a/database/permindex.go b/database/permindex.go new file mode 100644 index 0000000..5157e09 --- /dev/null +++ b/database/permindex.go @@ -0,0 +1,346 @@ +package blockchainDB + +import ( + "bytes" + "container/heap" + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io" + "os" + "sort" +) + +// The permanent layer's index primitives (proposal +// docs/proposals/2026-09-16-entries-written-once.md, step 2): a +// record that names where an entry lives, sorted runs of records with +// a filter, a k-way merge over runs, and a lookup in a run. Nothing +// here touches an entry's bytes; entries are written once and only +// ever named. +// +// A run on disk: +// +// header magic "PRUN"(4) count(4) bloomBytes(4) bloomK(4) crc(4) height(8) +// records count sorted 44-byte records: key(32) file(4) off(4) len(4) +// bloom bloomBytes of filter over the keys +// +// Runs are immutable once written; a merge writes a new run from +// several and the inputs are dropped by whoever owns them. + +// permRecord names an entry: its key and where it is. +type permRecord struct { + key [32]byte + file uint32 + off uint32 + n uint32 +} + +const ( + permRecSize = 32 + 4 + 4 + 4 + permRunHdr = 4 + 4 + 4 + 4 + 4 + 8 + permRunMagic = 0x5052554E // "PRUN" + permBloomBits = 12 // Bits per key in a run's filter + permBloomK = 3 +) + +func (r permRecord) put(buf []byte) { + copy(buf, r.key[:]) + binary.LittleEndian.PutUint32(buf[32:], r.file) + binary.LittleEndian.PutUint32(buf[36:], r.off) + binary.LittleEndian.PutUint32(buf[40:], r.n) +} + +func getPermRecord(buf []byte) (r permRecord) { + copy(r.key[:], buf) + r.file = binary.LittleEndian.Uint32(buf[32:]) + r.off = binary.LittleEndian.Uint32(buf[36:]) + r.n = binary.LittleEndian.Uint32(buf[40:]) + return r +} + +// permRun is a sorted run of records with its filter, as held by a +// bucket: the file it lives in, where, and how many; the filter is +// resident when loaded and probed cold otherwise. +type permRun struct { + path string + file uint32 // The run file's id + off int64 // Where the run starts in its file + count uint32 + height uint64 // The block a delta is; the watermark a retired run is; 0 for a bucket's run + bloomAt int64 // Where the filter starts + bloom *Bloom + k int + bytes uint32 + // Every permFenceEvery-th key, for a resident run: a lookup finds + // its chunk in memory and reads that chunk once, where a binary + // search over the file was a pread a probe -- a dozen a run, and + // 40% of a block's CPU under the load platform's reads. + fence [][32]byte +} + +// permFenceEvery is the records between fence keys: a chunk is one +// read of this many records, and the fence costs 32 bytes per chunk. +const permFenceEvery = 32 + +// buildFence takes every permFenceEvery-th key from records laid out +// in buf. +func buildFence(buf []byte, count uint32) [][32]byte { + fence := make([][32]byte, 0, (int(count)+permFenceEvery-1)/permFenceEvery) + for i := 0; i < int(count); i += permFenceEvery { + var k [32]byte + copy(k[:], buf[i*permRecSize:]) + fence = append(fence, k) + } + return fence +} + +// writePermRun writes records, which must be sorted by key and free +// of duplicates, as a run appended to w, and returns it. The caller +// fsyncs the file. +func writePermRun(w io.WriterAt, at int64, file uint32, recs []permRecord, height uint64) (*permRun, error) { + if !sort.SliceIsSorted(recs, func(i, j int) bool { return bytes.Compare(recs[i].key[:], recs[j].key[:]) < 0 }) { + return nil, errors.New("perm run: records are not sorted") + } + bloom := NewBloomSizedForKeys(uint64(len(recs)), permBloomK) + buf := make([]byte, permRunHdr+len(recs)*permRecSize+int(bloom.NumBytes)) + binary.LittleEndian.PutUint32(buf, permRunMagic) + binary.LittleEndian.PutUint32(buf[4:], uint32(len(recs))) + binary.LittleEndian.PutUint32(buf[8:], uint32(bloom.NumBytes)) + binary.LittleEndian.PutUint32(buf[12:], uint32(bloom.K)) + binary.LittleEndian.PutUint64(buf[20:], height) + p := permRunHdr + for i, r := range recs { + if i > 0 && recs[i-1].key == r.key { + return nil, errors.New("perm run: duplicate key") + } + r.put(buf[p:]) + bloom.Set(r.key) + p += permRecSize + } + copy(buf[p:], bloom.Map) + binary.LittleEndian.PutUint32(buf[16:], crc32.ChecksumIEEE(buf[permRunHdr:])) + if _, err := w.WriteAt(buf, at); err != nil { + return nil, err + } + return &permRun{path: permRunName(file), file: file, off: at, count: uint32(len(recs)), height: height, bloomAt: at + int64(p), bloom: bloom, k: bloom.K, bytes: uint32(len(buf)), + fence: buildFence(buf[permRunHdr:], uint32(len(recs)))}, nil +} + +// openPermRun reads a run's header at off in f and verifies the run. +func openPermRun(f *os.File, file uint32, off int64, resident bool) (*permRun, error) { + path := permRunName(file) + hdr := make([]byte, permRunHdr) + if _, err := f.ReadAt(hdr, off); err != nil { + return nil, err + } + if binary.LittleEndian.Uint32(hdr) != permRunMagic { + return nil, fmt.Errorf("perm run at %s:%d: bad magic", path, off) + } + r := &permRun{path: path, file: file, off: off, count: binary.LittleEndian.Uint32(hdr[4:]), k: int(binary.LittleEndian.Uint32(hdr[12:])), height: binary.LittleEndian.Uint64(hdr[20:])} + bloomBytes := binary.LittleEndian.Uint32(hdr[8:]) + r.bloomAt = off + permRunHdr + int64(r.count)*permRecSize + r.bytes = permRunHdr + r.count*permRecSize + bloomBytes + body := make([]byte, r.bytes-permRunHdr) + if _, err := f.ReadAt(body, off+permRunHdr); err != nil { + return nil, err + } + if crc32.ChecksumIEEE(body) != binary.LittleEndian.Uint32(hdr[16:]) { + return nil, fmt.Errorf("perm run at %s:%d: checksum failed", path, off) + } + if resident { + // Rebuilt from the stored byte count: ByteMask indexes by NumBytes + r.bloom = &Bloom{NumBytes: uint64(bloomBytes), SizeOfMap: float64(bloomBytes) / (1 << 20), K: r.k, + Map: make([]byte, bloomBytes), Capacity: uint64(bloomBytes) * 8 / BloomBitsPerKey, Count: uint64(r.count)} + copy(r.bloom.Map, body[int64(r.count)*permRecSize:]) + r.fence = buildFence(body, r.count) + } + return r, nil +} + +// lookup finds key in the run: the filter first (resident, or cold in +// the file), then a binary search over the records. +func (r *permRun) lookup(f *os.File, key [32]byte) (rec permRecord, found bool, err error) { + if r.bloom != nil { + if !r.bloom.Test(key) { + return rec, false, nil + } + } else if ok, err := r.bloomTestCold(f, key); err != nil || !ok { + return rec, false, err + } + if r.fence != nil { + // The chunk whose first key is the greatest not above key + i := sort.Search(len(r.fence), func(i int) bool { return bytes.Compare(r.fence[i][:], key[:]) > 0 }) - 1 + if i < 0 { + return rec, false, nil + } + first := int64(i) * permFenceEvery + n := int64(permFenceEvery) + if first+n > int64(r.count) { + n = int64(r.count) - first + } + buf := make([]byte, n*permRecSize) + if _, err := f.ReadAt(buf, r.off+permRunHdr+first*permRecSize); err != nil { + return rec, false, err + } + for j := int64(0); j < n; j++ { + switch c := bytes.Compare(buf[j*permRecSize:j*permRecSize+32], key[:]); { + case c == 0: + return getPermRecord(buf[j*permRecSize:]), true, nil + case c > 0: + return rec, false, nil + } + } + return rec, false, nil + } + lo, hi := int64(0), int64(r.count) + buf := make([]byte, permRecSize) + for lo < hi { + mid := (lo + hi) / 2 + if _, err := f.ReadAt(buf, r.off+permRunHdr+mid*permRecSize); err != nil { + return rec, false, err + } + switch c := bytes.Compare(buf[:32], key[:]); { + case c == 0: + return getPermRecord(buf), true, nil + case c < 0: + lo = mid + 1 + default: + hi = mid + } + } + return rec, false, nil +} + +// bloomTestCold probes the run's filter in the file: k one-byte +// reads, the way the segment store probes a cold filter. +func (r *permRun) bloomTestCold(f *os.File, key [32]byte) (bool, error) { + bloomBytes := r.bytes - permRunHdr - r.count*permRecSize + probe := &Bloom{NumBytes: uint64(bloomBytes), SizeOfMap: float64(bloomBytes) * 8, K: r.k} + one := make([]byte, 1) + for i := 0; i < r.k; i++ { + idx, mask := probe.ByteMask(key, i) + if _, err := f.ReadAt(one, r.bloomAt+int64(idx)); err != nil { + return false, err + } + if one[0]&mask == 0 { + return false, nil + } + } + return true, nil +} + +// records reads a run's records in order, for a merge. +func (r *permRun) records(f *os.File) ([]permRecord, error) { + buf := make([]byte, int64(r.count)*permRecSize) + if _, err := f.ReadAt(buf, r.off+permRunHdr); err != nil { + return nil, err + } + recs := make([]permRecord, r.count) + for i := range recs { + recs[i] = getPermRecord(buf[i*permRecSize:]) + } + return recs, nil +} + +// recordsBucketRange reads the records whose first key byte lies in +// [lo, hi]: the run is sorted, so they are one slice, found by a +// binary search that probes a byte per step and read at once. A +// merge takes its due buckets' records from every pending delta this +// way rather than reading each delta whole -- with buckets due in +// rotation that was a read of every pending delta, up to +// PermMergeEvery blocks of them, at every pass. +func (r *permRun) recordsBucketRange(f *os.File, lo, hi byte) ([]permRecord, error) { + first := func(i int) (byte, error) { + var b [1]byte + _, err := f.ReadAt(b[:], r.off+permRunHdr+int64(i)*permRecSize) + return b[0], err + } + // The first record whose key's first byte is not below target + search := func(target int) (int, error) { + i, j := 0, int(r.count) + for i < j { + m := int(uint(i+j) >> 1) + b, err := first(m) + if err != nil { + return 0, err + } + if int(b) < target { + i = m + 1 + } else { + j = m + } + } + return i, nil + } + i, err := search(int(lo)) + if err != nil { + return nil, err + } + j, err := search(int(hi) + 1) + if err != nil { + return nil, err + } + if j <= i { + return nil, nil + } + buf := make([]byte, (j-i)*permRecSize) + if _, err := f.ReadAt(buf, r.off+permRunHdr+int64(i)*permRecSize); err != nil { + return nil, err + } + recs := make([]permRecord, j-i) + for k := range recs { + recs[k] = getPermRecord(buf[k*permRecSize:]) + } + return recs, nil +} + +// mergePermRuns merges sorted record lists, oldest first, into one +// sorted list; a key present in several takes the newest. Permanent +// keys are written once, so a duplicate is a replay or a fault, and +// newest-wins matches the store's rule everywhere else. +func mergePermRuns(inputs [][]permRecord) []permRecord { + h := &permMergeHeap{} + total := 0 + for src, in := range inputs { + total += len(in) + if len(in) > 0 { + heap.Push(h, permMergeCursor{src: src, recs: in}) + } + } + out := make([]permRecord, 0, total) + for h.Len() > 0 { + c := heap.Pop(h).(permMergeCursor) + r := c.recs[0] + if n := len(out); n == 0 || out[n-1].key != r.key { + out = append(out, r) + } + // The heap orders equal keys newest first, so the first copy + // out is the one kept and the rest are dropped here + if len(c.recs) > 1 { + heap.Push(h, permMergeCursor{src: c.src, recs: c.recs[1:]}) + } + } + return out +} + +type permMergeCursor struct { + src int + recs []permRecord +} + +type permMergeHeap []permMergeCursor + +func (h permMergeHeap) Len() int { return len(h) } +func (h permMergeHeap) Less(i, j int) bool { + if c := bytes.Compare(h[i].recs[0].key[:], h[j].recs[0].key[:]); c != 0 { + return c < 0 + } + return h[i].src > h[j].src // Newest first on a tie +} +func (h permMergeHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *permMergeHeap) Push(x any) { *h = append(*h, x.(permMergeCursor)) } +func (h *permMergeHeap) Pop() any { old := *h; x := old[len(old)-1]; *h = old[:len(old)-1]; return x } +func sortPermRecords(recs []permRecord) { + sort.Slice(recs, func(i, j int) bool { return bytes.Compare(recs[i].key[:], recs[j].key[:]) < 0 }) +} diff --git a/database/permindex_test.go b/database/permindex_test.go new file mode 100644 index 0000000..056f268 --- /dev/null +++ b/database/permindex_test.go @@ -0,0 +1,95 @@ +package blockchainDB + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// A run round-trips: written sorted with its filter, reopened with the +// checksum verified, looked up resident and cold, merged newest-wins. +func TestPermRunWriteLookupMerge(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, permRunName(7)) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644) + require.NoError(t, err) + defer f.Close() + fr := NewFastRandom([]byte{11}) + older := make([]permRecord, 5000) + for i := range older { + older[i] = permRecord{key: fr.NextHash(), file: 1, off: uint32(i * 100), n: 64} + } + sortPermRecords(older) + // The newer run rewrites a tenth of the older keys and adds new ones + newer := make([]permRecord, 0, 1000) + for i := 0; i < 500; i++ { + r := older[i*10] + r.file, r.off = 2, uint32(i) + newer = append(newer, r) + } + for i := 0; i < 500; i++ { + newer = append(newer, permRecord{key: fr.NextHash(), file: 2, off: uint32(1000 + i), n: 32}) + } + sortPermRecords(newer) + + r1, err := writePermRun(f, 0, 7, older, 1) + require.NoError(t, err) + r2, err := writePermRun(f, int64(r1.bytes), 7, newer, 2) + require.NoError(t, err) + _, err = writePermRun(f, int64(r1.bytes+r2.bytes), 7, []permRecord{older[3], older[2]}, 3) + require.Error(t, err, "unsorted records are refused") + + // Reopen: resident and cold + rr, err := openPermRun(f, 7, 0, true) + require.NoError(t, err) + require.EqualValues(t, 5000, rr.count) + require.EqualValues(t, 1, rr.height) + cold, err := openPermRun(f, 7, int64(r1.bytes), false) + require.NoError(t, err) + require.Nil(t, cold.bloom) + for _, rec := range older[:200] { + got, found, err := rr.lookup(f, rec.key) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, rec, got) + } + for _, rec := range newer[:200] { + got, found, err := cold.lookup(f, rec.key) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, rec, got) + } + misses := 0 + for i := 0; i < 2000; i++ { + _, found, err := rr.lookup(f, fr.NextHash()) + require.NoError(t, err) + if !found { + misses++ + } + } + require.Equal(t, 2000, misses, "an absent key is never found") + + // Merge: the newer copy of a rewritten key wins + recs1, err := rr.records(f) + require.NoError(t, err) + recs2, err := cold.records(f) + require.NoError(t, err) + merged := mergePermRuns([][]permRecord{recs1, recs2}) + require.Len(t, merged, 5500) + byKey := map[[32]byte]permRecord{} + for _, r := range merged { + byKey[r.key] = r + } + for _, r := range newer { + require.Equal(t, r, byKey[r.key]) + } + require.EqualValues(t, 1, byKey[older[1].key].file, "an unrewritten key keeps its older record") + + // A damaged run is refused + _, err = f.WriteAt([]byte{0xff, 0xff}, permRunHdr+40) + require.NoError(t, err) + _, err = openPermRun(f, 7, 0, true) + require.ErrorContains(t, err, "checksum") +} diff --git a/database/segstore.go b/database/segstore.go index 25c3175..b842c31 100644 --- a/database/segstore.go +++ b/database/segstore.go @@ -814,6 +814,16 @@ type StoreStats struct { HistorySegments int // Segments in history now ActiveSegments int // Segments in the window now ResidentBloomBytes uint64 // History filter memory held, of BloomResidentBytes + + // The heap's own figures (heap.go), zero for a segment store: the + // key map's resident bytes, its data files, what is live and dead + // in them, and what the mover has scanned and copied so far. + ResidentIndexBytes uint64 + HeapFiles int + HeapLiveBytes uint64 + HeapDeadBytes uint64 + HeapScannedBytes uint64 + HeapMovedBytes uint64 } // storeCounters is StoreStats as the store keeps it: atomics, because @@ -2712,6 +2722,40 @@ func compactionRunWithin(history []*segment, ratio float64, budget uint64) (run // the merge's rule: an uncommitted output sits below the newest active // segment and recoverOrphans deletes it, while the inputs are still // named. +// beginBlockSync and compact are the dynaLayer surface (kv_2.go) +// over Sync and CompactHistory. +func (s *SegmentStore) beginBlockSync() (blockSync, error) { + p, err := s.beginSync() + if err != nil { + return nil, err + } + return p, nil +} + +func (s *SegmentStore) compact(int64) (bool, error) { return s.CompactHistory() } + +// beginPermSeal and mergeBelow are the permLayer surface (kv_2.go) +// over beginSeal and MergeBelow. +func (s *SegmentStore) beginPermSeal(height uint64) (blockSync, error) { + p, err := s.beginSeal(height) + if err != nil { + return nil, err + } + if p == nil { + return nil, nil + } + return segSeal{p}, nil +} + +type segSeal struct{ p *pendingSeal } + +func (s segSeal) finish() error { _, err := s.p.finish(); return err } + +func (s *SegmentStore) mergeBelow(height uint64) (bool, error) { + _, merged, err := s.MergeBelow(height) + return merged, err +} + func (s *SegmentStore) CompactHistory() (compacted bool, err error) { s.maint.Lock() defer s.maint.Unlock() diff --git a/docs/SPEC.md b/docs/SPEC.md index 6a3559d..214cdaa 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -527,9 +527,29 @@ resident filter memory; and the count of reads that returned a value other than the last written, which fails the run. A minute whose seal p90 exceeds `-seal-budget` (100 ms) is flagged. +It checks answers, not only times: permanent values derive from their +keys, so every sampled permanent read is checked for presence and +content; checked hot keys must read as last written; and at the end +every store is closed, reopened and every sampled key of both layers +read back. A run that loses one fails (1.8). + What a healthy store must show on it: block time under the interval, the seal bounded and flat with age, put and read p99 flat with age, -and maintenance passes bounded per pass (1.2). A change to the +maintenance passes bounded per pass (1.2), zero wrong answers, and +every store reading back after reopen. + +The instrument has to be sound before a number is the store's. The +disk is shared by every store under test, and an SSD that is never +trimmed stalls every fsync on the box for tens of seconds at a time +under a sustained write load; that reads as a seal tail and is not +one. A run is read with the per-minute heap split and the two-second +store and device timelines (`cmd/bdbench/tools`), which tell a device +stall from a store tail, and a run on a disk that stalls is compared +by its clean minutes or not at all. A disk that never receives +discards will do this, so the measurement disk must be trimmed and +verified trimmed before a run counts; the operator's notes for the +machine a given run was made on are kept with that machine, not in +this repository. A change to the protocol path or to maintenance is measured here before it is measured under Accumulate, because here the seal's wait is the store's own and not the executor's. diff --git a/docs/proposals/2026-09-16-entries-written-once.md b/docs/proposals/2026-09-16-entries-written-once.md index aadf39f..66e24ac 100644 --- a/docs/proposals/2026-09-16-entries-written-once.md +++ b/docs/proposals/2026-09-16-entries-written-once.md @@ -51,96 +51,255 @@ compaction, merge, pack -- operates on indexes, which are an order of magnitude smaller than what they index (a key and a location, ~40 B, against a ~300 B value) and can be rebuilt from the entries. -## The dynamic layer: a heap with holes +## The dynamic layer: files of entries, a map of keys The dynamic layer holds a bounded key set rewritten forever. Today every rewrite appends and the old copy is garbage until a fold -rewrites everything around it. Instead: - -- **Entries live in a heap file per shard.** An entry is - `[len][key][value][checksum]`; its location is `(file, offset)`. -- **A rewrite that fits reuses the slot.** A value no larger than the - slot it replaces is written in place. The soak's dynamic rewrites - are chain heads, BPT nodes and account state whose size is stable, - so this is the common case and it creates no garbage at all. -- **A rewrite that does not fit appends and frees a hole.** The new - entry goes to the end of the heap (or into a hole that fits); the - old slot becomes a hole. -- **Holes are filled, not swept.** Free space is kept by size class - (`8 << n` bytes); a new entry takes the smallest hole that fits, or - the end of the file. Fragmentation is bounded by the size classes - the way a slab allocator's is, and the store reports the ratio of - hole bytes to live bytes. -- **Moving is the only rewrite, and it is bounded.** When the hole - ratio exceeds a threshold, one pass moves one entry -- the last - live entry of the file into the largest hole that fits -- and stops. - A pass costs one entry, never the file. The file shrinks from the - end when its tail is a hole. +rewrites everything around it, index and filter included. Instead +(`database/heap.go`, opened with `NewKVShardHeapN` / `NewKV2Heap`, +recognised on open by its directory; `bdbench -dyna-heap`): + +- **An entry is written once, at the end of a file.** An entry is + `[len][height][key][value][checksum]` at its exact aligned length; + a key is `(file, offset, length)`. Data files are fixed-size + (`HeapFileBytes`, 16 MB); a block appends to the current file, so + the block sync is one sequential fsync per file the block touched. + (Filling holes at put time was built first and measured: a block's + 11k rewrites scattered over a 380 MB heap dirtied a page each, and + the barrier wrote 4x the ingest -- run 3, 443 MB/s against run 2's + 117. Size classes were measured too: a third of the store wasted + for nothing, once nothing was allocated from free lists.) +- **A rewrite within the block reuses the slot** when the entry keeps + its aligned size: nothing durable names it yet, and the file stays a + contiguous sequence of entries a scan can walk. +- **A rewrite in a later block appends.** The slot the last durable + index names is never overwritten, or a crash between the write and + the block's sync would expose an uncommitted value under a committed + name. The old slot is dead where it lies. +- **The mover makes the space.** On the maintenance cadence a pass + takes the deadest files -- once half dead, or whatever their ratio + while dead bytes exceed live, which bounds the heap at twice its + live set -- copies their live entries into a file of its own, and + marks a file left with nothing live for deletion by the sync after + the delta naming the copies. The mover's file is not the block's, + so their barriers never share an inode (measured: the two fsyncs on + one file flushed each other's pages, seal p50 193 ms). The pass + holds the shard's lock only to pick, to plan and to name, never + across a read, a write or an fsync (measured: reading a 16 MB file + under the lock put the seal at 229 ms). A key rewritten while its + copy is in flight leaves the copy dead on arrival. The store + reports bytes scanned against bytes copied: the heap's write + amplification. - **The key map is in memory** for the live dynamic key set (the - soak's half million keys are ~24 MB per store; 1.2 allows memory - that scales with the working set). It is also what the seal makes - durable, below. + soak's half million keys are ~50 MB per store in Go; 1.2 allows + memory that scales with the working set). What the seal makes + durable is a delta of the keys the block touched; a snapshot starts + a new index generation in a file of its own on the maintenance + cadence, written aside, fsynced and renamed into place, so no delta + is ever truncated away. The layer's size converges to O(live keys) by construction (1.5), -without the deeper fold 2.7 allows today. +without the deeper fold 2.7 allows today, and nothing but entries is +ever copied. ## The permanent layer: append-only data, merged indexes The permanent layer is write-once; it has no garbage. Its merges and -packs exist to bound the file count and the length of a history walk -(2.7). Both are properties of the *index*, so: - -- **A shard's data is one growing file** (or one per `PackEvery` - blocks). A block's records are appended; nothing is ever copied. -- **Each block seals an index delta**: the sorted `(key, offset)` pairs - of the records the block appended, with its filter (2.5). -- **Merge and pack fold index deltas** into one sorted index per shard - per level, on the adapter's cadence, off the protocol path, exactly - as today's tiers do -- but moving ~40 B per record instead of ~300. - The walk length and the file count are bounded by the index tiers; - the data file is never in the walk (a key resolves to an offset, - then one `pread`). +packs exist to bound the file count and the length of a lookup's walk +(2.7), and both are properties of the *index*. Today (`segstore.go`, +`seal.go`, `blockset.go`) each block seals a segment -- a body file and +an index file with an embedded filter -- and `MergeBelow` folds +segments with `concatSegments`, which copies every body byte-verbatim +into a new body and rebuilds the index over it; `PackFinalized` copies +the bodies again into a cross-shard set file. An index record is 48 +bytes, `key, offset, length`, with no file identifier: several bodies +are only ever addressed through an out-of-band base, which is why the +bodies have to be concatenated to be merged. Instead: + +- **A shard's data is a sequence of fixed-size files, appended and + never moved**, the heap's file model (`HeapFileBytes`): a block's + records go to the end of the current file, and a record is + `(file, offset, length)`. Nothing is ever copied; a file is deleted + only by `DropBelow` when a pack no longer needs it -- and packs no + longer copy either, so a data file lives until the retention policy + above the store says otherwise. +- **An index record carries the file.** `key(32) file(4) offset(4) + length(4)`, 44 bytes, sorted by key as today. `mergeIndexes` and + `indexWriter` already merge indexes without touching a body; with + the file in the record they need no bases. +- **Each block seals an index delta** with its filter: the sorted + records of what the block appended, ~40 bytes a record. The delta + is what the window is made of; the live key filters (`keyfilter.go`) + are built over deltas exactly as they are built over segments now. +- **The long search is bucketed, and merged a bucket at a time.** A + shard's index below the window is `PermBuckets` buckets (256 by + default; nothing about the number is special, and a modulus serves + as well as a mask), each holding a few sorted runs of 44-byte + records with a filter per run. Every block the maintenance step + takes the next bucket in rotation, gathers that bucket's records + from the deltas sealed since the bucket was last merged, writes them + as a new run, and folds the bucket's runs by ratio; a delta older + than every bucket's last merge is dropped. So the work every block + is a fixed slice, proportional to what arrived for one bucket, and + the big fold is rare and bounded per pass -- never a rewrite of the + shard's whole index, which would grow with the chain (1.2). A + merge swaps one bucket's runs under that bucket's lock; the other + buckets keep answering. A pack is the same fold across shards for + a block range, with no bodies in it. +- **Reads keep their protocol-path rule** (1.3): the window is the + last N blocks' deltas behind the live filters; an immutable key the + filters deny is absent. Below the window a key's bucket is probed + newest run first, one filter and one binary search per run, and + that is `GetDeep`'s walk; a lookup resolves a key to `(file, + offset, length)` and does one `pread` of the data file. + +### Sizing the buckets + +`TestPermSizingSim` (`go test ./database/ -run TestPermSizingSim -v +-args -sim`) models one shard at the soak's rate, 1,040 permanent +records a block, against the store as it is (segments folded by +copying bodies every 20 blocks under the ratio rule, packed every +1,000 blocks). Its first result was a warning: a bucket that is +never drained holds 1/B of the whole chain, so with any fixed count +the largest fold grows with the age of the store (256 buckets: 14 MB +at a day, 102 at a week, 460 at a month), against 1.2. + +The bound is the pack watermark, which the store already has. Every +`PackEvery` (1,000) blocks the finished history is packed and dropped +from the shard; the buckets cover only the history above the +watermark, and since the keys are hashes their shares are even. A +bucket then holds at most 1,000 blocks of its share -- ~4,000 +records, ~180 KB, at 256 buckets -- so a fold is bounded by the pack +period, not the chain, and nothing needs to split. The count is a +free choice; 256 keeps runs small and a merge's lock small. + +Per shard per block that comes to about: the block's delta (46 KB), +the bucket runs (46 KB), their folds (~46 KB) and the pack amortized +(46 MB of index every 1,000 blocks, 46 KB): ~180 KB a block against +today's 2.5 MB, every byte an index byte. The pack is index-only, +46 MB a shard in place of 358 MB of bodies, and the chain's growth +lives where it lives today: in the sets below the watermark, grouped +by block range with one filter per finished group, reached only by +`GetDeep`. Filters for the buckets and the newest sets stay +resident within a budget; older groups are probed cold, as now. + +Ratio 0.25 stays (0.1 halves the runs a lookup probes but nearly +doubles the bytes written); every bucket is merged every 256 blocks +(B/256 buckets a block), which keeps the recent sections at a few MB +a shard. + +### Shards partition keys, not files (measured) + +The dynamic layer alone, nine stores, five minutes: seal p50 54-65 ms +at 8 storage shards, 67-79 ms at 64, 75-86 ms at 128, with a burst of +lost blocks every other minute at 64 and 128 and 8x and 16x the +files. Every shard's files sync on their own at every block, so the +barrier count rises with the shard count; a hundred-plus shards are +right for spreading the retiring work and shrinking every lock, and +they must not each bring a barrier. A block's records for every +shard go into one per-store append file, shard-tagged, and each +shard's index points into it: one data fsync per store however many +shards. ## The seal: one commit point per store per block -With indexes as the sealed object a block is one barrier round per -store rather than four per shard: +Today a non-empty block costs each shard four barriers -- the data +fsync, the index file's, the manifest's temp file, the directory -- +and the store two more for its block record (`seal.go`, +`commitJSON`). Nine stores of eight shards ask the device for about +360 barriers a second before any maintenance runs, and the platform +measured every seal goroutine waiting in `fsync` (#94). -1. Every shard's data appends (permanent) and slot writes (dynamic) - for the block are fsynced, in parallel: one round. -2. The block's index deltas -- permanent `(key, offset)` and dynamic - `(key, location)` for every key the block touched -- are written to - one per-store block file and fsynced: one round. -3. The store's manifest names the block file: the existing commit, - one round. +With deltas as the sealed object, a block's permanent index delta is +appended to the same data file right after the block's records, and +the dynamic layer's delta after its entries the same way, so that: -Three rounds per store per block, none per shard beyond the parallel -data sync, in place of four rounds per shard. This is 1.8's "one -commit point" and closes #33. +1. Every shard fsyncs its data files once: entries and delta together, + in parallel across shards -- one round. +2. The store's block record names, for every shard, the file and + offset of the block's deltas: the existing `block.json` commit, + temp file and directory -- one round. + +Two rounds per store per block, in place of four per shard plus two. +Recovery trusts a delta only if its own checksum holds and every entry +it names checks, since one fsync does not order the delta's bytes +after the entries'; a delta that fails is the block that did not +commit, exactly as a torn delta is today. This is 1.8's "one commit +point" and closes #33. ## Durability and crash consistency (1.8) -- **A hole is reusable one seal late.** A slot freed in block N may - be reused only after N's seal is durable. Until then the durable - index still names the old slot, and a crash must find it intact. - Reuse is deferred the way 2.6 defers deletion. The adapter never - asks the store for an old version (its pre-images are memoized on - its side), so reuse waits on the seal and on nothing else. +- **A file is deleted one seal late, by the mover, and never while a + delta names it.** A file emptied by the mover leaves the map only + after the delta naming the mover's copies out of it is durable. + Until then the durable index still names its slots, and a crash + must find them intact: never unlink what a durable index names. + The deltas name ranges, and the mover's files are the destination + of theirs: a file every copy in which has died is still named by + the generation's deltas, and stays until the snapshot that + supersedes them, however dead it is. (Measured the other way + first: the platform's reopen check found a store closed cleanly + that would not open, its replay stopped at a delta naming an + unlinked mover file, and the derivation deleting what it then took + for unnamed.) Each file carries the bytes such deltas name in it; + the pinned-bytes rule counts those files, so the snapshot comes + early when they pile up. The unlink itself is the mover's, at + its next pass, so no block's seal holds the store lock over a + directory operation; between the two the file is unnamed on disk, + which an open deletes as such. The adapter never asks the store + for an old version (its pre-images are memoized on its side), so + deletion waits on the seal and on nothing else. +- **A seal is one fsync per file it touched, outside the lock.** The + block's delta is written into the data file behind the block's + entries before the fsync, so one barrier covers both, in both + layers; replay trusts the last delta only if every entry it names + checks. A put that lands during the seal belongs to the next + block: the seal takes the block's records under the lock and + releases it before the barrier. +- **What the manifest does not name is still replayed if a seal + wrote it.** The permanent layer's manifest is committed by + maintenance, not by seals, so the seals roll data files the + manifest has not seen. Open takes data files in id order past the + manifest's next id for as long as they exist, and replays their + deltas; a run file past the manifest's next run id is maintenance + output a crash left unnamed, which open removes, so the id is free + for exclusive creation again and no file is ever published over + (1.7). A merge lost this way is done again from the pending + deltas, which is why the manifest need only be committed every + eighth merge, or when a fold leaves run files to drop. - **A torn slot is detected, not misread.** Every entry carries its - length and a checksum; a slot whose checksum fails after a crash is - a hole, and the durable index never points at one, because an index - entry is durable only after the slot it names is. -- **The in-memory key map is rebuilt on open** from the last durable - index snapshot plus the block files after it, the same replay 2.8 - does for manifests today. A snapshot is taken on the pack cadence - so the replay is bounded. -- **Nothing durable is ever overwritten by a different entry** except - a slot the durable index no longer names. 1.7's identity rule - holds for files: a data file is appended, never republished. + length and a checksum; an entry above the committed height is a + block that never synced. An index entry is durable only after the + slot it names is, so a torn slot is never named; open cuts the + files back to their last named slot and drops a torn delta whole. +- **The key map is rebuilt on open** from the newest whole index + generation: its snapshot, then its deltas. A generation whose + write was interrupted is not whole and is removed. +- **Repair reads the keys from the data.** With the index gone, a + scan of the data files rebuilds the map: every entry carries its + key and the height that wrote it, so the highest committed copy of + a key wins (later in the scan on a tie) and an entry above the + committed height, which only the store above knows, is dropped. A + child process killed mid-block, three times over, reopens and + repairs to exactly the last durable block (`heap_crash_test.go`). +- **Nothing durable is ever overwritten by a different entry.** 1.7's + identity rule holds for files: a data file is appended, never + republished. ## What it is measured with -`cmd/bdbench -stores 9`, the run above, is the acceptance test: +`cmd/bdbench -stores 9`, the run above, is the acceptance test. It +checks answers, not only times: a permanent value is derived from its +key, so every sampled permanent read (through the deep read, as the +adapter reads anything older than the window) is checked for presence +and content, a checked hot key must read as last written, and at the +end every store is closed, reopened and every sampled key of both +layers read back -- a run that lost one fails. Until 2026-09-17 the +platform tolerated "not found" on every permanent read, and a data +file rolled between manifest commits was being lost on reopen without +a number moving. With the check in place: 200,000 sampled permanent +keys per store read back correctly after reopen, zero mismatches +under load. The acceptance run must show: - seal p90 under `-seal-budget` (100 ms) and flat from minute 1 to minute 30; @@ -152,10 +311,139 @@ commit point" and closes #33. ## Order of work -1. The dynamic heap, behind the existing `KV2` dynamic surface (`Put`, - `Get`, `Seal`, `CompactHistory` becoming the bounded move, - `Stats`), so the sharding and the adapter do not change. The - platform measures it alone (`-stores 9 -perm 0`). -2. The permanent index deltas and the single block file, which also - brings the seal to one commit point. -3. Merge and pack over indexes. +1. The dynamic heap, behind the existing `KV2` dynamic surface, so + the sharding and the adapter do not change. *Built (PR #97).* + Alone on the disk with nine stores (run 7): seal p50 53-61 ms in + every minute, read p99 1-2 µs, maintenance ~10 s a minute, store + 3.8 GB at five minutes, zero wrong answers; the segment layer alone + seals at 33-38 ms p50 but reads at 11-13 µs, spends 22-110 s a + minute compacting, and had a compaction storm in minute 4 (seal + max 1.5 s, 58 blocks missed). The heap's remaining tail (p90 + 150-250 ms in the minutes the mover copies most) is the mover's + own fsync volume in the device queue, which the pass size paces. +2. The permanent layer as files of records with index deltas + (`PermStore`, behind `KV2`'s permanent surface), the 44-byte index + record, the bucketed long search merged in rotation, and packs over + indexes. *First cut built and wired (`perm.go`, + `NewKVShardFilesN`, `bdbench -perm-files`); its first full-load run + put the seal at 59 ms p50 in minute 1, then climbing to 209 by + minute 5 because maintenance shared the seal's run file; fixed, + remeasured next.* Measured + alone first (`-stores 9 -dyna 0`), then with the heap under the + full load, which is the acceptance run. +3. The block's deltas in the data files: one barrier per shard per + layer. *Built for both layers.* The heap's delta names no key: + a block's entries are contiguous in its file and carry their + keys, so the delta is the ranges the block wrote (the mover's + named copies first, then the block's own) and the copies that + arrived dead, a few dozen bytes a block; a delta of 44-byte + records was a fifth of the heap's writes. Replay scans the + ranges and trusts the last delta only if every entry in it + checks. The permanent layer's delta stays a run with its filter, + since it is the window's search structure, but lives in the data + file too. Alone, the heap seals at 44-54 ms p50 at nine stores + (57-64 with two barriers). The mover finds a file's live entries + through the index rather than by scanning the file, because nine + stores' scans together starved the block loops for CPU. The + mover's budget is the store's (`HeapStoreCleanBytes`), shared + among its shards: with one shard and a shard's budget the store + grew to 6 GB in five minutes. A merge reads only its due + buckets' slice of each pending delta (the run is sorted, the + slice is found by a binary search on the first key byte) instead + of every pending delta whole at every pass. + + *Measured, 2026-09-17:* every "bad minute" of the day's runs -- + seal p50 unchanged, p90 200-450 ms, the heap's fsync average + 3-8x its usual 12-22 ms -- was the disk, not the store. A + 2-second timeline showed every fsync on the box stepping to + 100-330 ms for 30-40 s at once, with nothing in the store + changing (no snapshot, the same mover volume and release cadence) + and with one shard per store (9 barriers/s) exactly as with eight + (72/s). The instrument's root volume is on LUKS without discards + and has never been trimmed. Until it is, runs are compared by + their clean minutes; the per-minute row carries the heap's split + (fsync average and bytes, snapshots, releases, moved bytes) and + each store's seal p90 so a stall can be told from a tail. + + *Maintenance is a slice per call, in rotation.* The heap's + remaining tail in clean minutes was the mover's own delivery: on + the adapter's cadence every shard of every store ran its pass in + the same second -- 300 MB and 72 barriers at once at nine stores + -- and the seals' fsyncs queued behind it for the next four + seconds (device write time 24 s in a 2-second window against 1-3 + otherwise). A store's mover rate is now `HeapStoreCleanBytes` per + `HeapCleanPeriod` blocks, and a `Compress` call takes the shards + next in rotation, as many as the blocks since the last call earn, + each with its share: called every block it moves a little on one + shard, and a hundred shards spread the same rate a hundred ways + with one shard locked at a time. The permanent layer's merge + already worked this way (buckets due by blocks elapsed). + Snapshots are by block count, or early when the files held only + by their deltas outweigh two data files; the manifest commits by + block count. + + *Measured on that build (before the every-block cadence), nine + stores, eight shards, clean minutes:* the heap alone 36-38 ms p50 + / 48-60 p90 in every minute; the files store 36-38 / 50-52, where + the day before it was 60 / 370-450 in every minute after the first + -- the seal's second barrier, the merge's three, and its reread of + every pending delta were the tail. One shard per store: 37-39 / + 51-54. + + *What waits unmerged is bounded by the window.* A bucket was + merged once per 256 blocks, so up to 256 deltas waited unmerged, + and every deep read probed each one's filter and every merge read + a slice of each: over five minutes read p99 climbed from 7 to 47 + us and merge work from 157 to 305 s a minute. A bucket is now + merged once a window (`PermMergeEvery` = `MinFilterBlocks`), so + no more than a window of deltas waits; the extra folds are a + fraction of the index bytes, themselves a sixth of the data. + + *The cadence per layer.* On the every-block cadence the heap's + slice is cheap, but the permanent layer's merge ended every call + with an fsync of its run file: at a call per shard per block that + was 72-144 barriers a second at nine stores, merges cost 190-250 s + of work a minute, and every seal's fsync doubled. A bucket's run + is named by the manifest alone, so it needs durability before the + manifest commit and not before: the run files are fsynced at the + commit (every PermManifestBlocks, or at a fold), and a merge call + has no barrier. + + *Measured with every fix above, both layers, nine stores, eight + shards, maintenance every block, 2026-09-17 00:05:* seal 38-42 ms + p50 and 57-65 p90 in the four clean minutes (the fifth was a disk + stall), merges 3-8 s of work a minute, zero mismatches, and every + store reopened with all 200,000 sampled permanent keys and 1,024 + checked dynamic keys read back correctly. Block time drifted from + 157 to 210 ms: the profile put 40% of the CPU in the deep read's + binary search over bucket runs, a pread a probe; a resident run + now keeps a fence of every 32nd key and a lookup is one read. + Measured: read p99 6-7 us instead of 9-10, block time flat at + 178-181 ms from minute 3 on instead of climbing, all nine stores + read back clean again. What remains in the deep read is one read + of the run's chunk and one of the entry, which is the floor for + an index that does not fit in memory. + + Still to do: the store-level commit (one block record naming + every shard's deltas) and the per-store data file, so that a + hundred shards cost a block one barrier; one shard per store is + that layout by another name and measures the same as eight now. + The 30-minute acceptance run (2026-09-17, after the disk was + trimmed; nine files stores, maintenance every block, phase + offset): 16,164 blocks, every minute's seal p90 between 42 and + 57 ms against the 100 ms budget, the heap's fsync average flat + at 10-14 ms, no mismatch, and all nine stores reopened and read + every sampled key back with 0 wrong. The device's write ticks + per two-second window were p50 1.9 s, max 14.3 s: no stall. + Block p50 rose from 129 ms to ~170 ms over the first five + minutes as the window filled and then held at 170-193 ms. In + minutes 28 and 30 the process read 56 and 120 MB/s from the + drive and read p99 went from 7-8 us to 52-59 us, which at 40,000 + lookups a block put block p50 at 250-300 ms and p90 at 370-460 + ms; the seal did not move. The store was 68-74 GB on a 64 GB + box whose page cache had been full since minute 10: lookups of + permanent keys of all ages had begun to miss the cache. That + is the floor for an index that does not fit in memory, not a + cost that grows with the store's age in the store's own work, + and it is where a resident permanent index (or a bigger box) + would show. Run: accept-30m, 2026-09-17 08:00.