diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0272af9..ae8959b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,7 +17,7 @@ jobs:
matrix:
# Every module has its own go.mod and is linted independently. Keep
# this in step with MODULES in the Makefile.
- module: [".", "lfu", "policies", "policies/arc", "policies/fifo", "policies/tinylfu", "metrics", "bandit", "bandit/redis", "benchclient", "bench", "examples/basic", "examples/migration"]
+ module: [".", "lfu", "policies", "policies/arc", "policies/fifo", "policies/tinylfu", "metrics", "bandit", "benchclient", "bench", "examples/basic", "examples/migration"]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
@@ -37,7 +37,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- module: [".", "lfu", "policies", "policies/arc", "policies/fifo", "policies/tinylfu", "metrics", "bandit", "bandit/redis", "benchclient", "bench", "examples/basic", "examples/migration"]
+ module: [".", "lfu", "policies", "policies/arc", "policies/fifo", "policies/tinylfu", "metrics", "bandit", "benchclient", "bench", "examples/basic", "examples/migration"]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
@@ -84,52 +84,3 @@ jobs:
cache-dependency-path: go.sum
# Reads each go.mod through `go mod edit -json`, so no network is needed.
- run: ./scripts/release-check.sh
-
- # The test job above runs bandit/redis against miniredis, which is a fake.
- # This one runs the same suite against real servers, because the adapter
- # leans on three things a fake can be too permissive about: TIME called
- # inside a Lua script, SET with NX and PX, and HINCRBY on a key the script
- # names itself rather than declaring in KEYS.
- #
- # Both engines run, because both are documented as supported. Testing one
- # while documenting two is how that claim quietly stops being true. This is
- # the CI equivalent of `make redis-test`; docker-compose.yml is the local one.
- redis:
- name: test bandit/redis (${{ matrix.engine.name }})
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix:
- engine:
- # Deriving the bucket from the server's clock inside a script needs
- # effects replication, so the floor is Redis 7 / Valkey 7.2.
- - name: valkey 8
- image: valkey/valkey:8-alpine
- ping: valkey-cli ping
- - name: redis 7
- image: redis:7-alpine
- ping: redis-cli ping
- services:
- server:
- image: ${{ matrix.engine.image }}
- ports:
- - 6379:6379
- options: >-
- --health-cmd "${{ matrix.engine.ping }}"
- --health-interval 1s
- --health-timeout 3s
- --health-retries 20
- steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-go@v5
- with:
- go-version: "1.25"
- cache-dependency-path: bandit/redis/go.sum
- - name: go test
- working-directory: bandit/redis
- # Set, so the suite uses the service container rather than miniredis.
- # The suite fails rather than skips if this address is unreachable, so
- # a broken service container cannot pass as a green fake run.
- env:
- AS_CACHE_REDIS_ADDR: 127.0.0.1:6379
- run: go test -race -count=1 ./...
diff --git a/Makefile b/Makefile
index 45ae113..6aea1a5 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
# Each of these directories is a separate Go module (own go.mod), so tooling is
# run once per module. The root .golangci.yml is shared by all of them.
-MODULES := . lfu policies policies/arc policies/fifo policies/tinylfu metrics bandit bandit/redis benchclient bench examples/basic examples/migration
+MODULES := . lfu policies policies/arc policies/fifo policies/tinylfu metrics bandit benchclient bench examples/basic examples/migration
GOLANGCI_LINT_VERSION := v2.8.0
@@ -50,33 +50,6 @@ release-check: ## Check the repository could actually be released today
evidence: ## Replay the workload suite and print the policy comparison tables
( cd bench && go test -count=1 -timeout 20m -v ./... )
-# The bandit/redis tests run against miniredis by default, which is a fake.
-# The Lua the adapter depends on - TIME inside a script, SET NX PX, HINCRBY on
-# a key the script names itself - is exactly what a fake can be too permissive
-# about, so the same suite runs against real servers here. See
-# docker-compose.yml for why both engines are covered rather than one.
-COMPOSE ?= docker compose
-
-.PHONY: redis-up
-redis-up: ## Start the local Valkey and Redis containers and wait for them
- $(COMPOSE) up -d --wait
-
-.PHONY: redis-down
-redis-down: ## Stop the local Valkey and Redis containers
- $(COMPOSE) down -v
-
-.PHONY: redis-test
-redis-test: ## Run the bandit/redis suite against real Valkey and Redis
- @$(MAKE) redis-up
- @status=0; \
- for target in "valkey 127.0.0.1:63799" "redis 127.0.0.1:63798"; do \
- set -- $$target; \
- echo "==> bandit/redis against $$1 ($$2)"; \
- ( cd bandit/redis && AS_CACHE_REDIS_ADDR=$$2 go test -race -count=1 ./... ) || status=1; \
- done; \
- $(MAKE) redis-down; \
- exit $$status
-
.PHONY: tidy
tidy: ## Run go mod tidy across all modules
@set -e; for m in $(MODULES); do \
diff --git a/README.md b/README.md
index 8ad56fd..481eb98 100644
--- a/README.md
+++ b/README.md
@@ -40,9 +40,8 @@ been exercised under the race detector and adversarially reviewed. It is all in
| [Configuration](docs/configuration.md) | Every `Settings` field, migration strategies, sampling, stability gates, tuning |
| [Policies](docs/policies.md) | The nine ready-made arms, their caveats, adapting your own cache |
| [Advisor mode](docs/advisor-mode.md) | `ObserveOnly`, `Advice()`, and the `metrics` module |
-| [Evidence](docs/evidence.md) | Every measured claim: policy tables, competing libraries, real traces, sampling fidelity, fleets |
+| [Evidence](docs/evidence.md) | Every measured claim: policy tables, competing libraries, real traces, sampling fidelity |
| [Benchmarking](docs/benchmarking.md) | Reproducible replays, `benchclient`, `make evidence` |
-| [Running a fleet](docs/fleet.md) | Pooling evidence across replicas through Valkey or Redis |
| [Project site](https://sshaplygin.github.io/as-cache/) | Landing page, plus an interactive explorer of the bandit's decisions on a phase-shift run |
Past releases are recorded in the [changelog](CHANGELOG.md) and on the
diff --git a/bandit/config.go b/bandit/config.go
deleted file mode 100644
index 08c9fb6..0000000
--- a/bandit/config.go
+++ /dev/null
@@ -1,277 +0,0 @@
-package bandit
-
-import (
- "fmt"
- "math/rand/v2"
- "os"
- "time"
-)
-
-// Mode selects how a fleet turns pooled evidence into a policy choice.
-type Mode uint8
-
-const (
- // ModeLeader has one replica per coordination epoch read the fleet's
- // aggregate, choose, and publish that choice for every other replica to
- // apply. The fleet therefore runs one policy at a time.
- //
- // This is the default, and it is the mode that keeps the pooled numbers
- // honest. Every replica measures the same arm in the active role and every
- // other arm in the shadow role, so the systematic gap between the two
- // roles applies equally to every replica's report and largely cancels when
- // they are summed. See Role.
- ModeLeader Mode = iota + 1
-
- // ModeSharedPosterior has every replica draw its own selection from the
- // pooled posterior. No leader, no election, and a replica that loses the
- // store keeps working without any handover.
- //
- // The cost is that replicas may run different policies indefinitely, which
- // makes the role gap asymmetric: an arm active on most of the fleet is
- // mostly measured in the flattering role, so it accumulates an advantage
- // in proportion to how widely it is already deployed. Pair it with
- // EvidenceShadowOnly to remove that feedback, at the price of ignoring
- // what the serving policies measured.
- ModeSharedPosterior
-)
-
-// EvidenceMode selects which measurements feed the pooled posterior.
-type EvidenceMode uint8
-
-const (
- // EvidenceAll pools active-role and shadow-role counts together. It
- // matches what a single-node bandit sees exactly, and it is the default.
- EvidenceAll EvidenceMode = iota + 1
-
- // EvidenceShadowOnly discards active-role counts and compares arms purely
- // on shadow measurements, so every arm is measured the same way. It is
- // only meaningful under ModeSharedPosterior, where a diverse fleet leaves
- // every arm shadowing somewhere; under ModeLeader it is rejected.
- EvidenceShadowOnly
-)
-
-// Config configures a Distributed bandit.
-type Config struct {
- // Store is where the fleet coordinates. Required.
- Store Store
-
- // Namespace names the fleet. Required, and it must be the same string on
- // every replica that should pool with each other, and different on any
- // that should not.
- //
- // It is not the whole key: a fingerprint of the cache's measurement regime
- // - its arms, its capacity and its sample rate - is appended, so replicas
- // that share a name but are not measuring the same thing pool separately
- // instead of pooling wrongly. A hit rate from a 1000-entry cache says
- // nothing about a 100-entry one, and averaging them describes neither.
- Namespace string
-
- // NodeID identifies this replica within the fleet, and needs only to be
- // unique. Defaults to hostname-pid-random, which is unique enough and
- // stays readable in a leadership key.
- NodeID string
-
- // CoordinationEpoch is how often this replica syncs with the store, and
- // therefore how often the fleet can change its mind. Required.
- //
- // It is a separate, much slower clock than Settings.EpochDuration, and
- // deliberately so. Cache epochs are tuned in tens of milliseconds, which
- // is below the round trip to the store and below the clock agreement a
- // fleet can be assumed to have. Measure on the fast clock, coordinate on
- // the slow one. A second is a sensible starting point.
- CoordinationEpoch time.Duration
-
- // Window is how many past buckets the leader reads. Defaults to
- // DefaultWindow. Zero after defaulting means only the previous bucket.
- Window int
-
- // Decay weights each bucket by Decay^age before summing, so recent
- // evidence counts for more and old evidence fades out of the decision
- // rather than sitting in it forever. Defaults to DefaultDecay. A value of
- // 1 weights the whole window equally.
- //
- // A shared counter cannot be decayed in place - every replica applying the
- // multiplication would compound it once per replica - so the store holds
- // plain per-bucket sums and the weighting happens here, on read.
- Decay float64
-
- // MaxEvidence caps how many observations an arm's posterior is allowed to
- // rest on, after weighting. Defaults to DefaultMaxEvidence; a negative
- // value disables the cap.
- //
- // It exists because pooling multiplies evidence by the size of the fleet.
- // A Beta posterior narrows with the square root of what it has seen, so a
- // thousand replicas produce posteriors sharp enough that every Thompson
- // draw returns the same arm - the bandit stops exploring precisely at the
- // scale where a workload change is most expensive to miss. Capping keeps
- // the measured rate and discards the surplus certainty.
- MaxEvidence float64
-
- // Mode selects leader-elected or shared-posterior selection. Defaults to
- // ModeLeader.
- Mode Mode
-
- // Evidence selects which roles' counts feed the posterior. Defaults to
- // EvidenceAll.
- Evidence EvidenceMode
-
- // FallbackAfter is how long the store may go unreachable before this
- // replica stops waiting for the fleet and decides locally. Defaults to
- // three coordination epochs.
- FallbackAfter time.Duration
-
- // SyncTimeout bounds a single round trip to the store. Defaults to the
- // coordination epoch. It bounds how far behind a hung store can push this
- // replica's ticks; it never affects the cache, which is not waiting on any
- // of this.
- SyncTimeout time.Duration
-
- // LocalDiscount is the discount factor of the local Thompson bandit that
- // takes over when the store is unreachable. Defaults to
- // DefaultLocalDiscount.
- LocalDiscount float64
-
- // Jitter spreads each replica's sync across a fraction of the coordination
- // epoch, so a large fleet does not arrive at the store in lockstep on
- // every bucket boundary. Defaults to DefaultJitter. Buckets come from the
- // store's clock, so jitter can never put a replica in the wrong one.
- Jitter float64
-
- // Seed seeds the local fallback bandit and the jitter. Zero draws a random
- // one, which is what you want: seeding a fleet identically would
- // synchronise the jitter it exists to break up.
- Seed uint64
-
- // Now is the clock used for staleness and jitter, a seam for tests.
- // Defaults to time.Now. It is never used to derive a bucket - that is the
- // store's job precisely so a replica's clock cannot matter.
- Now func() time.Time
-}
-
-// Defaults applied to a zero-valued Config field.
-const (
- // DefaultWindow is how many buckets of history the leader reads. Ten
- // buckets of a one-second epoch is ten seconds of fleet evidence, which is
- // enough to be stable and short enough to still track a workload that
- // moves.
- DefaultWindow = 10
- // DefaultDecay weights each bucket at 0.8 of the one after it, so the
- // oldest bucket of a default window carries about a seventh of the weight
- // of the newest.
- DefaultDecay = 0.8
- // DefaultLocalDiscount is the fallback bandit's discount factor.
- DefaultLocalDiscount = 0.7
- // DefaultJitter spreads syncs over a tenth of the coordination epoch.
- DefaultJitter = 0.1
- // DefaultMaxEvidence caps an arm's posterior at a hundred thousand
- // weighted observations, which puts its standard deviation at roughly a
- // sixth of a percentage point: enough certainty to separate arms that
- // differ by half a point, little enough to keep exploring arms that do
- // not.
- DefaultMaxEvidence = 100_000.0
-)
-
-// validate fills in defaults and reports whatever cannot be defaulted.
-func (c *Config) validate() error {
- if c.Store == nil {
- return ErrNilStore
- }
- if c.Namespace == "" {
- return ErrEmptyNamespace
- }
- if c.CoordinationEpoch <= 0 {
- return fmt.Errorf("%w: got %s", ErrInvalidCoordinationEpoch, c.CoordinationEpoch)
- }
- if c.Window < 0 {
- return fmt.Errorf("%w: got %d", ErrInvalidWindow, c.Window)
- }
- if c.Decay < 0 || c.Decay > 1 {
- return fmt.Errorf("%w: got %v", ErrInvalidDecay, c.Decay)
- }
- if c.Jitter < 0 || c.Jitter >= 0.5 {
- return fmt.Errorf("%w: got %v", ErrInvalidJitter, c.Jitter)
- }
-
- // Both enums number from one, so their zero value names no mode at all
- // rather than happening to name the default. That has to be resolved here,
- // before anything compares against it: an unresolved zero Mode is neither
- // ModeLeader nor ModeSharedPosterior, so the replica would claim no
- // leadership and follow no leader, and the fleet would sync forever
- // without ever deciding anything.
- if c.Mode == 0 {
- c.Mode = ModeLeader
- }
- if c.Evidence == 0 {
- c.Evidence = EvidenceAll
- }
-
- if c.Mode == ModeLeader && c.Evidence == EvidenceShadowOnly {
- return ErrShadowOnlyUnderLeader
- }
-
- if c.Window == 0 {
- c.Window = DefaultWindow
- }
- if c.Decay == 0 {
- c.Decay = DefaultDecay
- }
- if c.Jitter == 0 {
- c.Jitter = DefaultJitter
- }
- if c.LocalDiscount <= 0 || c.LocalDiscount > 1 {
- c.LocalDiscount = DefaultLocalDiscount
- }
- if c.MaxEvidence == 0 {
- c.MaxEvidence = DefaultMaxEvidence
- }
- if c.FallbackAfter <= 0 {
- c.FallbackAfter = 3 * c.CoordinationEpoch
- }
- if c.SyncTimeout <= 0 {
- c.SyncTimeout = c.CoordinationEpoch
- }
- if c.Now == nil {
- c.Now = time.Now
- }
- if c.Seed == 0 {
- // Not a secret: it only has to differ between replicas, so that a
- // fleet's jitter is not synchronised.
- c.Seed = rand.Uint64() //nolint:gosec // see above
- }
- if c.NodeID == "" {
- c.NodeID = defaultNodeID(c.Seed)
- }
-
- return nil
-}
-
-// counterTTL is how long a bucket's counters outlive the bucket. The leader
-// reads Window buckets back, so anything shorter would leave holes in the
-// window; the margin covers a leader that ticks late.
-func (c *Config) counterTTL() time.Duration {
- return time.Duration(c.Window+2) * c.CoordinationEpoch
-}
-
-// leaderTTL is how long a leadership claim survives. Leadership is per-bucket
-// and produces one immutable decision, so an over-long claim costs nothing
-// beyond that bucket; two epochs covers a leader whose sync is slow.
-func (c *Config) leaderTTL() time.Duration {
- return 2 * c.CoordinationEpoch
-}
-
-// decisionTTL is how long a published decision remains readable. It outlives
-// its bucket so a replica that ticks late still finds it.
-func (c *Config) decisionTTL() time.Duration {
- return 3 * c.CoordinationEpoch
-}
-
-// defaultNodeID builds an identifier that is unique enough for leadership and
-// still legible in a key.
-func defaultNodeID(seed uint64) string {
- host, err := os.Hostname()
- if err != nil || host == "" {
- host = "unknown"
- }
-
- return fmt.Sprintf("%s-%d-%x", host, os.Getpid(), seed&0xffffff)
-}
diff --git a/bandit/config_test.go b/bandit/config_test.go
deleted file mode 100644
index 949a6a9..0000000
--- a/bandit/config_test.go
+++ /dev/null
@@ -1,195 +0,0 @@
-package bandit
-
-import (
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-func validConfig() Config {
- return Config{
- Store: NewMemStore(),
- Namespace: "test",
- CoordinationEpoch: time.Second,
- }
-}
-
-func TestConfig_RejectsWhatCannotBeDefaulted(t *testing.T) {
- tests := []struct {
- name string
- mutate func(*Config)
- wantErr error
- }{
- {
- name: "no store",
- mutate: func(c *Config) { c.Store = nil },
- wantErr: ErrNilStore,
- },
- {
- name: "no namespace",
- mutate: func(c *Config) { c.Namespace = "" },
- wantErr: ErrEmptyNamespace,
- },
- {
- name: "zero coordination epoch",
- mutate: func(c *Config) { c.CoordinationEpoch = 0 },
- wantErr: ErrInvalidCoordinationEpoch,
- },
- {
- name: "negative coordination epoch",
- mutate: func(c *Config) { c.CoordinationEpoch = -time.Second },
- wantErr: ErrInvalidCoordinationEpoch,
- },
- {
- name: "negative window",
- mutate: func(c *Config) { c.Window = -1 },
- wantErr: ErrInvalidWindow,
- },
- {
- name: "decay above one",
- mutate: func(c *Config) { c.Decay = 1.5 },
- wantErr: ErrInvalidDecay,
- },
- {
- name: "decay below zero",
- mutate: func(c *Config) { c.Decay = -0.1 },
- wantErr: ErrInvalidDecay,
- },
- {
- name: "jitter of half an epoch",
- mutate: func(c *Config) { c.Jitter = 0.5 },
- wantErr: ErrInvalidJitter,
- },
- {
- name: "negative jitter",
- mutate: func(c *Config) { c.Jitter = -0.1 },
- wantErr: ErrInvalidJitter,
- },
- {
- name: "shadow-only evidence under leader election",
- mutate: func(c *Config) {
- c.Mode = ModeLeader
- c.Evidence = EvidenceShadowOnly
- },
- wantErr: ErrShadowOnlyUnderLeader,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- cfg := validConfig()
- tt.mutate(&cfg)
-
- _, err := NewDistributed(cfg)
- assert.ErrorIs(t, err, tt.wantErr)
- })
- }
-}
-
-func TestConfig_ShadowOnlyIsAllowedUnderSharedPosterior(t *testing.T) {
- cfg := validConfig()
- cfg.Mode = ModeSharedPosterior
- cfg.Evidence = EvidenceShadowOnly
-
- bandit, err := NewDistributed(cfg)
- require.NoError(t, err)
- assert.NoError(t, bandit.Close())
-}
-
-func TestConfig_ZeroModeLeadsRatherThanDoingNothing(t *testing.T) {
- // Mode and EvidenceMode number from one, so their zero value names no mode
- // at all. Left unresolved it matches neither branch: the replica claims no
- // leadership and follows no leader, so the fleet syncs forever and never
- // decides anything - a configuration that looks healthy and selects
- // nothing.
- cfg := validConfig()
- require.Zero(t, uint8(cfg.Mode))
- require.NoError(t, cfg.validate())
-
- assert.Equal(t, ModeLeader, cfg.Mode)
- assert.Equal(t, EvidenceAll, cfg.Evidence)
-}
-
-func TestConfig_ZeroModeActuallyClaimsLeadership(t *testing.T) {
- store, clock := newFleetStore(t)
-
- subject := fleet(t, 1, store, clock, func(cfg *Config) {
- cfg.Mode = 0
- })[0]
-
- subject.report(t, 100, map[ascache.PolicyType]float64{ascache.LRU: 0.2, ascache.TinyLFU: 0.9})
- subject.bandit.sync()
- clock.advance(testEpoch)
- subject.report(t, 100, map[ascache.PolicyType]float64{ascache.LRU: 0.2, ascache.TinyLFU: 0.9})
- subject.bandit.sync()
-
- snapshot := subject.bandit.Snapshot()
- assert.Positive(t, snapshot.Leaderships)
- assert.Equal(t, ascache.TinyLFU.String(), snapshot.Selection)
-}
-
-func TestConfig_FillsInDefaults(t *testing.T) {
- cfg := validConfig()
- require.NoError(t, cfg.validate())
-
- assert.Equal(t, DefaultWindow, cfg.Window)
- assert.InDelta(t, DefaultDecay, cfg.Decay, 1e-9)
- assert.InDelta(t, DefaultJitter, cfg.Jitter, 1e-9)
- assert.InDelta(t, DefaultLocalDiscount, cfg.LocalDiscount, 1e-9)
- assert.InDelta(t, DefaultMaxEvidence, cfg.MaxEvidence, 1e-9)
- assert.Equal(t, 3*time.Second, cfg.FallbackAfter)
- assert.Equal(t, time.Second, cfg.SyncTimeout)
- assert.NotEmpty(t, cfg.NodeID)
- assert.NotZero(t, cfg.Seed)
- assert.NotNil(t, cfg.Now)
-}
-
-func TestConfig_UnseededReplicasDoNotShareJitter(t *testing.T) {
- // Seeding a fleet identically would synchronise the very jitter that
- // exists to keep it from arriving at the store in lockstep.
- seeds := make(map[uint64]struct{})
- for range 20 {
- cfg := validConfig()
- require.NoError(t, cfg.validate())
- seeds[cfg.Seed] = struct{}{}
- }
-
- assert.Greater(t, len(seeds), 15)
-}
-
-func TestConfig_ExplicitValuesSurviveValidation(t *testing.T) {
- cfg := validConfig()
- cfg.Window = 3
- cfg.Decay = 1
- cfg.Jitter = 0.25
- cfg.MaxEvidence = -1
- cfg.NodeID = "chosen"
- cfg.FallbackAfter = time.Minute
-
- require.NoError(t, cfg.validate())
-
- assert.Equal(t, 3, cfg.Window)
- assert.InDelta(t, 1.0, cfg.Decay, 1e-9)
- assert.InDelta(t, 0.25, cfg.Jitter, 1e-9)
- assert.InDelta(t, -1.0, cfg.MaxEvidence, 1e-9, "a negative cap disables it and must not be defaulted away")
- assert.Equal(t, "chosen", cfg.NodeID)
- assert.Equal(t, time.Minute, cfg.FallbackAfter)
-}
-
-func TestConfig_DerivedTTLsOutliveTheirWindow(t *testing.T) {
- cfg := validConfig()
- cfg.Window = 10
- require.NoError(t, cfg.validate())
-
- // A counter TTL shorter than the window the leader reads back would leave
- // holes in it, and the fleet would silently decide on less evidence than
- // it was configured for.
- assert.Greater(t, cfg.counterTTL(), time.Duration(cfg.Window)*cfg.CoordinationEpoch)
- assert.Positive(t, cfg.leaderTTL())
- assert.Greater(t, cfg.decisionTTL(), cfg.CoordinationEpoch,
- "a decision must outlive its bucket so a replica that ticks late still finds it")
-}
diff --git a/bandit/coordinate.go b/bandit/coordinate.go
deleted file mode 100644
index a7e065a..0000000
--- a/bandit/coordinate.go
+++ /dev/null
@@ -1,271 +0,0 @@
-// This file holds everything that runs on the coordination goroutine: the sync
-// loop and the one round trip it makes per coordination epoch. It is separate
-// from distributed.go, which holds the surface the cache itself calls, because
-// the two run under completely different constraints - nothing here is allowed
-// to be slow, and nothing there is allowed to block at all.
-
-package bandit
-
-import (
- "context"
- "time"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-// coordinate runs the sync loop until Close.
-func (d *Distributed) coordinate() {
- defer d.wg.Done()
-
- timer := time.NewTimer(d.nextInterval())
- defer timer.Stop()
-
- for {
- select {
- case <-d.ctx.Done():
- return
- case <-timer.C:
- d.sync()
- timer.Reset(d.nextInterval())
- }
- }
-}
-
-// nextInterval returns the coordination epoch with jitter applied, so a large
-// fleet spreads its round trips across the epoch instead of arriving together
-// on every boundary. Buckets are assigned by the store's clock, so jitter can
-// never put a replica's counts in the wrong window.
-func (d *Distributed) nextInterval() time.Duration {
- if d.cfg.Jitter <= 0 {
- return d.cfg.CoordinationEpoch
- }
-
- spread := float64(d.cfg.CoordinationEpoch) * d.cfg.Jitter
- offset := (d.rng.Float64()*2 - 1) * spread
-
- return time.Duration(float64(d.cfg.CoordinationEpoch) + offset)
-}
-
-// sync performs one coordination round: publish this replica's counts, then
-// work out what the fleet should be running.
-func (d *Distributed) sync() {
- req, ok := d.drain()
- if !ok {
- // No report has arrived yet, so the measurement regime is unknown and
- // there is nothing to publish or to key by.
- return
- }
-
- ctx, cancel := context.WithTimeout(d.ctx, d.cfg.SyncTimeout)
- defer cancel()
-
- result, err := d.cfg.Store.Sync(ctx, req)
- if err != nil {
- d.recordFailure(err)
- return
- }
-
- decision, decided := result.Decision, result.HasDecision
-
- switch {
- case d.cfg.Mode == ModeSharedPosterior:
- // Every replica reads the window and draws for itself. No leader is
- // elected and the decision key is never written or read.
- decision, decided = d.drawShared(ctx, req.Namespace, result.Bucket)
-
- case result.Leader:
- // This replica won the bucket. It decides for the fleet, and applies
- // its own decision now rather than reading it back a round later.
- if choice, ok := d.leadBucket(ctx, req.Namespace, result.Bucket); ok {
- decision, decided = choice, true
- }
- }
-
- d.applyResult(result.Bucket, decision, decided)
-}
-
-// drain takes everything buffered since the last sync and builds the request.
-// The buffer is cleared whether or not the sync goes on to succeed: counts
-// that missed their window are not worth carrying forward.
-func (d *Distributed) drain() (SyncRequest, bool) {
- d.mu.Lock()
- defer d.mu.Unlock()
-
- if !d.haveShape {
- return SyncRequest{}, false
- }
-
- counts := make([]ArmCounts, 0, len(d.pending))
- for key, stats := range d.pending {
- counts = append(counts, ArmCounts{
- Policy: key.Policy,
- Role: key.Role,
- Hits: stats.Hits,
- Misses: stats.Misses,
- })
- }
- clear(d.pending)
-
- return SyncRequest{
- Namespace: d.namespace,
- NodeID: d.cfg.NodeID,
- Counts: counts,
- EpochMillis: d.cfg.CoordinationEpoch.Milliseconds(),
- CounterTTL: d.cfg.counterTTL(),
- LeaderTTL: d.cfg.leaderTTL(),
- Lead: d.cfg.Mode == ModeLeader,
- }, true
-}
-
-// readPooled reads the fleet's recent buckets and returns the pooled, decayed,
-// evidence-capped posterior per arm.
-//
-// The window stops one bucket short of the current one: the fleet is still
-// writing into that bucket, so including it would weight whichever replicas
-// happened to have synced already.
-func (d *Distributed) readPooled(
- ctx context.Context,
- namespace string,
- bucket Bucket,
-) (map[ascache.PolicyType]weighted, bool) {
- newest := bucket - 1
- first := newest - Bucket(d.cfg.Window) + 1
-
- window, err := d.cfg.Store.Window(ctx, namespace, first, newest)
- if err != nil {
- d.recordFailure(err)
- return nil, false
- }
-
- pooled := aggregate(window, newest, d.cfg.Decay, d.cfg.Evidence)
- capEvidence(pooled, d.cfg.MaxEvidence)
- d.recordFleet(pooled)
-
- return pooled, true
-}
-
-// leadBucket decides for the fleet and publishes the decision. A failure
-// anywhere leaves the bucket without one, which every replica reads as "keep
-// doing what you are doing".
-func (d *Distributed) leadBucket(
- ctx context.Context,
- namespace string,
- bucket Bucket,
-) (ascache.PolicyType, bool) {
- d.mu.Lock()
- d.state.leaderships++
- d.mu.Unlock()
-
- pooled, ok := d.readPooled(ctx, namespace, bucket)
- if !ok {
- return ascache.Undefined, false
- }
-
- choice := draw(d.rng, pooled)
- if choice == ascache.Undefined {
- // Nothing in the window: the fleet has published no evidence yet.
- return ascache.Undefined, false
- }
-
- // The decision in force is what the store reports, not what was drawn: if
- // another replica somehow published first, the leader follows the fleet
- // rather than being the one machine running something else.
- inForce, err := d.cfg.Store.Decide(ctx, namespace, bucket, choice, d.cfg.decisionTTL())
- if err != nil {
- d.recordFailure(err)
- return ascache.Undefined, false
- }
-
- return inForce, inForce != ascache.Undefined
-}
-
-// drawShared reads the fleet's window and draws this replica's own selection
-// from it. Replicas share the evidence but not the draw, so they explore
-// independently and may run different policies at the same time.
-func (d *Distributed) drawShared(
- ctx context.Context,
- namespace string,
- bucket Bucket,
-) (ascache.PolicyType, bool) {
- pooled, ok := d.readPooled(ctx, namespace, bucket)
- if !ok {
- return ascache.Undefined, false
- }
-
- choice := draw(d.rng, pooled)
-
- return choice, choice != ascache.Undefined
-}
-
-// applyResult folds a successful sync into the selection and the observable
-// state.
-func (d *Distributed) applyResult(bucket Bucket, decision ascache.PolicyType, decided bool) {
- d.mu.Lock()
- defer d.mu.Unlock()
-
- d.state.lastSync = d.cfg.Now()
- d.state.lastBucket = bucket
- d.state.lastErr = nil
- d.state.syncs++
- d.state.fallback = false
-
- switch {
- case !decided:
- // No decision this round - the leader has not published yet, or the
- // window was empty. Keeping the previous selection is the mode's one
- // coordination epoch of built-in staleness, and it is bounded: the
- // decision will be there on the next sync.
-
- case !d.knownArmLocked(decision):
- // Something is publishing decisions for a policy this cache does not
- // have. The fingerprint in the namespace is supposed to make that
- // impossible, so this is a real misconfiguration rather than a race:
- // refuse it, count it, and keep serving.
- d.state.rejected++
-
- default:
- d.state.decisions++
- d.selection.Store(uint64(decision))
- }
-}
-
-func (d *Distributed) knownArmLocked(policy ascache.PolicyType) bool {
- _, ok := d.arms[policy]
-
- return ok
-}
-
-// recordFleet stores the pooled posterior for Snapshot to report.
-func (d *Distributed) recordFleet(pooled map[ascache.PolicyType]weighted) {
- d.mu.Lock()
- defer d.mu.Unlock()
-
- d.state.fleet = pooled
-}
-
-// recordFailure notes a failed round trip and, once the store has been
-// unreachable for longer than FallbackAfter, hands selection to the local
-// bandit.
-func (d *Distributed) recordFailure(err error) {
- d.mu.Lock()
- defer d.mu.Unlock()
-
- d.state.lastErr = err
- d.state.syncFailures++
-
- // A replica that has never synced falls back immediately rather than
- // waiting out a grace period measured from a sync that never happened.
- stale := d.state.lastSync.IsZero() ||
- d.cfg.Now().Sub(d.state.lastSync) > d.cfg.FallbackAfter
- if !stale {
- return
- }
-
- d.state.fallback = true
-
- choice := d.local.SelectPolicy()
- if choice == ascache.Undefined || !d.knownArmLocked(choice) {
- return
- }
- d.selection.Store(uint64(choice))
-}
diff --git a/bandit/distributed.go b/bandit/distributed.go
deleted file mode 100644
index 683e3c9..0000000
--- a/bandit/distributed.go
+++ /dev/null
@@ -1,235 +0,0 @@
-package bandit
-
-import (
- "context"
- "math/rand/v2"
- "sync"
- "sync/atomic"
- "time"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-var (
- _ ascache.Bandit = (*Distributed)(nil)
- _ ascache.EpochBandit = (*Distributed)(nil)
-)
-
-// Distributed pools every replica's measurements through a shared store, so a
-// fleet of caches selects on the fleet's evidence rather than on each
-// replica's own.
-//
-// It exists for the case the README gives as a reason not to use this library
-// at all: a cache that sees too few requests per epoch to tell its arms apart.
-// That is usually not a property of the traffic but of how it was divided - a
-// fleet of a hundred replicas each sees a hundredth of it. Pooling puts the
-// evidence back together without moving the caches.
-//
-// # Nothing here runs on the cache's path
-//
-// RecordEpoch folds numbers into a buffer and returns; SelectPolicy is an
-// atomic load. Both are called by the cache while it holds its write lock,
-// where a round trip to a store would stall every Get in the process for its
-// duration - and Go's RWMutex queues readers behind a waiting writer, so a
-// store that hangs would hang the cache. All I/O happens on this type's own
-// goroutine, once per coordination epoch.
-//
-// # When the store is unreachable
-//
-// Selection falls back to a local Thompson bandit fed by this replica's own
-// reports, which is exactly the behaviour of a cache that was never
-// distributed. Nothing fails and nothing blocks; [Distributed.Snapshot]
-// reports that it happened. Counts measured during the outage are discarded
-// rather than replayed on recovery: evidence that arrives in the wrong window
-// is worse than no evidence, because the whole scheme rests on recent buckets
-// describing recent traffic.
-type Distributed struct {
- cfg Config
-
- // local is the fallback, and it is fed on every report rather than only
- // during an outage - a bandit that started learning at the moment it was
- // needed would spend the outage exploring from scratch.
- local *Thompson
-
- // selection is what SelectPolicy returns, held as the numeric value of a
- // PolicyType. It is written only by the coordination goroutine and read on
- // the cache's epoch path, so it is atomic rather than mutex-guarded:
- // SelectPolicy must never wait on the goroutine, which may be
- // mid-round-trip.
- selection atomic.Uint64
-
- // rng is confined to the coordination goroutine: jitter and, under
- // ModeSharedPosterior, this replica's own draw. It is deliberately not
- // shared with local, which has its own.
- rng *rand.Rand
-
- mu sync.Mutex
- // pending accumulates what the cache has reported since the last sync.
- pending map[ArmKey]ascache.PolicyStats
- // shape is the measurement regime the reports describe and namespace is
- // the fingerprinted namespace derived from it. Both are set by the first
- // report and updated if the cache's shape ever changes - a resize moves
- // this replica to a different namespace, because its numbers stopped being
- // comparable with the fleet's.
- shape regime
- haveShape bool
- namespace string
- // arms is the set of policies this cache actually has. A decision naming
- // anything else is refused: it means something with a different build or
- // configuration is publishing into this namespace.
- arms map[ascache.PolicyType]struct{}
- state state
-
- // ctx is cancelled by Close, and every round trip derives its context from
- // it. Without that a Close arriving while a sync is in flight would wait
- // out the whole SyncTimeout against a store that has stopped answering -
- // so shutting down behind an unreachable store would take as long as the
- // store was allowed to take.
- ctx context.Context
- cancel context.CancelFunc
- wg sync.WaitGroup
- stopOnce sync.Once
-}
-
-// state is the observable part, guarded by mu.
-type state struct {
- lastSync time.Time
- lastBucket Bucket
- lastErr error
- syncs int64
- syncFailures int64
- leaderships int64
- decisions int64
- rejected int64
- fallback bool
- fleet map[ascache.PolicyType]weighted
-}
-
-// NewDistributed starts a distributed bandit and its coordination goroutine.
-// Callers must call Close to stop it.
-func NewDistributed(cfg Config) (*Distributed, error) {
- d, err := newDistributed(cfg)
- if err != nil {
- return nil, err
- }
-
- d.wg.Add(1)
- go d.coordinate()
-
- return d, nil
-}
-
-// newDistributed builds the bandit without starting its goroutine, so a test
-// can drive sync rounds itself instead of racing a timer.
-func newDistributed(cfg Config) (*Distributed, error) {
- if err := cfg.validate(); err != nil {
- return nil, err
- }
-
- ctx, cancel := context.WithCancel(context.Background())
-
- d := &Distributed{
- cfg: cfg,
- local: NewThompson(cfg.LocalDiscount, cfg.Seed),
- pending: make(map[ArmKey]ascache.PolicyStats),
- arms: make(map[ascache.PolicyType]struct{}),
- //nolint:gosec // deliberate: a seeded, reproducible source, not a secret
- rng: rand.New(rand.NewPCG(cfg.Seed, cfg.Seed^0x2545f4914f6cdd1d)),
- ctx: ctx,
- cancel: cancel,
- }
- d.selection.Store(uint64(ascache.Undefined))
-
- return d, nil
-}
-
-// RecordStats folds a single arm's report into the buffer.
-//
-// The cache calls RecordEpoch instead, since this type implements
-// [ascache.EpochBandit]. This exists so a Distributed can still stand in
-// wherever a plain [ascache.Bandit] is expected; the role is unknowable on
-// this path, so counts are attributed to RoleShadow.
-func (d *Distributed) RecordStats(stats ascache.ShadowStats) {
- d.local.RecordStats(stats)
-
- d.mu.Lock()
- defer d.mu.Unlock()
-
- d.arms[stats.Policy] = struct{}{}
- d.addLocked(ArmKey{Policy: stats.Policy, Role: RoleShadow}, stats)
-}
-
-// RecordEpoch folds one reporting epoch into the buffer and into the local
-// fallback bandit. It performs no I/O and holds one uncontended mutex for the
-// length of a few map writes, because the cache is holding its write lock
-// while it runs.
-func (d *Distributed) RecordEpoch(report ascache.EpochReport) {
- shape := regimeOf(report)
-
- for _, stats := range report.Stats {
- d.local.RecordStats(stats)
- }
-
- d.mu.Lock()
- defer d.mu.Unlock()
-
- if !d.haveShape || !d.shape.equal(shape) {
- // The cache's measurement regime changed - a Resize, most likely. Its
- // numbers are no longer comparable with what it published before, nor
- // with a fleet still running the old shape, so it moves to a different
- // namespace and starts accumulating there. Whatever was buffered was
- // measured under the old regime and goes with it.
- d.shape = shape
- d.haveShape = true
- d.namespace = scopedNamespace(d.cfg.Namespace, shape)
- clear(d.pending)
-
- d.arms = make(map[ascache.PolicyType]struct{}, len(report.Stats))
- for _, stats := range report.Stats {
- d.arms[stats.Policy] = struct{}{}
- }
- }
-
- for _, stats := range report.Stats {
- role := RoleShadow
- if stats.Policy == report.Active {
- role = RoleActive
- }
-
- d.addLocked(ArmKey{Policy: stats.Policy, Role: role}, stats)
- }
-}
-
-func (d *Distributed) addLocked(key ArmKey, stats ascache.ShadowStats) {
- counts := d.pending[key]
- counts.Hits += stats.Hits
- counts.Misses += stats.Misses
- d.pending[key] = counts
-}
-
-// SelectPolicy returns the policy the fleet has settled on, or - while the
-// store is unreachable - the one this replica's own evidence favours. It
-// returns [ascache.Undefined] until the first coordination round completes,
-// which the cache reads as "no change".
-//
-// It is a single atomic load. The answer changes at the coordination cadence
-// rather than the cache's epoch cadence, so a cache on a 50ms epoch and a
-// bandit on a 1s one will see the same answer twenty times over. That is the
-// design: a fleet-wide decision moves at the fleet's pace.
-func (d *Distributed) SelectPolicy() ascache.PolicyType {
- // Only a PolicyType is ever stored here, by this type, so the round trip
- // through uint64 is lossless.
- return ascache.PolicyType(d.selection.Load())
-}
-
-// Close stops the coordination goroutine and waits for it to exit, cancelling
-// any round trip in flight. It is idempotent. It does not close the store,
-// which the caller supplied and may still be using.
-func (d *Distributed) Close() error {
- d.stopOnce.Do(func() {
- d.cancel()
- d.wg.Wait()
- })
-
- return nil
-}
diff --git a/bandit/distributed_test.go b/bandit/distributed_test.go
deleted file mode 100644
index caeffe5..0000000
--- a/bandit/distributed_test.go
+++ /dev/null
@@ -1,673 +0,0 @@
-package bandit
-
-import (
- "context"
- "errors"
- "sync"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-// replica is one simulated cache: a bandit driven by hand, plus the traffic
-// pattern its arms measure.
-type replica struct {
- bandit *Distributed
- active ascache.PolicyType
-}
-
-// report feeds one epoch's measurements, expressed as a hit rate per arm.
-func (r *replica) report(t *testing.T, requests int64, rates map[ascache.PolicyType]float64) {
- t.Helper()
-
- arms := make([]ascache.PolicyType, 0, len(rates))
- for policy := range rates {
- arms = append(arms, policy)
- }
- // EpochReport documents its Stats as PolicyType-ordered, and the regime
- // fingerprint depends on that order, so a test must honour it too.
- sortPolicies(arms)
-
- stats := make([]ascache.ShadowStats, 0, len(arms))
- for _, policy := range arms {
- hits := int64(float64(requests) * rates[policy])
- stats = append(stats, ascache.ShadowStats{
- Policy: policy,
- Hits: hits,
- Misses: requests - hits,
- })
- }
-
- r.bandit.RecordEpoch(ascache.EpochReport{
- Active: r.active,
- Stats: stats,
- Capacity: 1000,
- SampleRate: 1,
- })
-}
-
-func sortPolicies(arms []ascache.PolicyType) {
- for i := 1; i < len(arms); i++ {
- for j := i; j > 0 && arms[j] < arms[j-1]; j-- {
- arms[j], arms[j-1] = arms[j-1], arms[j]
- }
- }
-}
-
-// fleet builds n replicas sharing one store, each with its own bandit, and
-// with the coordination goroutine left unstarted so the test drives sync
-// rounds itself.
-func fleet(t *testing.T, n int, store Store, clock *testClock, tune func(*Config)) []*replica {
- t.Helper()
-
- replicas := make([]*replica, 0, n)
- for i := range n {
- cfg := Config{
- Store: store,
- Namespace: "test",
- NodeID: string(rune('a' + i)),
- CoordinationEpoch: testEpoch,
- Seed: uint64(i + 1),
- Now: clock.now,
- Jitter: 0,
- }
- if tune != nil {
- tune(&cfg)
- }
-
- bandit, err := newDistributed(cfg)
- require.NoError(t, err)
- t.Cleanup(func() { _ = bandit.Close() })
-
- replicas = append(replicas, &replica{bandit: bandit, active: ascache.LRU})
- }
-
- return replicas
-}
-
-func newFleetStore(t *testing.T) (*MemStore, *testClock) {
- t.Helper()
-
- clock := newTestClock()
- store := NewMemStore()
- store.SetClock(clock.now)
- t.Cleanup(func() { _ = store.Close() })
-
- return store, clock
-}
-
-// ---------------------------------------------------------------------------
-// The contract that matters most: nothing here touches the network
-// ---------------------------------------------------------------------------
-
-// blockingStore blocks forever on every call. A bandit that did its I/O on the
-// cache's path would deadlock the cache against it; this one does not, so the
-// test simply completes.
-type blockingStore struct {
- entered chan struct{}
- once sync.Once
-}
-
-func (s *blockingStore) enter(ctx context.Context) error {
- s.once.Do(func() { close(s.entered) })
- <-ctx.Done()
-
- return ctx.Err()
-}
-
-func (s *blockingStore) Sync(ctx context.Context, _ SyncRequest) (SyncResult, error) {
- return SyncResult{}, s.enter(ctx)
-}
-
-func (s *blockingStore) Window(ctx context.Context, _ string, _, _ Bucket) ([]WindowCounts, error) {
- return nil, s.enter(ctx)
-}
-
-func (s *blockingStore) Decide(
- ctx context.Context,
- _ string,
- _ Bucket,
- _ ascache.PolicyType,
- _ time.Duration,
-) (ascache.PolicyType, error) {
- return ascache.Undefined, s.enter(ctx)
-}
-
-func (s *blockingStore) Close() error { return nil }
-
-func TestDistributed_RecordAndSelectNeverWaitOnTheStore(t *testing.T) {
- // The cache calls both of these while holding its write lock, so a round
- // trip on either path stalls every Get in the process. This is the
- // invariant the whole design is arranged around.
- store := &blockingStore{entered: make(chan struct{})}
- clock := newTestClock()
-
- bandit, err := newDistributed(Config{
- Store: store,
- Namespace: "test",
- CoordinationEpoch: testEpoch,
- SyncTimeout: time.Hour,
- Now: clock.now,
- })
- require.NoError(t, err)
- t.Cleanup(func() { _ = bandit.Close() })
-
- r := &replica{bandit: bandit, active: ascache.LRU}
- r.report(t, 100, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.9})
-
- // Start a sync and wait until it is genuinely inside the store and stuck.
- syncing := make(chan struct{})
- go func() {
- defer close(syncing)
- bandit.sync()
- }()
- <-store.entered
-
- done := make(chan struct{})
- go func() {
- defer close(done)
- for range 1000 {
- r.report(t, 10, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.9})
- bandit.SelectPolicy()
- }
- }()
-
- select {
- case <-done:
- case <-time.After(10 * time.Second):
- t.Fatal("RecordEpoch or SelectPolicy blocked behind an in-flight store call")
- }
-
- require.NoError(t, bandit.Close())
- <-syncing
-}
-
-// ---------------------------------------------------------------------------
-// Leader mode
-// ---------------------------------------------------------------------------
-
-func TestDistributed_FleetConvergesOnTheBetterArm(t *testing.T) {
- store, clock := newFleetStore(t)
- replicas := fleet(t, 8, store, clock, nil)
-
- // Each replica sees a twentieth of the traffic: 50 requests an epoch is
- // too thin to rank arms locally, which is the whole reason to pool.
- rates := map[ascache.PolicyType]float64{ascache.LRU: 0.40, ascache.TinyLFU: 0.75}
-
- for range 12 {
- for _, r := range replicas {
- r.report(t, 50, rates)
- r.bandit.sync()
- }
- clock.advance(testEpoch)
- }
-
- for _, r := range replicas {
- assert.Equal(t, ascache.TinyLFU, r.bandit.SelectPolicy(),
- "every replica should be running the arm the fleet's evidence favours")
- }
-}
-
-func TestDistributed_FleetRunsOnePolicyAtATime(t *testing.T) {
- store, clock := newFleetStore(t)
- replicas := fleet(t, 6, store, clock, nil)
-
- // Arms close enough together that exploration keeps moving the decision.
- rates := map[ascache.PolicyType]float64{ascache.LRU: 0.50, ascache.TinyLFU: 0.51}
-
- for range 15 {
- for _, r := range replicas {
- r.report(t, 200, rates)
- r.bandit.sync()
- }
- clock.advance(testEpoch)
-
- // Everyone syncs again in the new bucket so the leader's decision has
- // reached them all.
- for _, r := range replicas {
- r.bandit.sync()
- }
-
- selections := make(map[ascache.PolicyType]int)
- for _, r := range replicas {
- selections[r.bandit.SelectPolicy()]++
- }
- assert.Len(t, selections, 1,
- "under leader election the fleet applies one decision: %v", selections)
- }
-}
-
-func TestDistributed_ExactlyOneReplicaLeadsEachBucket(t *testing.T) {
- store, clock := newFleetStore(t)
- replicas := fleet(t, 10, store, clock, nil)
-
- const buckets = 6
- for range buckets {
- for _, r := range replicas {
- r.report(t, 100, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.6})
- r.bandit.sync()
- }
- clock.advance(testEpoch)
- }
-
- total := int64(0)
- for _, r := range replicas {
- total += r.bandit.Snapshot().Leaderships
- }
- assert.Equal(t, int64(buckets), total)
-}
-
-func TestDistributed_LeaderAppliesItsOwnDecisionImmediately(t *testing.T) {
- store, clock := newFleetStore(t)
- replicas := fleet(t, 3, store, clock, nil)
-
- for range 4 {
- for _, r := range replicas {
- r.report(t, 500, map[ascache.PolicyType]float64{ascache.LRU: 0.2, ascache.TinyLFU: 0.9})
- r.bandit.sync()
- }
- clock.advance(testEpoch)
- }
-
- // Whichever replica led last must already be on the decision it published,
- // not waiting a further round to read it back.
- led := false
- for _, r := range replicas {
- snapshot := r.bandit.Snapshot()
- if snapshot.Leaderships == 0 {
- continue
- }
- led = true
- assert.Positive(t, snapshot.Decisions, "%s led but applied nothing", snapshot.NodeID)
- }
- require.True(t, led, "no replica ever led")
-}
-
-// ---------------------------------------------------------------------------
-// Shared-posterior mode
-// ---------------------------------------------------------------------------
-
-func TestDistributed_SharedPosteriorSelectsWithoutALeader(t *testing.T) {
- store, clock := newFleetStore(t)
- replicas := fleet(t, 6, store, clock, func(cfg *Config) {
- cfg.Mode = ModeSharedPosterior
- })
-
- rates := map[ascache.PolicyType]float64{ascache.LRU: 0.30, ascache.TinyLFU: 0.80}
-
- for range 10 {
- for _, r := range replicas {
- r.report(t, 100, rates)
- r.bandit.sync()
- }
- clock.advance(testEpoch)
- }
-
- for _, r := range replicas {
- snapshot := r.bandit.Snapshot()
- assert.Zero(t, snapshot.Leaderships, "shared-posterior mode elects nobody")
- assert.Equal(t, ascache.TinyLFU.String(), snapshot.Selection)
- assert.NotEmpty(t, snapshot.Fleet, "every replica reads the window for itself")
- }
-}
-
-func TestDistributed_SharedPosteriorRejectsShadowOnlyUnderLeader(t *testing.T) {
- _, err := NewDistributed(Config{
- Store: NewMemStore(),
- Namespace: "test",
- CoordinationEpoch: testEpoch,
- Mode: ModeLeader,
- Evidence: EvidenceShadowOnly,
- })
- assert.ErrorIs(t, err, ErrShadowOnlyUnderLeader)
-}
-
-func TestDistributed_ShadowOnlyIgnoresTheActiveArmsFlattery(t *testing.T) {
- store, clock := newFleetStore(t)
- replicas := fleet(t, 4, store, clock, func(cfg *Config) {
- cfg.Mode = ModeSharedPosterior
- cfg.Evidence = EvidenceShadowOnly
- })
-
- // LRU is the active arm everywhere and reports far better than it measures
- // as a shadow - the systematic role gap, exaggerated. Shadow-only evidence
- // should see straight through it.
- for range 10 {
- for _, r := range replicas {
- r.bandit.RecordEpoch(ascache.EpochReport{
- Active: ascache.LRU,
- Stats: []ascache.ShadowStats{
- {Policy: ascache.LRU, Hits: 950, Misses: 50},
- {Policy: ascache.TinyLFU, Hits: 700, Misses: 300},
- },
- Capacity: 1000,
- SampleRate: 1,
- })
- r.bandit.sync()
- }
- clock.advance(testEpoch)
- }
-
- for _, r := range replicas {
- snapshot := r.bandit.Snapshot()
- assert.Equal(t, ascache.TinyLFU.String(), snapshot.Selection,
- "the incumbent's active-role numbers must not decide this")
- }
-}
-
-// ---------------------------------------------------------------------------
-// Failure and recovery
-// ---------------------------------------------------------------------------
-
-func TestDistributed_FallsBackToLocalWhenTheStoreIsUnreachable(t *testing.T) {
- store, clock := newFleetStore(t)
- replicas := fleet(t, 3, store, clock, nil)
-
- rates := map[ascache.PolicyType]float64{ascache.LRU: 0.2, ascache.TinyLFU: 0.9}
- for range 3 {
- for _, r := range replicas {
- r.report(t, 200, rates)
- r.bandit.sync()
- }
- clock.advance(testEpoch)
- }
-
- subject := replicas[0]
- require.False(t, subject.bandit.Snapshot().Fallback)
-
- store.Fail(errors.New("connection refused"))
-
- // Inside the grace period the replica holds the fleet's last decision
- // rather than reacting to one failed round trip.
- subject.report(t, 200, rates)
- subject.bandit.sync()
- assert.False(t, subject.bandit.Snapshot().Fallback, "one failure is not an outage")
-
- clock.advance(4 * testEpoch)
- subject.report(t, 200, rates)
- subject.bandit.sync()
-
- snapshot := subject.bandit.Snapshot()
- assert.True(t, snapshot.Fallback)
- assert.Contains(t, snapshot.LastError, "connection refused")
- assert.Equal(t, ascache.TinyLFU.String(), snapshot.Selection,
- "the local bandit has been learning all along, so it does not start from nothing")
-}
-
-func TestDistributed_RecoversWhenTheStoreComesBack(t *testing.T) {
- store, clock := newFleetStore(t)
- replicas := fleet(t, 3, store, clock, nil)
- subject := replicas[0]
-
- store.Fail(errors.New("down"))
- subject.report(t, 100, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.6})
- subject.bandit.sync()
- require.True(t, subject.bandit.Snapshot().Fallback)
-
- store.Fail(nil)
- for range 4 {
- for _, r := range replicas {
- r.report(t, 200, map[ascache.PolicyType]float64{ascache.LRU: 0.2, ascache.TinyLFU: 0.9})
- r.bandit.sync()
- }
- clock.advance(testEpoch)
- }
-
- snapshot := subject.bandit.Snapshot()
- assert.False(t, snapshot.Fallback)
- assert.Positive(t, snapshot.Syncs)
- assert.Equal(t, ascache.TinyLFU.String(), snapshot.Selection)
-}
-
-func TestDistributed_OutageCountsAreDiscardedNotReplayed(t *testing.T) {
- store, clock := newFleetStore(t)
- subject := fleet(t, 1, store, clock, nil)[0]
-
- store.Fail(errors.New("down"))
- for range 5 {
- subject.report(t, 1000, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.6})
- subject.bandit.sync()
- }
- store.Fail(nil)
-
- subject.report(t, 10, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.6})
- result, err := store.Sync(t.Context(), testSyncRequest(subject.bandit.namespace, "probe"))
- require.NoError(t, err)
- subject.bandit.sync()
-
- window, err := store.Window(t.Context(), subject.bandit.namespace, result.Bucket, result.Bucket)
- require.NoError(t, err)
- require.Len(t, window, 1)
-
- published := int64(0)
- for _, stats := range window[0].Arms {
- published += stats.Hits + stats.Misses
- }
- // 10 requests across two arms. The 5000 measured during the outage stayed
- // out: evidence that missed its window would land in the wrong bucket and
- // describe traffic that is no longer current.
- assert.Equal(t, int64(20), published)
-}
-
-func TestDistributed_NeverSyncedFallsBackWithoutWaiting(t *testing.T) {
- store, clock := newFleetStore(t)
- store.Fail(errors.New("down from the start"))
-
- subject := fleet(t, 1, store, clock, func(cfg *Config) {
- cfg.FallbackAfter = time.Hour
- })[0]
-
- subject.report(t, 500, map[ascache.PolicyType]float64{ascache.LRU: 0.2, ascache.TinyLFU: 0.9})
- subject.bandit.sync()
-
- snapshot := subject.bandit.Snapshot()
- assert.True(t, snapshot.Fallback,
- "a grace period measured from a sync that never happened would never expire")
- assert.Equal(t, ascache.TinyLFU.String(), snapshot.Selection)
-}
-
-func TestDistributed_RefusesADecisionForAnArmItDoesNotHave(t *testing.T) {
- store, clock := newFleetStore(t)
- subject := fleet(t, 1, store, clock, nil)[0]
-
- subject.report(t, 100, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.6})
- subject.bandit.sync()
- before := subject.bandit.SelectPolicy()
-
- // Something else takes the next bucket's leadership and publishes a
- // decision for a policy this cache was not built with. The namespace
- // fingerprint is supposed to prevent it, so reaching here means a genuine
- // misconfiguration - and applying it would hand the cache a policy it
- // cannot look up.
- clock.advance(testEpoch)
- strangerReq := testSyncRequest(subject.bandit.namespace, "stranger")
- strangerReq.Lead = true
- result, err := store.Sync(t.Context(), strangerReq)
- require.NoError(t, err)
- require.True(t, result.Leader)
-
- inForce, err := store.Decide(t.Context(), subject.bandit.namespace, result.Bucket, ascache.ARC, time.Minute)
- require.NoError(t, err)
- require.Equal(t, ascache.ARC, inForce)
-
- subject.report(t, 100, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.6})
- subject.bandit.sync()
-
- snapshot := subject.bandit.Snapshot()
- assert.Positive(t, snapshot.Rejected)
- assert.Equal(t, before, subject.bandit.SelectPolicy())
- assert.NotEqual(t, ascache.ARC.String(), snapshot.Selection)
-}
-
-// ---------------------------------------------------------------------------
-// Regime changes
-// ---------------------------------------------------------------------------
-
-func TestDistributed_ResizeMovesTheReplicaToItsOwnNamespace(t *testing.T) {
- store, clock := newFleetStore(t)
- subject := fleet(t, 1, store, clock, nil)[0]
-
- subject.report(t, 100, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.6})
- before := subject.bandit.Snapshot().Namespace
-
- subject.bandit.RecordEpoch(ascache.EpochReport{
- Active: ascache.LRU,
- Stats: []ascache.ShadowStats{
- {Policy: ascache.LRU, Hits: 1, Misses: 1},
- {Policy: ascache.TinyLFU, Hits: 1, Misses: 1},
- },
- Capacity: 4000, // resized
- SampleRate: 1,
- })
-
- after := subject.bandit.Snapshot().Namespace
- assert.NotEqual(t, before, after,
- "a hit rate measured at 4000 entries says nothing about one measured at 1000")
- assert.Contains(t, after, "test:")
-}
-
-func TestDistributed_ReplicasWithDifferentShapesDoNotPool(t *testing.T) {
- store, clock := newFleetStore(t)
- replicas := fleet(t, 2, store, clock, nil)
-
- replicas[0].report(t, 100, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.6})
- replicas[1].bandit.RecordEpoch(ascache.EpochReport{
- Active: ascache.LRU,
- Stats: []ascache.ShadowStats{
- {Policy: ascache.LRU, Hits: 50, Misses: 50},
- {Policy: ascache.TinyLFU, Hits: 60, Misses: 40},
- },
- Capacity: 100, // a much smaller cache
- SampleRate: 1,
- })
-
- assert.NotEqual(t,
- replicas[0].bandit.Snapshot().Namespace,
- replicas[1].bandit.Snapshot().Namespace)
-}
-
-// ---------------------------------------------------------------------------
-// Lifecycle
-// ---------------------------------------------------------------------------
-
-func TestDistributed_CloseCancelsAnInFlightRoundTrip(t *testing.T) {
- // Close waits for the coordination goroutine, which may be inside a store
- // call. Deriving that call's context from Background rather than from the
- // bandit's own would make shutting down behind an unreachable store take
- // the whole SyncTimeout - an hour here, and in production however long the
- // timeout was set to.
- store := &blockingStore{entered: make(chan struct{})}
-
- bandit, err := newDistributed(Config{
- Store: store,
- Namespace: "test",
- CoordinationEpoch: testEpoch,
- SyncTimeout: time.Hour,
- })
- require.NoError(t, err)
-
- r := &replica{bandit: bandit, active: ascache.LRU}
- r.report(t, 100, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.9})
-
- bandit.wg.Add(1)
- go func() {
- defer bandit.wg.Done()
- bandit.sync()
- }()
- <-store.entered
-
- closed := make(chan error, 1)
- go func() { closed <- bandit.Close() }()
-
- select {
- case err := <-closed:
- assert.NoError(t, err)
- case <-time.After(10 * time.Second):
- t.Fatal("Close waited on an in-flight store call instead of cancelling it")
- }
-}
-
-func TestDistributed_CloseIsIdempotentAndStopsTheGoroutine(t *testing.T) {
- store, _ := newFleetStore(t)
-
- bandit, err := NewDistributed(Config{
- Store: store,
- Namespace: "test",
- CoordinationEpoch: time.Millisecond,
- })
- require.NoError(t, err)
-
- require.NoError(t, bandit.Close())
- require.NoError(t, bandit.Close())
- require.NoError(t, bandit.Close())
-}
-
-func TestDistributed_SyncBeforeAnyReportPublishesNothing(t *testing.T) {
- store, clock := newFleetStore(t)
- subject := fleet(t, 1, store, clock, nil)[0]
-
- // The measurement regime is unknown until the cache reports, so there is
- // nothing to key the counts by.
- subject.bandit.sync()
-
- snapshot := subject.bandit.Snapshot()
- assert.Zero(t, snapshot.Syncs)
- assert.Zero(t, snapshot.SyncFailures)
- assert.Equal(t, ascache.Undefined.String(), snapshot.Selection)
-}
-
-func TestDistributed_SelectPolicyIsUndefinedUntilTheFirstRound(t *testing.T) {
- store, clock := newFleetStore(t)
- subject := fleet(t, 1, store, clock, nil)[0]
-
- subject.report(t, 100, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.6})
-
- // The cache reads Undefined as "no change", which is what stops it acting
- // on a bandit that has not heard from the fleet yet.
- assert.Equal(t, ascache.Undefined, subject.bandit.SelectPolicy())
-}
-
-func TestDistributed_RecordStatsStandsInForAPlainBandit(t *testing.T) {
- store, clock := newFleetStore(t)
- subject := fleet(t, 1, store, clock, nil)[0]
-
- // A Distributed used through the plain Bandit interface still works; it
- // just cannot tell which arm was active.
- for range 3 {
- subject.bandit.RecordStats(ascache.ShadowStats{Policy: ascache.LRU, Hits: 20, Misses: 80})
- subject.bandit.RecordStats(ascache.ShadowStats{Policy: ascache.TinyLFU, Hits: 90, Misses: 10})
- }
-
- assert.Equal(t, ascache.TinyLFU, subject.bandit.local.SelectPolicy())
-}
-
-func TestDistributed_ConcurrentReportsAndSelections(t *testing.T) {
- store, clock := newFleetStore(t)
- subject := fleet(t, 1, store, clock, nil)[0]
-
- var wg sync.WaitGroup
- for range 4 {
- wg.Add(1)
- go func() {
- defer wg.Done()
- for range 200 {
- subject.report(t, 10, map[ascache.PolicyType]float64{ascache.LRU: 0.5, ascache.TinyLFU: 0.6})
- subject.bandit.SelectPolicy()
- }
- }()
- }
- wg.Add(1)
- go func() {
- defer wg.Done()
- for range 50 {
- subject.bandit.sync()
- subject.bandit.Snapshot()
- }
- }()
-
- wg.Wait()
-}
diff --git a/bandit/doc.go b/bandit/doc.go
index a308cf2..9db2ad6 100644
--- a/bandit/doc.go
+++ b/bandit/doc.go
@@ -14,34 +14,8 @@
// and exists as a control: it shows what the adaptive layer achieves with no
// exploration at all.
//
-// # Distributed
-//
-// [Distributed] pools evidence across a fleet of replicas through a shared
-// store, so a cache seeing too little traffic to tell its arms apart can still
-// benefit from selection. Each replica publishes its per-epoch counts; one
-// replica per coordination epoch reads the fleet's aggregate and publishes the
-// decision the others apply.
-//
-// b, err := bandit.NewDistributed(bandit.Config{
-// Store: store, // from .../bandit/redis
-// Namespace: "sessions",
-// CoordinationEpoch: time.Second,
-// })
-// defer b.Close()
-//
-// The store is an interface, not a client: [MemStore] runs a whole fleet in
-// one process for tests, and github.com/sshaplygin/as-cache/bandit/redis backs
-// it with Valkey or Redis.
-//
// # What crosses the wire
//
-// Per-policy hit and miss integers, and a policy name. No cache keys and no
-// cache values ever leave the process.
-//
-// # Cost
-//
-// One round trip per replica per coordination epoch, plus two more for
-// whichever replica is leading. Nothing touches the network on the cache's hot
-// path, or while the cache holds a lock: [Distributed.RecordEpoch] folds
-// numbers into a buffer and [Distributed.SelectPolicy] is an atomic load.
+// Nothing. These bandits are in-process: they see per-policy hit and miss
+// counts and return a policy name.
package bandit
diff --git a/bandit/errors.go b/bandit/errors.go
deleted file mode 100644
index 28d6b58..0000000
--- a/bandit/errors.go
+++ /dev/null
@@ -1,45 +0,0 @@
-package bandit
-
-import "errors"
-
-// ErrNilStore is returned by NewDistributed when Config.Store is nil. There is
-// no useful default: a distributed bandit with nowhere to coordinate is just a
-// local one, and silently becoming one would hide a misconfiguration behind
-// behaviour that looks fine.
-var ErrNilStore = errors.New("bandit: store must not be nil")
-
-// ErrInvalidCoordinationEpoch is returned by NewDistributed when
-// Config.CoordinationEpoch is zero or negative.
-var ErrInvalidCoordinationEpoch = errors.New("bandit: coordination epoch must be positive")
-
-// ErrInvalidWindow is returned by NewDistributed when Config.Window is
-// negative.
-var ErrInvalidWindow = errors.New("bandit: window must not be negative")
-
-// ErrInvalidDecay is returned by NewDistributed when Config.Decay falls
-// outside (0,1].
-var ErrInvalidDecay = errors.New("bandit: decay must be in (0,1]")
-
-// ErrInvalidJitter is returned by NewDistributed when Config.Jitter falls
-// outside [0,0.5). Half an epoch of jitter would let one replica's tick
-// overtake another's by a whole bucket.
-var ErrInvalidJitter = errors.New("bandit: jitter must be in [0,0.5)")
-
-// ErrEmptyNamespace is returned by NewDistributed when Config.Namespace is
-// empty. Two unrelated fleets sharing a store and pooling each other's
-// evidence is a failure with no symptom other than bad decisions, so the name
-// is required rather than defaulted.
-var ErrEmptyNamespace = errors.New("bandit: namespace must not be empty")
-
-// ErrShadowOnlyUnderLeader is returned by NewDistributed for the combination
-// of ModeLeader and EvidenceShadowOnly.
-//
-// Under leader election every replica runs the same active policy, so that
-// policy is nobody's shadow and shadow-only evidence contains nothing about
-// it. Its posterior would stay at the uniform prior forever: it could still be
-// drawn, but only by chance, and never on the strength of how it is actually
-// performing. Discarding evidence for the one arm that is serving all the
-// traffic is not a tuning choice, it is a broken configuration.
-var ErrShadowOnlyUnderLeader = errors.New(
- "bandit: EvidenceShadowOnly cannot be used with ModeLeader: " +
- "the fleet-wide active policy has no shadow measurements anywhere")
diff --git a/bandit/fingerprint.go b/bandit/fingerprint.go
deleted file mode 100644
index 7baf6ed..0000000
--- a/bandit/fingerprint.go
+++ /dev/null
@@ -1,82 +0,0 @@
-package bandit
-
-import (
- "fmt"
- "hash/fnv"
- "strings"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-// regime is the identity of what a cache is measuring: which arms, at what
-// capacity, over how much of the keyspace.
-//
-// Two replicas may only pool evidence when these match. A hit rate is a
-// statement about a specific cache size against a specific substream, and
-// summing one taken at 1000 entries with one taken at 100 produces a number
-// that describes neither cache - it just looks like a hit rate. Nothing in the
-// counts themselves reveals the mismatch, and the resulting decision looks
-// entirely reasonable, so this is checked structurally rather than left to be
-// noticed.
-//
-// Epoch duration is deliberately not part of it. A replica reporting twice as
-// often contributes twice the counts, but at the same rate, and rates are what
-// the comparison is made on. Fleets may be tuned per replica.
-type regime struct {
- arms []ascache.PolicyType
- capacity int
- sampleRate float64
-}
-
-// String renders the regime in the form that gets hashed. It is legible on
-// purpose: when a fleet mysteriously splits into two namespaces, this is the
-// string worth logging.
-func (r regime) String() string {
- names := make([]string, 0, len(r.arms))
- for _, arm := range r.arms {
- names = append(names, arm.String())
- }
-
- // The rate is quantised before it is rendered. It arrives as a float that
- // has been through a capacity-floor calculation, so two replicas
- // configured identically can differ in the last bits and would otherwise
- // fingerprint apart for no reason a caller could ever see.
- return fmt.Sprintf("arms=%s;cap=%d;rate=%.4f",
- strings.Join(names, ","), r.capacity, r.sampleRate)
-}
-
-// fingerprint is the short form appended to the namespace.
-func (r regime) fingerprint() string {
- h := fnv.New64a()
- // Hash.Write never returns an error, per the hash.Hash contract.
- _, _ = h.Write([]byte(r.String()))
-
- return fmt.Sprintf("%012x", h.Sum64()&0xffffffffffff)
-}
-
-// regimeOf reads the measurement regime out of an epoch report. The arms are
-// already sorted by PolicyType - EpochReport documents that ordering - so the
-// fingerprint does not depend on anything the cache could vary between runs.
-func regimeOf(report ascache.EpochReport) regime {
- arms := make([]ascache.PolicyType, 0, len(report.Stats))
- for _, stats := range report.Stats {
- arms = append(arms, stats.Policy)
- }
-
- return regime{
- arms: arms,
- capacity: report.Capacity,
- sampleRate: report.SampleRate,
- }
-}
-
-// equal reports whether two regimes may pool. Sample rates are compared at the
-// precision they are fingerprinted at, so the two can never disagree.
-func (r regime) equal(other regime) bool {
- return r.String() == other.String()
-}
-
-// scopedNamespace is the namespace a replica in this regime coordinates under.
-func scopedNamespace(namespace string, r regime) string {
- return namespace + ":" + r.fingerprint()
-}
diff --git a/bandit/fingerprint_test.go b/bandit/fingerprint_test.go
deleted file mode 100644
index aec5143..0000000
--- a/bandit/fingerprint_test.go
+++ /dev/null
@@ -1,107 +0,0 @@
-package bandit
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-func report(capacity int, rate float64, arms ...ascache.PolicyType) ascache.EpochReport {
- stats := make([]ascache.ShadowStats, 0, len(arms))
- for _, arm := range arms {
- stats = append(stats, ascache.ShadowStats{Policy: arm})
- }
-
- return ascache.EpochReport{Stats: stats, Capacity: capacity, SampleRate: rate}
-}
-
-func TestRegime_SeparatesWhatCannotBePooled(t *testing.T) {
- base := regimeOf(report(1000, 1, ascache.LRU, ascache.TinyLFU))
-
- tests := []struct {
- name string
- other ascache.EpochReport
- }{
- {
- name: "different capacity",
- other: report(100, 1, ascache.LRU, ascache.TinyLFU),
- },
- {
- name: "different sample rate",
- other: report(1000, 0.05, ascache.LRU, ascache.TinyLFU),
- },
- {
- name: "different arms",
- other: report(1000, 1, ascache.LRU, ascache.LFU),
- },
- {
- name: "extra arm",
- other: report(1000, 1, ascache.LRU, ascache.TinyLFU, ascache.LFU),
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- other := regimeOf(tt.other)
-
- assert.False(t, base.equal(other))
- assert.NotEqual(t,
- scopedNamespace("app", base),
- scopedNamespace("app", other))
- })
- }
-}
-
-func TestRegime_IdenticalShapesPool(t *testing.T) {
- left := regimeOf(report(1000, 0.05, ascache.LRU, ascache.TinyLFU))
- right := regimeOf(report(1000, 0.05, ascache.LRU, ascache.TinyLFU))
-
- assert.True(t, left.equal(right))
- assert.Equal(t, scopedNamespace("app", left), scopedNamespace("app", right))
-}
-
-func TestRegime_ToleratesFloatNoiseInTheSampleRate(t *testing.T) {
- // The rate reaches the bandit having been through a capacity-floor
- // calculation, so two replicas configured identically can differ in the
- // last bits. Splitting a fleet over that would be invisible and
- // undiagnosable.
- exact := regimeOf(report(1000, 0.05, ascache.LRU))
- noisy := regimeOf(report(1000, 0.05+1e-12, ascache.LRU))
-
- assert.True(t, exact.equal(noisy))
-}
-
-func TestRegime_DistinguishesRatesThatActuallyDiffer(t *testing.T) {
- // The tolerance must not be so wide that genuinely different sampling
- // regimes pool with each other.
- coarse := regimeOf(report(1000, 0.05, ascache.LRU))
- finer := regimeOf(report(1000, 0.06, ascache.LRU))
-
- assert.False(t, coarse.equal(finer))
-}
-
-func TestRegime_StringIsLegible(t *testing.T) {
- // When a fleet mysteriously splits, this is the string worth logging, so
- // it has to say what actually differs.
- r := regimeOf(report(2048, 0.25, ascache.LRU, ascache.TinyLFU))
-
- assert.Equal(t, "arms=LRU,TinyLFU;cap=2048;rate=0.2500", r.String())
-}
-
-func TestRegime_FingerprintIsStableAndShort(t *testing.T) {
- r := regimeOf(report(1000, 1, ascache.LRU, ascache.TinyLFU))
-
- first := r.fingerprint()
- assert.Len(t, first, 12)
- for range 10 {
- assert.Equal(t, first, r.fingerprint())
- }
-}
-
-func TestScopedNamespace_KeepsTheCallersNameReadable(t *testing.T) {
- r := regimeOf(report(1000, 1, ascache.LRU))
-
- assert.Contains(t, scopedNamespace("sessions", r), "sessions:")
-}
diff --git a/bandit/integration_test.go b/bandit/integration_test.go
deleted file mode 100644
index 2295eea..0000000
--- a/bandit/integration_test.go
+++ /dev/null
@@ -1,373 +0,0 @@
-package bandit
-
-import (
- "strconv"
- "sync"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-// evictOrder selects which end of the insertion order a testPolicy discards.
-type evictOrder int
-
-const (
- // evictOldest is LRU: discard the least recently used entry.
- evictOldest evictOrder = iota + 1
- // evictNewest is MRU: discard the most recently used entry. It is the
- // right policy for a cyclic scan just larger than the cache, which is
- // exactly where LRU serves nothing at all - so the two make an
- // unambiguous pair to test selection with.
- evictNewest
-)
-
-// testPolicy is a small recency-ordered cache. The bandit module depends on
-// the core module alone, so it cannot reach for the ready-made policies; this
-// is enough to drive a real AdaptiveCache.
-type testPolicy struct {
- mu sync.Mutex
- entries map[string]int
- order []string
- capacity int
- evict evictOrder
- policy ascache.PolicyType
- hits int64
- misses int64
-}
-
-func newTestPolicy(policy ascache.PolicyType, capacity int, order evictOrder) *testPolicy {
- return &testPolicy{
- entries: make(map[string]int, capacity),
- capacity: capacity,
- evict: order,
- policy: policy,
- }
-}
-
-func (p *testPolicy) touchLocked(key string) {
- for i, existing := range p.order {
- if existing == key {
- p.order = append(p.order[:i], p.order[i+1:]...)
- break
- }
- }
- p.order = append(p.order, key)
-}
-
-func (p *testPolicy) Add(key string, value int) bool {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- _, existed := p.entries[key]
- if !existed && p.capacity > 0 && len(p.entries) >= p.capacity {
- victim := p.order[0]
- if p.evict == evictNewest {
- victim = p.order[len(p.order)-1]
- }
- delete(p.entries, victim)
- p.removeFromOrderLocked(victim)
- }
-
- if p.capacity <= 0 {
- return false
- }
-
- p.entries[key] = value
- p.touchLocked(key)
-
- return !existed
-}
-
-func (p *testPolicy) removeFromOrderLocked(key string) {
- for i, existing := range p.order {
- if existing == key {
- p.order = append(p.order[:i], p.order[i+1:]...)
- return
- }
- }
-}
-
-func (p *testPolicy) Get(key string) (int, bool) {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- value, ok := p.entries[key]
- if ok {
- p.hits++
- p.touchLocked(key)
- } else {
- p.misses++
- }
-
- return value, ok
-}
-
-func (p *testPolicy) Peek(key string) (int, bool) {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- value, ok := p.entries[key]
-
- return value, ok
-}
-
-func (p *testPolicy) Contains(key string) bool {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- _, ok := p.entries[key]
-
- return ok
-}
-
-func (p *testPolicy) Remove(key string) bool {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- _, ok := p.entries[key]
- delete(p.entries, key)
- p.removeFromOrderLocked(key)
-
- return ok
-}
-
-func (p *testPolicy) Purge() {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- p.entries = make(map[string]int, p.capacity)
- p.order = nil
-}
-
-func (p *testPolicy) Keys() []string {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- keys := make([]string, len(p.order))
- copy(keys, p.order)
-
- return keys
-}
-
-func (p *testPolicy) Values() []int {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- values := make([]int, 0, len(p.order))
- for _, key := range p.order {
- values = append(values, p.entries[key])
- }
-
- return values
-}
-
-func (p *testPolicy) Len() int {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- return len(p.entries)
-}
-
-func (p *testPolicy) Cap() int {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- return p.capacity
-}
-
-func (p *testPolicy) Resize(size int) int {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- p.capacity = size
-
- evicted := 0
- for len(p.entries) > size && len(p.order) > 0 {
- victim := p.order[0]
- delete(p.entries, victim)
- p.order = p.order[1:]
- evicted++
- }
-
- return evicted
-}
-
-func (p *testPolicy) GetStats() ascache.PolicyStats {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- return ascache.PolicyStats{Hits: p.hits, Misses: p.misses}
-}
-
-func (p *testPolicy) ResetStats() {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- p.hits, p.misses = 0, 0
-}
-
-func (p *testPolicy) GetType() ascache.PolicyType { return p.policy }
-
-// TestIntegration_FleetOfCachesSwitchesToTheBetterPolicy wires the whole thing
-// together: real AdaptiveCaches with their own epoch goroutines, real bandits
-// with their own coordination goroutines, one shared store.
-//
-// The workload is a cyclic scan slightly larger than the cache, where LRU
-// serves nothing at all and MRU serves most of it. Each replica sees a
-// fraction of the traffic.
-func TestIntegration_FleetOfCachesSwitchesToTheBetterPolicy(t *testing.T) {
- if testing.Short() {
- t.Skip("timing-dependent: runs goroutines at millisecond epochs")
- }
-
- const (
- replicas = 4
- capacity = 32
- workingSet = 40
- )
-
- store := NewMemStore()
- t.Cleanup(func() { _ = store.Close() })
-
- caches := make([]*ascache.AdaptiveCache[string, int], 0, replicas)
- bandits := make([]*Distributed, 0, replicas)
-
- for i := range replicas {
- b, err := NewDistributed(Config{
- Store: store,
- Namespace: "integration",
- NodeID: string(rune('a' + i)),
- CoordinationEpoch: 20 * time.Millisecond,
- Window: 4,
- Seed: uint64(i + 1),
- })
- require.NoError(t, err)
- t.Cleanup(func() { _ = b.Close() })
- bandits = append(bandits, b)
-
- cache, err := ascache.NewAdaptiveCache(
- []ascache.Policy[string, int]{
- newTestPolicy(ascache.LRU, capacity, evictOldest),
- newTestPolicy(ascache.TinyLFU, capacity, evictNewest),
- },
- b,
- &ascache.Settings{
- EpochDuration: 5 * time.Millisecond,
- EvictPartialCapacityFilling: true,
- MigrationStrategy: ascache.MigrationWarm,
- },
- )
- require.NoError(t, err)
- t.Cleanup(func() { _ = cache.Close() })
- caches = append(caches, cache)
- }
-
- stop := make(chan struct{})
- var wg sync.WaitGroup
- for _, cache := range caches {
- wg.Go(func() {
- for round := 0; ; round++ {
- select {
- case <-stop:
- return
- default:
- }
-
- key := keyOf(round % workingSet)
- if _, ok := cache.Get(key); !ok {
- cache.Add(key, round)
- }
- }
- })
- }
-
- // Every replica should end up on the arm the fleet's pooled evidence
- // favours, through a decision none of them made alone.
- require.Eventually(t, func() bool {
- for _, cache := range caches {
- if cache.ActivePolicy() != ascache.TinyLFU {
- return false
- }
- }
-
- return true
- }, 15*time.Second, 25*time.Millisecond)
-
- close(stop)
- wg.Wait()
-
- for _, b := range bandits {
- snapshot := b.Snapshot()
- assert.False(t, snapshot.Fallback, "%s fell back despite a healthy store", snapshot.NodeID)
- assert.Positive(t, snapshot.Syncs)
- }
-
- // Leadership is claimed once per bucket across the fleet, so no single
- // replica can have led every one of them.
- led := 0
- for _, b := range bandits {
- if b.Snapshot().Leaderships > 0 {
- led++
- }
- }
- assert.Positive(t, led)
-}
-
-func keyOf(i int) string {
- return "k" + strconv.Itoa(i)
-}
-
-// TestIntegration_CacheSurvivesABanditThatNeverReachesItsStore is the failure
-// case that matters most: the store is down from the start, so the bandit is
-// in fallback for the whole run. The cache must keep serving throughout.
-func TestIntegration_CacheSurvivesABanditThatNeverReachesItsStore(t *testing.T) {
- if testing.Short() {
- t.Skip("timing-dependent: runs goroutines at millisecond epochs")
- }
-
- store := NewMemStore()
- store.Fail(assert.AnError)
- t.Cleanup(func() { _ = store.Close() })
-
- b, err := NewDistributed(Config{
- Store: store,
- Namespace: "outage",
- CoordinationEpoch: 5 * time.Millisecond,
- })
- require.NoError(t, err)
- t.Cleanup(func() { _ = b.Close() })
-
- cache, err := ascache.NewAdaptiveCache(
- []ascache.Policy[string, int]{
- newTestPolicy(ascache.LRU, 16, evictOldest),
- newTestPolicy(ascache.TinyLFU, 16, evictNewest),
- },
- b,
- &ascache.Settings{
- EpochDuration: 2 * time.Millisecond,
- EvictPartialCapacityFilling: true,
- },
- )
- require.NoError(t, err)
- t.Cleanup(func() { _ = cache.Close() })
-
- deadline := time.Now().Add(2 * time.Second)
- for round := 0; time.Now().Before(deadline); round++ {
- key := keyOf(round % 20)
- if _, ok := cache.Get(key); !ok {
- cache.Add(key, round)
- }
- }
-
- stats := cache.Stats()
- assert.Positive(t, stats.Hits+stats.Misses, "the cache must keep serving through an outage")
-
- snapshot := b.Snapshot()
- assert.True(t, snapshot.Fallback)
- assert.Positive(t, snapshot.SyncFailures)
- assert.NotEqual(t, ascache.Undefined, cache.ActivePolicy())
-}
diff --git a/bandit/memstore.go b/bandit/memstore.go
deleted file mode 100644
index 53811ae..0000000
--- a/bandit/memstore.go
+++ /dev/null
@@ -1,254 +0,0 @@
-package bandit
-
-import (
- "context"
- "sync"
- "time"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-var _ Store = (*MemStore)(nil)
-
-// MemStore is a Store held in memory, shared by every replica in one process.
-//
-// It is not a stand-in for a real store in production - a fleet in one process
-// is not a fleet - but it is the right thing for three jobs: testing a
-// distributed bandit without a server, simulating a fleet to see whether
-// pooling helps before deploying anything, and giving the store contract one
-// executable definition that the Valkey and Redis adapters are checked
-// against.
-//
-// It implements the same semantics the adapters must: buckets come from the
-// store's clock and never a replica's, leadership is first-come per bucket,
-// a published decision is immutable, and everything expires.
-type MemStore struct {
- mu sync.Mutex
-
- now func() time.Time
-
- counts map[bucketKey]*bucketEntry
- leaders map[bucketKey]entry[string]
- decisions map[bucketKey]entry[ascache.PolicyType]
-
- // failure, when non-nil, is returned by every call. It exists so a test
- // can take the store away mid-run and watch replicas fall back.
- failure error
-}
-
-type bucketKey struct {
- namespace string
- bucket Bucket
-}
-
-type entry[T any] struct {
- value T
- expires time.Time
-}
-
-type bucketEntry struct {
- arms map[ArmKey]ascache.PolicyStats
- expires time.Time
-}
-
-// NewMemStore returns an empty in-memory store using the wall clock.
-func NewMemStore() *MemStore {
- return &MemStore{
- now: time.Now,
- counts: make(map[bucketKey]*bucketEntry),
- leaders: make(map[bucketKey]entry[string]),
- decisions: make(map[bucketKey]entry[ascache.PolicyType]),
- }
-}
-
-// SetClock replaces the store's clock, which is what assigns buckets. A test
-// that drives this clock controls bucket boundaries exactly, with no sleeping
-// and no dependence on how fast the machine is.
-func (s *MemStore) SetClock(now func() time.Time) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- s.now = now
-}
-
-// Fail makes every subsequent call return err, or restores normal operation
-// when err is nil. It is how a test simulates the store going away.
-func (s *MemStore) Fail(err error) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- s.failure = err
-}
-
-// Sync publishes one replica's counts, claims the bucket if asked and if it is
-// unclaimed, and reports any decision already published for it.
-func (s *MemStore) Sync(ctx context.Context, req SyncRequest) (SyncResult, error) {
- if err := ctx.Err(); err != nil {
- return SyncResult{}, err
- }
-
- s.mu.Lock()
- defer s.mu.Unlock()
-
- if s.failure != nil {
- return SyncResult{}, s.failure
- }
-
- now := s.now()
- s.expireLocked(now)
-
- bucket := bucketAt(now, req.EpochMillis)
- key := bucketKey{namespace: req.Namespace, bucket: bucket}
-
- if len(req.Counts) > 0 {
- state, ok := s.counts[key]
- if !ok {
- state = &bucketEntry{arms: make(map[ArmKey]ascache.PolicyStats)}
- s.counts[key] = state
- }
- // The TTL is refreshed by every writer, so a bucket outlives its last
- // contribution rather than its first.
- state.expires = now.Add(req.CounterTTL)
-
- for _, count := range req.Counts {
- armKey := ArmKey{Policy: count.Policy, Role: count.Role}
- stats := state.arms[armKey]
- stats.Hits += count.Hits
- stats.Misses += count.Misses
- state.arms[armKey] = stats
- }
- }
-
- result := SyncResult{Bucket: bucket}
-
- if req.Lead {
- if _, taken := s.leaders[key]; !taken {
- s.leaders[key] = entry[string]{value: req.NodeID, expires: now.Add(req.LeaderTTL)}
- result.Leader = true
- }
- }
-
- if decision, ok := s.decisions[key]; ok {
- result.Decision, result.HasDecision = decision.value, true
- }
-
- return result, nil
-}
-
-// Window returns the buckets in [first, last] that hold counts. Buckets that
-// were never written, or that have expired, are omitted rather than reported
-// as zero.
-func (s *MemStore) Window(ctx context.Context, namespace string, first, last Bucket) ([]WindowCounts, error) {
- if err := ctx.Err(); err != nil {
- return nil, err
- }
-
- s.mu.Lock()
- defer s.mu.Unlock()
-
- if s.failure != nil {
- return nil, s.failure
- }
-
- s.expireLocked(s.now())
-
- // The span is caller-supplied, and sizing the slice from it would let a
- // wide range allocate for buckets that were never written. Most windows
- // hold a handful of buckets; append covers the rest.
- window := make([]WindowCounts, 0, min(max(0, int(last-first)+1), 64))
- for bucket := first; bucket <= last; bucket++ {
- state, ok := s.counts[bucketKey{namespace: namespace, bucket: bucket}]
- if !ok {
- continue
- }
-
- arms := make(map[ArmKey]ascache.PolicyStats, len(state.arms))
- for armKey, stats := range state.arms {
- arms[armKey] = stats
- }
- window = append(window, WindowCounts{Bucket: bucket, Arms: arms})
- }
-
- return window, nil
-}
-
-// Decide publishes a decision for a bucket, leaving any existing one in place,
-// and returns whichever decision is in force.
-func (s *MemStore) Decide(
- ctx context.Context,
- namespace string,
- bucket Bucket,
- policy ascache.PolicyType,
- ttl time.Duration,
-) (ascache.PolicyType, error) {
- if err := ctx.Err(); err != nil {
- return ascache.Undefined, err
- }
-
- s.mu.Lock()
- defer s.mu.Unlock()
-
- if s.failure != nil {
- return ascache.Undefined, s.failure
- }
-
- now := s.now()
- s.expireLocked(now)
-
- key := bucketKey{namespace: namespace, bucket: bucket}
- if existing, exists := s.decisions[key]; exists {
- // Immutable once published: replicas act on it, and one that changed
- // underneath them would move the fleet mid-epoch for no reason.
- return existing.value, nil
- }
-
- s.decisions[key] = entry[ascache.PolicyType]{value: policy, expires: now.Add(ttl)}
-
- return policy, nil
-}
-
-// Close discards everything the store holds.
-func (s *MemStore) Close() error {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- clear(s.counts)
- clear(s.leaders)
- clear(s.decisions)
-
- return nil
-}
-
-// expireLocked drops everything past its TTL. A real store does this itself;
-// doing it here keeps a long-running simulation from growing without bound and
-// keeps the two implementations behaving the same when a leader reads back
-// further than the counters were kept for.
-func (s *MemStore) expireLocked(now time.Time) {
- for key, state := range s.counts {
- if now.After(state.expires) {
- delete(s.counts, key)
- }
- }
- for key, state := range s.leaders {
- if now.After(state.expires) {
- delete(s.leaders, key)
- }
- }
- for key, state := range s.decisions {
- if now.After(state.expires) {
- delete(s.decisions, key)
- }
- }
-}
-
-// bucketAt divides the store's clock into coordination epochs. Every replica
-// gets its bucket from here rather than computing one locally, which is what
-// makes the scheme immune to clock skew across the fleet: a replica's own
-// clock is never consulted, so it cannot be wrong in a way that matters.
-func bucketAt(now time.Time, epochMillis int64) Bucket {
- if epochMillis <= 0 {
- epochMillis = 1
- }
-
- return Bucket(now.UnixMilli() / epochMillis)
-}
diff --git a/bandit/memstore_test.go b/bandit/memstore_test.go
deleted file mode 100644
index a58cb94..0000000
--- a/bandit/memstore_test.go
+++ /dev/null
@@ -1,298 +0,0 @@
-package bandit
-
-import (
- "context"
- "errors"
- "sync"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-// testClock is a clock a test drives by hand, so bucket boundaries are exact
-// and nothing has to sleep.
-type testClock struct {
- mu sync.Mutex
- at time.Time
-}
-
-func newTestClock() *testClock {
- // An arbitrary fixed instant. Buckets are derived from Unix milliseconds,
- // so any starting point works as long as it does not move on its own.
- return &testClock{at: time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC)}
-}
-
-func (c *testClock) now() time.Time {
- c.mu.Lock()
- defer c.mu.Unlock()
-
- return c.at
-}
-
-func (c *testClock) advance(d time.Duration) {
- c.mu.Lock()
- defer c.mu.Unlock()
-
- c.at = c.at.Add(d)
-}
-
-const testEpoch = time.Second
-
-func testSyncRequest(namespace, node string, counts ...ArmCounts) SyncRequest {
- return SyncRequest{
- Namespace: namespace,
- NodeID: node,
- Counts: counts,
- EpochMillis: testEpoch.Milliseconds(),
- CounterTTL: 12 * testEpoch,
- LeaderTTL: 2 * testEpoch,
- }
-}
-
-func shadow(policy ascache.PolicyType, hits, misses int64) ArmCounts {
- return ArmCounts{Policy: policy, Role: RoleShadow, Hits: hits, Misses: misses}
-}
-
-func TestMemStore_BucketComesFromTheStoreClock(t *testing.T) {
- clock := newTestClock()
- store := NewMemStore()
- store.SetClock(clock.now)
- t.Cleanup(func() { _ = store.Close() })
-
- first, err := store.Sync(t.Context(), testSyncRequest("ns", "a"))
- require.NoError(t, err)
-
- clock.advance(testEpoch)
- second, err := store.Sync(t.Context(), testSyncRequest("ns", "a"))
- require.NoError(t, err)
-
- assert.Equal(t, first.Bucket+1, second.Bucket)
-}
-
-func TestMemStore_SumsCountsAcrossReplicas(t *testing.T) {
- store := NewMemStore()
- t.Cleanup(func() { _ = store.Close() })
-
- result, err := store.Sync(t.Context(), testSyncRequest("ns", "a", shadow(ascache.LRU, 10, 5)))
- require.NoError(t, err)
- _, err = store.Sync(t.Context(), testSyncRequest("ns", "b", shadow(ascache.LRU, 20, 15)))
- require.NoError(t, err)
-
- window, err := store.Window(t.Context(), "ns", result.Bucket, result.Bucket)
- require.NoError(t, err)
- require.Len(t, window, 1)
-
- assert.Equal(t,
- ascache.PolicyStats{Hits: 30, Misses: 20},
- window[0].Arms[ArmKey{Policy: ascache.LRU, Role: RoleShadow}])
-}
-
-func TestMemStore_NamespacesDoNotPool(t *testing.T) {
- store := NewMemStore()
- t.Cleanup(func() { _ = store.Close() })
-
- result, err := store.Sync(t.Context(), testSyncRequest("one", "a", shadow(ascache.LRU, 10, 0)))
- require.NoError(t, err)
- _, err = store.Sync(t.Context(), testSyncRequest("two", "b", shadow(ascache.LRU, 999, 0)))
- require.NoError(t, err)
-
- window, err := store.Window(t.Context(), "one", result.Bucket, result.Bucket)
- require.NoError(t, err)
- require.Len(t, window, 1)
-
- assert.Equal(t, int64(10), window[0].Arms[ArmKey{Policy: ascache.LRU, Role: RoleShadow}].Hits)
-}
-
-func TestMemStore_OneLeaderPerBucket(t *testing.T) {
- clock := newTestClock()
- store := NewMemStore()
- store.SetClock(clock.now)
- t.Cleanup(func() { _ = store.Close() })
-
- leaders := 0
- for _, node := range []string{"a", "b", "c", "d", "e"} {
- req := testSyncRequest("ns", node)
- req.Lead = true
- result, err := store.Sync(t.Context(), req)
- require.NoError(t, err)
- if result.Leader {
- leaders++
- }
- }
- assert.Equal(t, 1, leaders, "leadership of a bucket is claimed once")
-
- clock.advance(testEpoch)
- req := testSyncRequest("ns", "f")
- req.Lead = true
- result, err := store.Sync(t.Context(), req)
- require.NoError(t, err)
- assert.True(t, result.Leader, "the next bucket is up for grabs again")
-}
-
-func TestMemStore_LeadershipIsNotClaimedUnlessAsked(t *testing.T) {
- store := NewMemStore()
- t.Cleanup(func() { _ = store.Close() })
-
- result, err := store.Sync(t.Context(), testSyncRequest("ns", "a"))
- require.NoError(t, err)
- assert.False(t, result.Leader)
-}
-
-func TestMemStore_DecisionIsImmutable(t *testing.T) {
- store := NewMemStore()
- t.Cleanup(func() { _ = store.Close() })
-
- first, err := store.Decide(t.Context(), "ns", 100, ascache.TinyLFU, time.Minute)
- require.NoError(t, err)
- assert.Equal(t, ascache.TinyLFU, first)
-
- // Republishing is not an error, and reports what is actually in force -
- // which is how a leader that lost a race still ends up agreeing with the
- // fleet instead of running its own draw alone.
- second, err := store.Decide(t.Context(), "ns", 100, ascache.LRU, time.Minute)
- require.NoError(t, err)
- assert.Equal(t, ascache.TinyLFU, second)
-
- clock := newTestClock()
- store.SetClock(clock.now)
-
- // Read it back through a sync landing in the same bucket.
- req := testSyncRequest("ns", "a")
- req.EpochMillis = 1
- result, err := store.Sync(t.Context(), req)
- require.NoError(t, err)
-
- _, err = store.Decide(t.Context(), "ns", result.Bucket, ascache.LFU, time.Minute)
- require.NoError(t, err)
-
- again, err := store.Sync(t.Context(), req)
- require.NoError(t, err)
-
- require.True(t, again.HasDecision)
- assert.Equal(t, ascache.LFU, again.Decision)
-}
-
-func TestMemStore_ExpiredBucketsLeaveHolesRatherThanZeros(t *testing.T) {
- clock := newTestClock()
- store := NewMemStore()
- store.SetClock(clock.now)
- t.Cleanup(func() { _ = store.Close() })
-
- req := testSyncRequest("ns", "a", shadow(ascache.LRU, 10, 0))
- req.CounterTTL = 2 * testEpoch
- first, err := store.Sync(t.Context(), req)
- require.NoError(t, err)
-
- clock.advance(5 * testEpoch)
- latest, err := store.Sync(t.Context(), req)
- require.NoError(t, err)
-
- window, err := store.Window(t.Context(), "ns", first.Bucket, latest.Bucket)
- require.NoError(t, err)
-
- require.Len(t, window, 1, "the expired bucket is absent, not present and empty")
- assert.Equal(t, latest.Bucket, window[0].Bucket)
-}
-
-func TestMemStore_WindowSkipsBucketsNeverWritten(t *testing.T) {
- store := NewMemStore()
- t.Cleanup(func() { _ = store.Close() })
-
- result, err := store.Sync(t.Context(), testSyncRequest("ns", "a", shadow(ascache.LRU, 1, 1)))
- require.NoError(t, err)
-
- window, err := store.Window(t.Context(), "ns", result.Bucket-10, result.Bucket)
- require.NoError(t, err)
- assert.Len(t, window, 1)
-}
-
-func TestMemStore_WindowCopiesItsCounts(t *testing.T) {
- store := NewMemStore()
- t.Cleanup(func() { _ = store.Close() })
-
- result, err := store.Sync(t.Context(), testSyncRequest("ns", "a", shadow(ascache.LRU, 10, 0)))
- require.NoError(t, err)
-
- window, err := store.Window(t.Context(), "ns", result.Bucket, result.Bucket)
- require.NoError(t, err)
- require.Len(t, window, 1)
-
- // A caller mutating what it read must not corrupt the store's own state,
- // or a fleet simulation would poison itself.
- window[0].Arms[ArmKey{Policy: ascache.LRU, Role: RoleShadow}] = ascache.PolicyStats{Hits: 1 << 40}
-
- again, err := store.Window(t.Context(), "ns", result.Bucket, result.Bucket)
- require.NoError(t, err)
- assert.Equal(t, int64(10), again[0].Arms[ArmKey{Policy: ascache.LRU, Role: RoleShadow}].Hits)
-}
-
-func TestMemStore_FailAffectsEveryCall(t *testing.T) {
- store := NewMemStore()
- t.Cleanup(func() { _ = store.Close() })
-
- boom := errors.New("store is down")
- store.Fail(boom)
-
- _, err := store.Sync(t.Context(), testSyncRequest("ns", "a"))
- assert.ErrorIs(t, err, boom)
-
- _, err = store.Window(t.Context(), "ns", 0, 10)
- assert.ErrorIs(t, err, boom)
-
- _, err = store.Decide(t.Context(), "ns", 1, ascache.LRU, time.Minute)
- assert.ErrorIs(t, err, boom)
-
- store.Fail(nil)
- _, err = store.Sync(t.Context(), testSyncRequest("ns", "a"))
- assert.NoError(t, err)
-}
-
-func TestMemStore_RespectsContextCancellation(t *testing.T) {
- store := NewMemStore()
- t.Cleanup(func() { _ = store.Close() })
-
- ctx, cancel := context.WithCancel(t.Context())
- cancel()
-
- _, err := store.Sync(ctx, testSyncRequest("ns", "a"))
- assert.ErrorIs(t, err, context.Canceled)
-
- _, err = store.Window(ctx, "ns", 0, 1)
- assert.ErrorIs(t, err, context.Canceled)
-
- _, err = store.Decide(ctx, "ns", 1, ascache.LRU, time.Minute)
- assert.ErrorIs(t, err, context.Canceled)
-}
-
-func TestMemStore_ConcurrentReplicas(t *testing.T) {
- store := NewMemStore()
- t.Cleanup(func() { _ = store.Close() })
-
- const replicas = 16
- const rounds = 50
-
- var wg sync.WaitGroup
- for node := range replicas {
- wg.Add(1)
- go func() {
- defer wg.Done()
- for range rounds {
- req := testSyncRequest("ns", string(rune('a'+node)), shadow(ascache.LRU, 1, 1))
- req.Lead = true
- if _, err := store.Sync(t.Context(), req); err != nil {
- assert.NoError(t, err)
- return
- }
- if _, err := store.Window(t.Context(), "ns", 0, 32); err != nil {
- assert.NoError(t, err)
- return
- }
- }
- }()
- }
- wg.Wait()
-}
diff --git a/bandit/redis/LICENSE b/bandit/redis/LICENSE
deleted file mode 100644
index d0a1fa1..0000000
--- a/bandit/redis/LICENSE
+++ /dev/null
@@ -1,373 +0,0 @@
-Mozilla Public License Version 2.0
-==================================
-
-1. Definitions
---------------
-
-1.1. "Contributor"
- means each individual or legal entity that creates, contributes to
- the creation of, or owns Covered Software.
-
-1.2. "Contributor Version"
- means the combination of the Contributions of others (if any) used
- by a Contributor and that particular Contributor's Contribution.
-
-1.3. "Contribution"
- means Covered Software of a particular Contributor.
-
-1.4. "Covered Software"
- means Source Code Form to which the initial Contributor has attached
- the notice in Exhibit A, the Executable Form of such Source Code
- Form, and Modifications of such Source Code Form, in each case
- including portions thereof.
-
-1.5. "Incompatible With Secondary Licenses"
- means
-
- (a) that the initial Contributor has attached the notice described
- in Exhibit B to the Covered Software; or
-
- (b) that the Covered Software was made available under the terms of
- version 1.1 or earlier of the License, but not also under the
- terms of a Secondary License.
-
-1.6. "Executable Form"
- means any form of the work other than Source Code Form.
-
-1.7. "Larger Work"
- means a work that combines Covered Software with other material, in
- a separate file or files, that is not Covered Software.
-
-1.8. "License"
- means this document.
-
-1.9. "Licensable"
- means having the right to grant, to the maximum extent possible,
- whether at the time of the initial grant or subsequently, any and
- all of the rights conveyed by this License.
-
-1.10. "Modifications"
- means any of the following:
-
- (a) any file in Source Code Form that results from an addition to,
- deletion from, or modification of the contents of Covered
- Software; or
-
- (b) any new file in Source Code Form that contains any Covered
- Software.
-
-1.11. "Patent Claims" of a Contributor
- means any patent claim(s), including without limitation, method,
- process, and apparatus claims, in any patent Licensable by such
- Contributor that would be infringed, but for the grant of the
- License, by the making, using, selling, offering for sale, having
- made, import, or transfer of either its Contributions or its
- Contributor Version.
-
-1.12. "Secondary License"
- means either the GNU General Public License, Version 2.0, the GNU
- Lesser General Public License, Version 2.1, the GNU Affero General
- Public License, Version 3.0, or any later versions of those
- licenses.
-
-1.13. "Source Code Form"
- means the form of the work preferred for making modifications.
-
-1.14. "You" (or "Your")
- means an individual or a legal entity exercising rights under this
- License. For legal entities, "You" includes any entity that
- controls, is controlled by, or is under common control with You. For
- purposes of this definition, "control" means (a) the power, direct
- or indirect, to cause the direction or management of such entity,
- whether by contract or otherwise, or (b) ownership of more than
- fifty percent (50%) of the outstanding shares or beneficial
- ownership of such entity.
-
-2. License Grants and Conditions
---------------------------------
-
-2.1. Grants
-
-Each Contributor hereby grants You a world-wide, royalty-free,
-non-exclusive license:
-
-(a) under intellectual property rights (other than patent or trademark)
- Licensable by such Contributor to use, reproduce, make available,
- modify, display, perform, distribute, and otherwise exploit its
- Contributions, either on an unmodified basis, with Modifications, or
- as part of a Larger Work; and
-
-(b) under Patent Claims of such Contributor to make, use, sell, offer
- for sale, have made, import, and otherwise transfer either its
- Contributions or its Contributor Version.
-
-2.2. Effective Date
-
-The licenses granted in Section 2.1 with respect to any Contribution
-become effective for each Contribution on the date the Contributor first
-distributes such Contribution.
-
-2.3. Limitations on Grant Scope
-
-The licenses granted in this Section 2 are the only rights granted under
-this License. No additional rights or licenses will be implied from the
-distribution or licensing of Covered Software under this License.
-Notwithstanding Section 2.1(b) above, no patent license is granted by a
-Contributor:
-
-(a) for any code that a Contributor has removed from Covered Software;
- or
-
-(b) for infringements caused by: (i) Your and any other third party's
- modifications of Covered Software, or (ii) the combination of its
- Contributions with other software (except as part of its Contributor
- Version); or
-
-(c) under Patent Claims infringed by Covered Software in the absence of
- its Contributions.
-
-This License does not grant any rights in the trademarks, service marks,
-or logos of any Contributor (except as may be necessary to comply with
-the notice requirements in Section 3.4).
-
-2.4. Subsequent Licenses
-
-No Contributor makes additional grants as a result of Your choice to
-distribute the Covered Software under a subsequent version of this
-License (see Section 10.2) or under the terms of a Secondary License (if
-permitted under the terms of Section 3.3).
-
-2.5. Representation
-
-Each Contributor represents that the Contributor believes its
-Contributions are its original creation(s) or it has sufficient rights
-to grant the rights to its Contributions conveyed by this License.
-
-2.6. Fair Use
-
-This License is not intended to limit any rights You have under
-applicable copyright doctrines of fair use, fair dealing, or other
-equivalents.
-
-2.7. Conditions
-
-Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
-in Section 2.1.
-
-3. Responsibilities
--------------------
-
-3.1. Distribution of Source Form
-
-All distribution of Covered Software in Source Code Form, including any
-Modifications that You create or to which You contribute, must be under
-the terms of this License. You must inform recipients that the Source
-Code Form of the Covered Software is governed by the terms of this
-License, and how they can obtain a copy of this License. You may not
-attempt to alter or restrict the recipients' rights in the Source Code
-Form.
-
-3.2. Distribution of Executable Form
-
-If You distribute Covered Software in Executable Form then:
-
-(a) such Covered Software must also be made available in Source Code
- Form, as described in Section 3.1, and You must inform recipients of
- the Executable Form how they can obtain a copy of such Source Code
- Form by reasonable means in a timely manner, at a charge no more
- than the cost of distribution to the recipient; and
-
-(b) You may distribute such Executable Form under the terms of this
- License, or sublicense it under different terms, provided that the
- license for the Executable Form does not attempt to limit or alter
- the recipients' rights in the Source Code Form under this License.
-
-3.3. Distribution of a Larger Work
-
-You may create and distribute a Larger Work under terms of Your choice,
-provided that You also comply with the requirements of this License for
-the Covered Software. If the Larger Work is a combination of Covered
-Software with a work governed by one or more Secondary Licenses, and the
-Covered Software is not Incompatible With Secondary Licenses, this
-License permits You to additionally distribute such Covered Software
-under the terms of such Secondary License(s), so that the recipient of
-the Larger Work may, at their option, further distribute the Covered
-Software under the terms of either this License or such Secondary
-License(s).
-
-3.4. Notices
-
-You may not remove or alter the substance of any license notices
-(including copyright notices, patent notices, disclaimers of warranty,
-or limitations of liability) contained within the Source Code Form of
-the Covered Software, except that You may alter any license notices to
-the extent required to remedy known factual inaccuracies.
-
-3.5. Application of Additional Terms
-
-You may choose to offer, and to charge a fee for, warranty, support,
-indemnity or liability obligations to one or more recipients of Covered
-Software. However, You may do so only on Your own behalf, and not on
-behalf of any Contributor. You must make it absolutely clear that any
-such warranty, support, indemnity, or liability obligation is offered by
-You alone, and You hereby agree to indemnify every Contributor for any
-liability incurred by such Contributor as a result of warranty, support,
-indemnity or liability terms You offer. You may include additional
-disclaimers of warranty and limitations of liability specific to any
-jurisdiction.
-
-4. Inability to Comply Due to Statute or Regulation
----------------------------------------------------
-
-If it is impossible for You to comply with any of the terms of this
-License with respect to some or all of the Covered Software due to
-statute, judicial order, or regulation then You must: (a) comply with
-the terms of this License to the maximum extent possible; and (b)
-describe the limitations and the code they affect. Such description must
-be placed in a text file included with all distributions of the Covered
-Software under this License. Except to the extent prohibited by statute
-or regulation, such description must be sufficiently detailed for a
-recipient of ordinary skill to be able to understand it.
-
-5. Termination
---------------
-
-5.1. The rights granted under this License will terminate automatically
-if You fail to comply with any of its terms. However, if You become
-compliant, then the rights granted under this License from a particular
-Contributor are reinstated (a) provisionally, unless and until such
-Contributor explicitly and finally terminates Your grants, and (b) on an
-ongoing basis, if such Contributor fails to notify You of the
-non-compliance by some reasonable means prior to 60 days after You have
-come back into compliance. Moreover, Your grants from a particular
-Contributor are reinstated on an ongoing basis if such Contributor
-notifies You of the non-compliance by some reasonable means, this is the
-first time You have received notice of non-compliance with this License
-from such Contributor, and You become compliant prior to 30 days after
-Your receipt of the notice.
-
-5.2. If You initiate litigation against any entity by asserting a patent
-infringement claim (excluding declaratory judgment actions,
-counter-claims, and cross-claims) alleging that a Contributor Version
-directly or indirectly infringes any patent, then the rights granted to
-You by any and all Contributors for the Covered Software under Section
-2.1 of this License shall terminate.
-
-5.3. In the event of termination under Sections 5.1 or 5.2 above, all
-end user license agreements (excluding distributors and resellers) which
-have been validly granted by You or Your distributors under this License
-prior to termination shall survive termination.
-
-************************************************************************
-* *
-* 6. Disclaimer of Warranty *
-* ------------------------- *
-* *
-* Covered Software is provided under this License on an "as is" *
-* basis, without warranty of any kind, either expressed, implied, or *
-* statutory, including, without limitation, warranties that the *
-* Covered Software is free of defects, merchantable, fit for a *
-* particular purpose or non-infringing. The entire risk as to the *
-* quality and performance of the Covered Software is with You. *
-* Should any Covered Software prove defective in any respect, You *
-* (not any Contributor) assume the cost of any necessary servicing, *
-* repair, or correction. This disclaimer of warranty constitutes an *
-* essential part of this License. No use of any Covered Software is *
-* authorized under this License except under this disclaimer. *
-* *
-************************************************************************
-
-************************************************************************
-* *
-* 7. Limitation of Liability *
-* -------------------------- *
-* *
-* Under no circumstances and under no legal theory, whether tort *
-* (including negligence), contract, or otherwise, shall any *
-* Contributor, or anyone who distributes Covered Software as *
-* permitted above, be liable to You for any direct, indirect, *
-* special, incidental, or consequential damages of any character *
-* including, without limitation, damages for lost profits, loss of *
-* goodwill, work stoppage, computer failure or malfunction, or any *
-* and all other commercial damages or losses, even if such party *
-* shall have been informed of the possibility of such damages. This *
-* limitation of liability shall not apply to liability for death or *
-* personal injury resulting from such party's negligence to the *
-* extent applicable law prohibits such limitation. Some *
-* jurisdictions do not allow the exclusion or limitation of *
-* incidental or consequential damages, so this exclusion and *
-* limitation may not apply to You. *
-* *
-************************************************************************
-
-8. Litigation
--------------
-
-Any litigation relating to this License may be brought only in the
-courts of a jurisdiction where the defendant maintains its principal
-place of business and such litigation shall be governed by laws of that
-jurisdiction, without reference to its conflict-of-law provisions.
-Nothing in this Section shall prevent a party's ability to bring
-cross-claims or counter-claims.
-
-9. Miscellaneous
-----------------
-
-This License represents the complete agreement concerning the subject
-matter hereof. If any provision of this License is held to be
-unenforceable, such provision shall be reformed only to the extent
-necessary to make it enforceable. Any law or regulation which provides
-that the language of a contract shall be construed against the drafter
-shall not be used to construe this License against a Contributor.
-
-10. Versions of the License
----------------------------
-
-10.1. New Versions
-
-Mozilla Foundation is the license steward. Except as provided in Section
-10.3, no one other than the license steward has the right to modify or
-publish new versions of this License. Each version will be given a
-distinguishing version number.
-
-10.2. Effect of New Versions
-
-You may distribute the Covered Software under the terms of the version
-of the License under which You originally received the Covered Software,
-or under the terms of any subsequent version published by the license
-steward.
-
-10.3. Modified Versions
-
-If you create software not governed by this License, and you want to
-create a new license for such software, you may create and use a
-modified version of this License if you rename the license and remove
-any references to the name of the license steward (except to note that
-such modified license differs from this License).
-
-10.4. Distributing Source Code Form that is Incompatible With Secondary
-Licenses
-
-If You choose to distribute Source Code Form that is Incompatible With
-Secondary Licenses under the terms of this version of the License, the
-notice described in Exhibit B of this License must be attached.
-
-Exhibit A - Source Code Form License Notice
--------------------------------------------
-
- This Source Code Form is subject to the terms of the Mozilla Public
- License, v. 2.0. If a copy of the MPL was not distributed with this
- file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-If it is not possible or desirable to put the notice in a particular
-file, then You may include the notice in a location (such as a LICENSE
-file in a relevant directory) where a recipient would be likely to look
-for such a notice.
-
-You may add additional accurate notices of copyright ownership.
-
-Exhibit B - "Incompatible With Secondary Licenses" Notice
----------------------------------------------------------
-
- This Source Code Form is "Incompatible With Secondary Licenses", as
- defined by the Mozilla Public License, v. 2.0.
diff --git a/bandit/redis/TESTING.md b/bandit/redis/TESTING.md
deleted file mode 100644
index 5c0679b..0000000
--- a/bandit/redis/TESTING.md
+++ /dev/null
@@ -1,136 +0,0 @@
-# Testing the Valkey/Redis store
-
-This adapter is the only part of as-cache that depends on a server being there
-and behaving. Everything else can be tested by calling it; this cannot.
-
-The suite runs against [miniredis](https://github.com/alicebob/miniredis) by
-default so `make test` stays offline, and against real servers on demand. Both
-matter, for different reasons.
-
-## Why a fake is not enough here
-
-The adapter leans on three things that a fake can accept while a real server
-rejects them, or accept in a way a real server would not:
-
-1. **`redis.call('TIME')` inside a Lua script.** This is how a bucket is
- derived from the *server's* clock rather than a replica's, which is what
- frees a fleet from needing clock agreement. It requires effects replication;
- on Redis before 7.0 a script calling a non-deterministic command before a
- write was rejected outright. A fake with a permissive Lua interpreter will
- happily run a script that a Redis 6 server refuses.
-2. **Keys the script names itself.** A bucket is not known when the call is
- made, so `syncScript` builds its key names from a base passed in `ARGV`
- rather than declaring them in `KEYS`. That works because every key for a
- namespace carries the same hash tag and therefore hashes to one slot. A
- single-instance fake has no slots at all, so it cannot notice a mistake
- here.
-3. **`SET ... NX PX` truthiness and reply shapes.** Leadership depends on
- distinguishing "I claimed it" from "someone else has it", and the sync reply
- is a Lua table whose third element is deliberately an empty string rather
- than a nil, because a nil inside a Lua table truncates the array Redis
- builds from it and would silently take the bucket and leader values with it.
-
-Testing one engine while the documentation claims two is how a support claim
-quietly stops being true, so both are covered.
-
-## Running it
-
-```sh
-make redis-up # start Valkey and Redis, wait for their healthchecks
-make redis-test # start both, run the suite against each, tear down
-make redis-down # stop both and remove their volumes
-```
-
-Against a server you already have:
-
-```sh
-AS_CACHE_REDIS_ADDR=127.0.0.1:6379 go test -race -count=1 ./...
-```
-
-[`docker-compose.yml`](../../docker-compose.yml) in the repository root defines
-both servers. Ports are deliberately off the defaults — Valkey on `63799`,
-Redis on `63798` — so this cannot collide with a Redis you are already running
-for something else. Neither container persists anything.
-
-### A shell trap worth knowing
-
-Do not write the two-engine loop as
-
-```sh
-for target in "valkey 127.0.0.1:63799" "redis 127.0.0.1:63798"; do
- set -- $target
- AS_CACHE_REDIS_ADDR=$2 go test ./... # WRONG under zsh
-done
-```
-
-**zsh does not word-split unquoted parameter expansions**, so `$2` is empty,
-`AS_CACHE_REDIS_ADDR` is unset, and the run silently falls back to miniredis
-while printing every sign of having used a real server. It passes, quickly, and
-proves nothing. The `redis-test` recipe in the Makefile is safe because make
-runs recipes under `/bin/sh`, which does split. When running by hand, pass the
-address literally.
-
-The tell is `TestStore_CountersExpire`: it waits out a real TTL on a real
-server and takes about 0.6s, and fast-forwards a fake clock in 0.00s.
-
-## Verified
-
-Run on 2026-07-26, macOS 26.2 arm64, Go 1.25.5, `go test -race -count=1`.
-
-| Backend | Version | Result |
-| --- | --- | --- |
-| miniredis | v2.38.0 | 20 passed, 0 skipped, 0 failed |
-| Valkey | 8.1.9 (reports `redis_version` 7.2.4) | 20 passed, 0 skipped, 0 failed |
-| Redis | 7.4.10 | 20 passed, 0 skipped, 0 failed |
-
-```text
-==> bandit/redis against valkey (127.0.0.1:63799)
-ok github.com/sshaplygin/as-cache/bandit/redis 7.081s
-==> bandit/redis against redis (127.0.0.1:63798)
-ok github.com/sshaplygin/as-cache/bandit/redis 7.108s
-```
-
-Every test runs against every backend; nothing is skipped anywhere. The two
-slow tests are slow on purpose: `TestStore_ReportsAnUnreachableServer` waits
-out a dial timeout against a closed port, and `TestStore_CountersExpire` waits
-out a real TTL.
-
-## What each test establishes
-
-| Test | What it would catch |
-| --- | --- |
-| `TestNew_RejectsANilClient` | A store constructed with nowhere to talk to |
-| `TestStore_SyncReportsAServerDerivedBucket` | A script returning a constant instead of reading the server clock; the bucket is checked against the client's own clock, within tolerance |
-| `TestStore_SumsCountsAcrossReplicas` | Counts overwriting rather than accumulating — the whole point of pooling |
-| `TestStore_KeepsRolesApart` | Active-role and shadow-role counts merging, which would hide the measurement bias the two modes exist to manage |
-| `TestStore_NamespacesDoNotPool` | Two fleets sharing a store and silently pooling each other's evidence |
-| `TestStore_OneLeaderPerBucket` | Two replicas both believing they lead, which under `ModeLeader` means two decisions for one epoch |
-| `TestStore_LeadershipIsNotClaimedUnlessAsked` | A shared-posterior replica claiming leadership it never wanted, starving whoever did |
-| `TestStore_DecisionIsImmutableAndReportsWhatIsInForce` | A decision changing under replicas mid-epoch, and a leader that lost a race running its own draw alone |
-| `TestStore_SyncReadsBackAPublishedDecision` | Followers never seeing what the leader decided |
-| `TestStore_WindowOmitsBucketsThatHoldNothing` | A missing bucket read as a measured zero hit rate |
-| `TestStore_WindowOfAnInvertedRangeIsEmpty` | A reversed range spinning or erroring instead of returning nothing |
-| `TestStore_ZeroCountsCreateNoCounters` | An arm that measured nothing being recorded as an arm that measured badly |
-| `TestStore_CountersExpire` | Counters outliving their window, so a leader decides on stale traffic |
-| `TestStore_EveryKeyItWritesHasATTL` | Any key written without an expiry — a fleet that stops running must leave nothing behind in a shared store |
-| `TestStore_EveryKeyForANamespaceSharesOneSlot` | A key missing its hash tag, which breaks the window pipeline and the scripts' computed key names on Redis Cluster |
-| `TestStore_SurvivesAStrayFieldWrittenBySomethingElse` | A foreign field in a counter hash breaking a fleet's read of its own counters |
-| `TestStore_ReportsAnUnreachableServer` | A store failure being swallowed, which would leave the bandit waiting instead of falling back to local selection |
-| `TestStore_RespectsContextCancellation` | A call that ignores its context, which would stop `Close` from cancelling an in-flight round trip |
-| `TestParseCountField_RoundTrips` | The field encoding and its parser drifting apart |
-| `TestParseCountField_RejectsJunk` | Malformed fields being parsed into plausible-looking counts |
-
-## Not covered
-
-- **Redis Cluster.** The hash tags are verified against a real keyspace, so the
- property clustering depends on holds, but nothing here runs against an actual
- clustered deployment. Cross-slot behaviour is unproven.
-- **Redis 6 and earlier.** Documented as unsupported rather than tested as
- unsupported. The expected failure is the sync script being rejected for
- calling `TIME` before a write.
-- **Failover, replication lag and partitions.** A replica losing the store is
- covered in the `bandit` module's own tests via a store that fails on demand;
- what a real Valkey does mid-failover is not.
-- **Load at fleet scale.** The cost model in the package documentation — one
- round trip per replica per coordination epoch, two more for the leader — is
- arithmetic, not a measurement.
diff --git a/bandit/redis/doc.go b/bandit/redis/doc.go
deleted file mode 100644
index 4c836db..0000000
--- a/bandit/redis/doc.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Package redis backs a distributed bandit with Valkey or Redis.
-//
-// It implements github.com/sshaplygin/as-cache/bandit.Store over
-// github.com/redis/go-redis/v9, which speaks to both.
-//
-// client := goredis.NewClient(&goredis.Options{Addr: "localhost:6379"})
-// store, err := redisstore.New(redisstore.Options{Client: client})
-// if err != nil {
-// return err
-// }
-// defer store.Close()
-//
-// b, err := bandit.NewDistributed(bandit.Config{
-// Store: store,
-// Namespace: "sessions",
-// CoordinationEpoch: time.Second,
-// })
-//
-// # What it stores
-//
-// Per-policy hit and miss integers, a leader's node identifier, and a policy
-// name. No cache keys and no cache values ever leave the process, so the store
-// holds nothing that needs protecting beyond the usual.
-//
-// Everything written carries a TTL, sized from the bandit's window. A fleet
-// that stops running leaves nothing behind.
-//
-// # Load
-//
-// One round trip per replica per coordination epoch, plus two more for the
-// replica leading that epoch. At a one-second epoch a thousand replicas
-// produce about a thousand small pipelined calls a second against a single
-// key slot, which is not much - but it is the number to check before running a
-// coordination epoch faster than a second.
-//
-// # Requirements
-//
-// Redis 7.0 or Valkey 7.2 and above. Buckets are derived from the server's
-// clock inside a Lua script, which needs effects replication - the default
-// since Redis 7 - and which is the point: no replica's clock is ever
-// consulted, so a fleet needs no clock agreement at all.
-//
-// # Redis Cluster
-//
-// Every key for one namespace shares a hash tag, so a namespace lives in a
-// single slot and the scripts and pipelines here work unchanged on a cluster.
-package redis
diff --git a/bandit/redis/go.mod b/bandit/redis/go.mod
deleted file mode 100644
index e17de8f..0000000
--- a/bandit/redis/go.mod
+++ /dev/null
@@ -1,24 +0,0 @@
-module github.com/sshaplygin/as-cache/bandit/redis
-
-go 1.25.2
-
-require (
- github.com/alicebob/miniredis/v2 v2.38.0
- github.com/redis/go-redis/v9 v9.21.0
- github.com/sshaplygin/as-cache v0.3.1
- github.com/sshaplygin/as-cache/bandit v0.3.1
- github.com/stretchr/testify v1.11.1
-)
-
-require (
- github.com/cespare/xxhash/v2 v2.3.0 // indirect
- github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/yuin/gopher-lua v1.1.1 // indirect
- go.uber.org/atomic v1.11.0 // indirect
- gopkg.in/yaml.v3 v3.0.1 // indirect
-)
-
-replace github.com/sshaplygin/as-cache => ../..
-
-replace github.com/sshaplygin/as-cache/bandit => ..
diff --git a/bandit/redis/go.sum b/bandit/redis/go.sum
deleted file mode 100644
index ccc390e..0000000
--- a/bandit/redis/go.sum
+++ /dev/null
@@ -1,30 +0,0 @@
-github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw=
-github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
-github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
-github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
-github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
-github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
-github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
-github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.21.0 h1:FPBE4hhbAke+TLmcY3WkpbDffJEomdqPn3HYiqAtL9E=
-github.com/redis/go-redis/v9 v9.21.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
-github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
-github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
-go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
-go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/bandit/redis/keys.go b/bandit/redis/keys.go
deleted file mode 100644
index ca72b2e..0000000
--- a/bandit/redis/keys.go
+++ /dev/null
@@ -1,93 +0,0 @@
-package redis
-
-import (
- "strconv"
- "strings"
-
- ascache "github.com/sshaplygin/as-cache"
- "github.com/sshaplygin/as-cache/bandit"
-)
-
-// DefaultKeyPrefix is what every key this package writes begins with.
-const DefaultKeyPrefix = "asc"
-
-// keyBase is the common prefix of every key belonging to a namespace,
-// including the hash tag.
-//
-// The braces are what put a whole namespace in one Redis Cluster slot. That
-// matters twice: the window read is a pipeline across many bucket keys, and
-// the scripts compute key names themselves - a bucket comes from the server's
-// clock and so cannot be known when the call is made, and Cluster refuses a
-// script access outside the slot its declared keys hash to.
-func (s *Store) keyBase(namespace string) string {
- return s.prefix + ":{" + namespace + "}"
-}
-
-// anchorKey is the key handed to a script as KEYS[1], so Cluster routes it to
-// the slot the computed keys live in. Nothing is ever stored under it.
-func (s *Store) anchorKey(namespace string) string {
- return s.keyBase(namespace) + ":anchor"
-}
-
-// countsKey holds one bucket's per-arm counters as a hash. It must match the
-// expression the Lua builds.
-func (s *Store) countsKey(namespace string, bucket bandit.Bucket) string {
- return s.keyBase(namespace) + ":c:" + strconv.FormatInt(int64(bucket), 10)
-}
-
-// countField names one counter within a bucket's hash.
-//
-// The policy is written as its numeric PolicyType rather than its name. Names
-// come from stringer and are a presentation detail: renaming one would
-// silently split a fleet's counters in two, with every replica still reporting
-// and none of them agreeing, and nothing in the data would show why. The role
-// is "a" for active or "s" for shadow, and the suffix is "h" for hits or "m"
-// for misses.
-func countField(policy ascache.PolicyType, role bandit.Role, hits bool) string {
- kind := "m"
- if hits {
- kind = "h"
- }
-
- roleTag := "s"
- if role == bandit.RoleActive {
- roleTag = "a"
- }
-
- return strconv.FormatUint(uint64(policy), 10) + ":" + roleTag + ":" + kind
-}
-
-// parseCountField reverses countField. An unrecognised field is reported as
-// not ok rather than as an error: the store may be shared, and a stray field
-// written by something else must not stop a fleet reading its own counters.
-func parseCountField(field string) (policy ascache.PolicyType, role bandit.Role, hits, ok bool) {
- parts := strings.Split(field, ":")
- if len(parts) != 3 {
- return 0, 0, false, false
- }
-
- number, err := strconv.ParseUint(parts[0], 10, 64)
- if err != nil {
- return 0, 0, false, false
- }
-
- switch parts[1] {
- case "a":
- role = bandit.RoleActive
- case "s":
- role = bandit.RoleShadow
- default:
- return 0, 0, false, false
- }
-
- switch parts[2] {
- case "h":
- hits = true
- case "m":
- hits = false
- default:
- return 0, 0, false, false
- }
-
- return ascache.PolicyType(number), role, hits, true
-}
diff --git a/bandit/redis/scripts.go b/bandit/redis/scripts.go
deleted file mode 100644
index 304e85c..0000000
--- a/bandit/redis/scripts.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package redis
-
-import goredis "github.com/redis/go-redis/v9"
-
-// syncScript publishes one replica's counts, optionally claims the bucket, and
-// reads back whatever decision has been published for it - in one round trip,
-// because every replica makes this call on every coordination epoch and it is
-// the only one that scales with fleet size.
-//
-// The bucket comes from the server's clock. That is the whole reason this is a
-// script rather than a pipeline: if each replica divided its own clock into
-// buckets, a fleet would need clock agreement finer than its coordination
-// epoch, and one machine with a skewed clock would write its counts into a
-// window nobody reads while dragging the fleet's view of its traffic with it.
-// Asking the server means there is one clock and no agreement to reach.
-//
-// KEYS[1] anchor key, present so Cluster routes the script to the slot
-// the computed keys live in
-// ARGV[1] key base, including the hash tag: "asc:{namespace}"
-// ARGV[2] coordination epoch in milliseconds
-// ARGV[3] counter TTL in milliseconds
-// ARGV[4] leader TTL in milliseconds
-// ARGV[5] node id
-// ARGV[6] "1" to claim leadership of the bucket
-// ARGV[7...] alternating field name and delta
-var syncScript = goredis.NewScript(`
-local base = ARGV[1]
-local epoch = tonumber(ARGV[2])
-
-local t = redis.call('TIME')
-local nowMs = tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000)
-local bucket = math.floor(nowMs / epoch)
-
-if #ARGV >= 7 then
- local countsKey = base .. ':c:' .. bucket
- for i = 7, #ARGV, 2 do
- redis.call('HINCRBY', countsKey, ARGV[i], ARGV[i + 1])
- end
- -- Refreshed by every writer, so a bucket outlives its last contribution
- -- rather than its first.
- redis.call('PEXPIRE', countsKey, ARGV[3])
-end
-
-local leader = 0
-if ARGV[6] == '1' then
- if redis.call('SET', base .. ':l:' .. bucket, ARGV[5], 'NX', 'PX', ARGV[4]) then
- leader = 1
- end
-end
-
--- An empty string rather than a nil: a nil inside a Lua table terminates the
--- array Redis builds from it, so returning one here would truncate the reply
--- and take the bucket and leader values with it.
-local decision = redis.call('GET', base .. ':d:' .. bucket)
-if not decision then
- decision = ''
-end
-
-return {bucket, leader, decision}
-`)
-
-// decideScript publishes the decision for a bucket if none has been published,
-// and returns whichever decision is in force either way.
-//
-// It returns rather than acknowledges because the leader has to act on the
-// same policy as everyone following it. A leader that applied its own draw
-// while the fleet applied an earlier one would be the one replica running
-// something different, and it would be the one making the decisions.
-//
-// KEYS[1] anchor key, for Cluster routing
-// ARGV[1] key base, including the hash tag
-// ARGV[2] bucket
-// ARGV[3] policy
-// ARGV[4] decision TTL in milliseconds
-var decideScript = goredis.NewScript(`
-local key = ARGV[1] .. ':d:' .. ARGV[2]
-
-if redis.call('SET', key, ARGV[3], 'NX', 'PX', ARGV[4]) then
- return ARGV[3]
-end
-
-return redis.call('GET', key)
-`)
diff --git a/bandit/redis/store.go b/bandit/redis/store.go
deleted file mode 100644
index 1295731..0000000
--- a/bandit/redis/store.go
+++ /dev/null
@@ -1,273 +0,0 @@
-package redis
-
-import (
- "context"
- "errors"
- "fmt"
- "strconv"
- "time"
-
- goredis "github.com/redis/go-redis/v9"
-
- ascache "github.com/sshaplygin/as-cache"
- "github.com/sshaplygin/as-cache/bandit"
-)
-
-var _ bandit.Store = (*Store)(nil)
-
-// ErrNilClient is returned by New when Options.Client is nil.
-var ErrNilClient = errors.New("redis: client must not be nil")
-
-// Options configures a Store.
-type Options struct {
- // Client is the connection to Valkey or Redis. Required.
- //
- // It is a UniversalClient so a plain client, a cluster client and a ring
- // are all accepted, and it is supplied rather than dialled here because
- // connection settings - timeouts, TLS, credentials, pool sizes - belong to
- // the application, which almost always has a client already.
- Client goredis.UniversalClient
-
- // KeyPrefix begins every key this store writes. Defaults to
- // DefaultKeyPrefix.
- KeyPrefix string
-}
-
-// Store implements bandit.Store over Valkey or Redis.
-type Store struct {
- client goredis.UniversalClient
- prefix string
-}
-
-// New returns a Store over the given client. It does not take ownership of the
-// client: Close leaves it open for the application to keep using.
-func New(opts Options) (*Store, error) {
- if opts.Client == nil {
- return nil, ErrNilClient
- }
-
- prefix := opts.KeyPrefix
- if prefix == "" {
- prefix = DefaultKeyPrefix
- }
-
- return &Store{client: opts.Client, prefix: prefix}, nil
-}
-
-// Sync publishes one replica's counts, claims the bucket if asked and if it is
-// unclaimed, and reports any decision published for it - in a single round
-// trip.
-func (s *Store) Sync(ctx context.Context, req bandit.SyncRequest) (bandit.SyncResult, error) {
- epochMillis := req.EpochMillis
- if epochMillis <= 0 {
- epochMillis = 1
- }
-
- lead := "0"
- if req.Lead {
- lead = "1"
- }
-
- args := make([]any, 0, 6+4*len(req.Counts))
- args = append(args,
- s.keyBase(req.Namespace),
- strconv.FormatInt(epochMillis, 10),
- strconv.FormatInt(millis(req.CounterTTL), 10),
- strconv.FormatInt(millis(req.LeaderTTL), 10),
- req.NodeID,
- lead,
- )
-
- for _, count := range req.Counts {
- // Zero deltas are dropped rather than sent. They would cost a HINCRBY
- // each and create the counter field for an arm that measured nothing,
- // which reads back as evidence of a zero hit rate rather than as an
- // absence of evidence.
- if count.Hits != 0 {
- args = append(args,
- countField(count.Policy, count.Role, true),
- strconv.FormatInt(count.Hits, 10))
- }
- if count.Misses != 0 {
- args = append(args,
- countField(count.Policy, count.Role, false),
- strconv.FormatInt(count.Misses, 10))
- }
- }
-
- raw, err := syncScript.Run(ctx, s.client, []string{s.anchorKey(req.Namespace)}, args...).Result()
- if err != nil {
- return bandit.SyncResult{}, fmt.Errorf("redis: sync: %w", err)
- }
-
- return parseSyncReply(raw)
-}
-
-// parseSyncReply converts the script's three-element reply.
-func parseSyncReply(raw any) (bandit.SyncResult, error) {
- values, ok := raw.([]any)
- if !ok || len(values) != 3 {
- return bandit.SyncResult{}, fmt.Errorf("redis: sync: unexpected reply %T %v", raw, raw)
- }
-
- bucket, ok := values[0].(int64)
- if !ok {
- return bandit.SyncResult{}, fmt.Errorf("redis: sync: bucket is %T, want integer", values[0])
- }
-
- leader, ok := values[1].(int64)
- if !ok {
- return bandit.SyncResult{}, fmt.Errorf("redis: sync: leader flag is %T, want integer", values[1])
- }
-
- result := bandit.SyncResult{
- Bucket: bandit.Bucket(bucket),
- Leader: leader == 1,
- }
-
- decision, ok := values[2].(string)
- if !ok {
- return bandit.SyncResult{}, fmt.Errorf("redis: sync: decision is %T, want string", values[2])
- }
- if decision != "" {
- policy, err := parsePolicy(decision)
- if err != nil {
- return bandit.SyncResult{}, err
- }
- result.Decision, result.HasDecision = policy, true
- }
-
- return result, nil
-}
-
-// Window reads the aggregated counts for a range of buckets, pipelined into
-// one round trip. Buckets that expired or were never written come back empty
-// and are omitted, so a leader can tell a quiet epoch from a missing one.
-func (s *Store) Window(
- ctx context.Context,
- namespace string,
- first, last bandit.Bucket,
-) ([]bandit.WindowCounts, error) {
- if first > last {
- return nil, nil
- }
-
- buckets := make([]bandit.Bucket, 0, last-first+1)
- commands := make([]*goredis.MapStringStringCmd, 0, last-first+1)
-
- pipe := s.client.Pipeline()
- for bucket := first; bucket <= last; bucket++ {
- buckets = append(buckets, bucket)
- commands = append(commands, pipe.HGetAll(ctx, s.countsKey(namespace, bucket)))
- }
-
- // redis.Nil surfaces here only if a command in the pipeline missed, which
- // HGETALL never does - it returns an empty map. Any other error is real.
- if _, err := pipe.Exec(ctx); err != nil && !errors.Is(err, goredis.Nil) {
- return nil, fmt.Errorf("redis: window: %w", err)
- }
-
- window := make([]bandit.WindowCounts, 0, len(commands))
- for i, cmd := range commands {
- fields, err := cmd.Result()
- if err != nil {
- if errors.Is(err, goredis.Nil) {
- continue
- }
-
- return nil, fmt.Errorf("redis: window: bucket %d: %w", buckets[i], err)
- }
- if len(fields) == 0 {
- continue
- }
-
- arms := parseArms(fields)
- if len(arms) == 0 {
- continue
- }
-
- window = append(window, bandit.WindowCounts{Bucket: buckets[i], Arms: arms})
- }
-
- return window, nil
-}
-
-// parseArms turns a bucket's hash into per-arm counts, skipping anything it
-// does not recognise.
-func parseArms(fields map[string]string) map[bandit.ArmKey]ascache.PolicyStats {
- arms := make(map[bandit.ArmKey]ascache.PolicyStats, len(fields)/2)
-
- for field, value := range fields {
- policy, role, hits, ok := parseCountField(field)
- if !ok {
- continue
- }
-
- count, err := strconv.ParseInt(value, 10, 64)
- if err != nil {
- continue
- }
-
- key := bandit.ArmKey{Policy: policy, Role: role}
- stats := arms[key]
- if hits {
- stats.Hits += count
- } else {
- stats.Misses += count
- }
- arms[key] = stats
- }
-
- return arms
-}
-
-// Decide publishes the policy the fleet should run for a bucket and returns
-// the decision actually in force for it.
-func (s *Store) Decide(
- ctx context.Context,
- namespace string,
- bucket bandit.Bucket,
- policy ascache.PolicyType,
- ttl time.Duration,
-) (ascache.PolicyType, error) {
- raw, err := decideScript.Run(ctx, s.client,
- []string{s.anchorKey(namespace)},
- s.keyBase(namespace),
- strconv.FormatInt(int64(bucket), 10),
- strconv.FormatUint(uint64(policy), 10),
- strconv.FormatInt(millis(ttl), 10),
- ).Result()
- if err != nil {
- return ascache.Undefined, fmt.Errorf("redis: decide: %w", err)
- }
-
- text, ok := raw.(string)
- if !ok {
- return ascache.Undefined, fmt.Errorf("redis: decide: reply is %T, want string", raw)
- }
-
- return parsePolicy(text)
-}
-
-// Close releases the store. The client belongs to the caller and is left open.
-func (s *Store) Close() error { return nil }
-
-// parsePolicy reads the numeric PolicyType a decision is stored as.
-func parsePolicy(text string) (ascache.PolicyType, error) {
- number, err := strconv.ParseUint(text, 10, 64)
- if err != nil {
- return ascache.Undefined, fmt.Errorf("redis: decision %q is not a policy: %w", text, err)
- }
-
- return ascache.PolicyType(number), nil
-}
-
-// millis rounds a TTL up to at least one millisecond. PEXPIRE with a
-// non-positive argument deletes the key it was meant to keep alive.
-func millis(d time.Duration) int64 {
- if ms := d.Milliseconds(); ms > 0 {
- return ms
- }
-
- return 1
-}
diff --git a/bandit/redis/store_test.go b/bandit/redis/store_test.go
deleted file mode 100644
index 2a58044..0000000
--- a/bandit/redis/store_test.go
+++ /dev/null
@@ -1,411 +0,0 @@
-package redis
-
-import (
- "context"
- "os"
- "strconv"
- "testing"
- "time"
-
- "github.com/alicebob/miniredis/v2"
- goredis "github.com/redis/go-redis/v9"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- ascache "github.com/sshaplygin/as-cache"
- "github.com/sshaplygin/as-cache/bandit"
-)
-
-const testEpoch = time.Second
-
-// runID separates one run's keys from the last one's on a real server, which
-// unlike miniredis is not thrown away between runs.
-var runID = strconv.FormatInt(time.Now().UnixNano(), 36)
-
-// newStore returns a Store over miniredis, or over a real server when
-// AS_CACHE_REDIS_ADDR is set.
-//
-// The fake covers the logic; the real server covers the assumption the logic
-// rests on, which is that the Lua runs at all - TIME inside a script, SET NX
-// with PX, and HINCRBY on a hash the script names itself. Those are exactly
-// the things a fake can get wrong in the direction of being too permissive,
-// so the same tests run against both.
-func newStore(t *testing.T) (*Store, goredis.UniversalClient, *miniredis.Miniredis) {
- t.Helper()
-
- if addr := os.Getenv("AS_CACHE_REDIS_ADDR"); addr != "" {
- client := goredis.NewClient(&goredis.Options{Addr: addr})
- require.NoError(t, client.Ping(t.Context()).Err(), "AS_CACHE_REDIS_ADDR is set but unreachable")
-
- // A real server is shared and outlives the run, so each test gets its
- // own key prefix rather than flushing someone else's data. The run id
- // is part of it because buckets come from the clock: two runs within
- // the same second would otherwise land in the same bucket and the
- // second would find the first's leadership already claimed and its
- // decisions already published. Everything written carries a TTL, so
- // the extra prefixes expire on their own.
- store, err := New(Options{
- Client: client,
- KeyPrefix: "asctest:" + runID + ":" + t.Name(),
- })
- require.NoError(t, err)
- t.Cleanup(func() {
- _ = store.Close()
- _ = client.Close()
- })
-
- return store, client, nil
- }
-
- server := miniredis.RunT(t)
- client := goredis.NewClient(&goredis.Options{Addr: server.Addr()})
- t.Cleanup(func() { _ = client.Close() })
-
- store, err := New(Options{Client: client})
- require.NoError(t, err)
- t.Cleanup(func() { _ = store.Close() })
-
- return store, client, server
-}
-
-// expire advances past a TTL. A fake is fast-forwarded; a real server has to
-// be waited out, which is why the TTLs in these tests are short.
-func expire(t *testing.T, server *miniredis.Miniredis, d time.Duration) {
- t.Helper()
-
- if server != nil {
- server.FastForward(d)
-
- return
- }
- time.Sleep(d)
-}
-
-func syncRequest(namespace, node string, counts ...bandit.ArmCounts) bandit.SyncRequest {
- return bandit.SyncRequest{
- Namespace: namespace,
- NodeID: node,
- Counts: counts,
- EpochMillis: testEpoch.Milliseconds(),
- CounterTTL: 12 * testEpoch,
- LeaderTTL: 2 * testEpoch,
- }
-}
-
-func shadow(policy ascache.PolicyType, hits, misses int64) bandit.ArmCounts {
- return bandit.ArmCounts{Policy: policy, Role: bandit.RoleShadow, Hits: hits, Misses: misses}
-}
-
-func TestNew_RejectsANilClient(t *testing.T) {
- _, err := New(Options{})
- assert.ErrorIs(t, err, ErrNilClient)
-}
-
-func TestStore_SyncReportsAServerDerivedBucket(t *testing.T) {
- store, _, _ := newStore(t)
-
- result, err := store.Sync(t.Context(), syncRequest("ns", "a"))
- require.NoError(t, err)
-
- // The bucket is the server's clock divided by the epoch. Checking it lands
- // near the client's clock proves the script read a real time rather than
- // returning a constant.
- expected := bandit.Bucket(time.Now().UnixMilli() / testEpoch.Milliseconds())
- assert.InDelta(t, int64(expected), int64(result.Bucket), 5)
-}
-
-func TestStore_SumsCountsAcrossReplicas(t *testing.T) {
- store, _, _ := newStore(t)
-
- result, err := store.Sync(t.Context(), syncRequest("ns", "a", shadow(ascache.LRU, 10, 5)))
- require.NoError(t, err)
- _, err = store.Sync(t.Context(), syncRequest("ns", "b", shadow(ascache.LRU, 20, 15)))
- require.NoError(t, err)
-
- window, err := store.Window(t.Context(), "ns", result.Bucket, result.Bucket)
- require.NoError(t, err)
- require.Len(t, window, 1)
-
- assert.Equal(t,
- ascache.PolicyStats{Hits: 30, Misses: 20},
- window[0].Arms[bandit.ArmKey{Policy: ascache.LRU, Role: bandit.RoleShadow}])
-}
-
-func TestStore_KeepsRolesApart(t *testing.T) {
- store, _, _ := newStore(t)
-
- result, err := store.Sync(t.Context(), syncRequest("ns", "a",
- bandit.ArmCounts{Policy: ascache.LRU, Role: bandit.RoleActive, Hits: 90, Misses: 10},
- bandit.ArmCounts{Policy: ascache.LRU, Role: bandit.RoleShadow, Hits: 40, Misses: 60},
- ))
- require.NoError(t, err)
-
- window, err := store.Window(t.Context(), "ns", result.Bucket, result.Bucket)
- require.NoError(t, err)
- require.Len(t, window, 1)
-
- assert.Equal(t, ascache.PolicyStats{Hits: 90, Misses: 10},
- window[0].Arms[bandit.ArmKey{Policy: ascache.LRU, Role: bandit.RoleActive}])
- assert.Equal(t, ascache.PolicyStats{Hits: 40, Misses: 60},
- window[0].Arms[bandit.ArmKey{Policy: ascache.LRU, Role: bandit.RoleShadow}])
-}
-
-func TestStore_NamespacesDoNotPool(t *testing.T) {
- store, _, _ := newStore(t)
-
- result, err := store.Sync(t.Context(), syncRequest("one", "a", shadow(ascache.LRU, 10, 0)))
- require.NoError(t, err)
- _, err = store.Sync(t.Context(), syncRequest("two", "b", shadow(ascache.LRU, 999, 0)))
- require.NoError(t, err)
-
- window, err := store.Window(t.Context(), "one", result.Bucket, result.Bucket)
- require.NoError(t, err)
- require.Len(t, window, 1)
-
- assert.Equal(t, int64(10),
- window[0].Arms[bandit.ArmKey{Policy: ascache.LRU, Role: bandit.RoleShadow}].Hits)
-}
-
-func TestStore_OneLeaderPerBucket(t *testing.T) {
- store, _, _ := newStore(t)
-
- leaders := 0
- for _, node := range []string{"a", "b", "c", "d", "e"} {
- req := syncRequest("ns", node)
- req.Lead = true
- result, err := store.Sync(t.Context(), req)
- require.NoError(t, err)
- if result.Leader {
- leaders++
- }
- }
-
- assert.Equal(t, 1, leaders)
-}
-
-func TestStore_LeadershipIsNotClaimedUnlessAsked(t *testing.T) {
- store, _, _ := newStore(t)
-
- result, err := store.Sync(t.Context(), syncRequest("ns", "a"))
- require.NoError(t, err)
- assert.False(t, result.Leader)
-
- // And a later replica that does ask still gets it.
- req := syncRequest("ns", "b")
- req.Lead = true
- result, err = store.Sync(t.Context(), req)
- require.NoError(t, err)
- assert.True(t, result.Leader)
-}
-
-func TestStore_DecisionIsImmutableAndReportsWhatIsInForce(t *testing.T) {
- store, _, _ := newStore(t)
-
- first, err := store.Decide(t.Context(), "ns", 42, ascache.TinyLFU, time.Minute)
- require.NoError(t, err)
- assert.Equal(t, ascache.TinyLFU, first)
-
- second, err := store.Decide(t.Context(), "ns", 42, ascache.LRU, time.Minute)
- require.NoError(t, err)
- assert.Equal(t, ascache.TinyLFU, second,
- "a leader that lost the race must follow the fleet, not its own draw")
-}
-
-func TestStore_SyncReadsBackAPublishedDecision(t *testing.T) {
- store, _, _ := newStore(t)
-
- result, err := store.Sync(t.Context(), syncRequest("ns", "a"))
- require.NoError(t, err)
- assert.False(t, result.HasDecision)
-
- _, err = store.Decide(t.Context(), "ns", result.Bucket, ascache.TwoQueue, time.Minute)
- require.NoError(t, err)
-
- again, err := store.Sync(t.Context(), syncRequest("ns", "b"))
- require.NoError(t, err)
- require.True(t, again.HasDecision)
- assert.Equal(t, ascache.TwoQueue, again.Decision)
-}
-
-func TestStore_WindowOmitsBucketsThatHoldNothing(t *testing.T) {
- store, _, _ := newStore(t)
-
- result, err := store.Sync(t.Context(), syncRequest("ns", "a", shadow(ascache.LRU, 1, 1)))
- require.NoError(t, err)
-
- window, err := store.Window(t.Context(), "ns", result.Bucket-8, result.Bucket)
- require.NoError(t, err)
-
- require.Len(t, window, 1, "a bucket nobody wrote is absent, not present and zero")
- assert.Equal(t, result.Bucket, window[0].Bucket)
-}
-
-func TestStore_WindowOfAnInvertedRangeIsEmpty(t *testing.T) {
- store, _, _ := newStore(t)
-
- window, err := store.Window(t.Context(), "ns", 100, 10)
- require.NoError(t, err)
- assert.Empty(t, window)
-}
-
-func TestStore_ZeroCountsCreateNoCounters(t *testing.T) {
- store, _, _ := newStore(t)
-
- // An arm that measured nothing has produced no evidence. Writing a zero
- // would read back as evidence of a zero hit rate.
- result, err := store.Sync(t.Context(), syncRequest("ns", "a", shadow(ascache.LRU, 0, 0)))
- require.NoError(t, err)
-
- window, err := store.Window(t.Context(), "ns", result.Bucket, result.Bucket)
- require.NoError(t, err)
- assert.Empty(t, window)
-}
-
-func TestStore_CountersExpire(t *testing.T) {
- store, _, server := newStore(t)
-
- // A fleet that stops running must leave nothing behind in a store it
- // shares with everything else, so this checks both halves of that: the TTL
- // is actually attached, and the data really does go.
- const ttl = 300 * time.Millisecond
-
- req := syncRequest("ns", "a", shadow(ascache.LRU, 10, 0))
- req.CounterTTL = ttl
- result, err := store.Sync(t.Context(), req)
- require.NoError(t, err)
-
- window, err := store.Window(t.Context(), "ns", result.Bucket, result.Bucket)
- require.NoError(t, err)
- require.Len(t, window, 1)
-
- expire(t, server, 2*ttl)
-
- window, err = store.Window(t.Context(), "ns", result.Bucket, result.Bucket)
- require.NoError(t, err)
- assert.Empty(t, window)
-}
-
-func TestStore_EveryKeyItWritesHasATTL(t *testing.T) {
- store, client, _ := newStore(t)
-
- req := syncRequest("ns", "a", shadow(ascache.LRU, 1, 1))
- req.Lead = true
- result, err := store.Sync(t.Context(), req)
- require.NoError(t, err)
-
- _, err = store.Decide(t.Context(), "ns", result.Bucket, ascache.LRU, time.Minute)
- require.NoError(t, err)
-
- keys, err := client.Keys(t.Context(), store.prefix+":*").Result()
- require.NoError(t, err)
- require.NotEmpty(t, keys)
-
- // A key without an expiry is a key that outlives the fleet that wrote it.
- for _, key := range keys {
- ttl, err := client.PTTL(t.Context(), key).Result()
- require.NoError(t, err)
- assert.Positive(t, ttl, "key %q was written without a TTL", key)
- }
-}
-
-func TestStore_EveryKeyForANamespaceSharesOneSlot(t *testing.T) {
- store, client, _ := newStore(t)
-
- req := syncRequest("ns", "a", shadow(ascache.LRU, 1, 1))
- req.Lead = true
- result, err := store.Sync(t.Context(), req)
- require.NoError(t, err)
-
- _, err = store.Decide(t.Context(), "ns", result.Bucket, ascache.LRU, time.Minute)
- require.NoError(t, err)
-
- keys, err := client.Keys(t.Context(), store.prefix+":*").Result()
- require.NoError(t, err)
- require.NotEmpty(t, keys)
-
- // The hash tag is what keeps a namespace in one Redis Cluster slot, which
- // is what lets the window pipeline and the scripts' computed key names
- // work on a cluster at all. Nothing here runs against a cluster, so this
- // checks the property the cluster behaviour rests on.
- for _, key := range keys {
- assert.Contains(t, key, "{ns}", "key %q is missing the hash tag", key)
- }
-}
-
-func TestStore_SurvivesAStrayFieldWrittenBySomethingElse(t *testing.T) {
- store, client, _ := newStore(t)
-
- result, err := store.Sync(t.Context(), syncRequest("ns", "a", shadow(ascache.LRU, 10, 5)))
- require.NoError(t, err)
-
- // The store may be shared. A field written by something else must not stop
- // a fleet reading its own counters.
- require.NoError(t, client.HSet(t.Context(),
- store.countsKey("ns", result.Bucket), "not-a-field", "nonsense").Err())
-
- window, err := store.Window(t.Context(), "ns", result.Bucket, result.Bucket)
- require.NoError(t, err)
- require.Len(t, window, 1)
-
- assert.Equal(t, ascache.PolicyStats{Hits: 10, Misses: 5},
- window[0].Arms[bandit.ArmKey{Policy: ascache.LRU, Role: bandit.RoleShadow}])
- assert.Len(t, window[0].Arms, 1)
-}
-
-func TestStore_ReportsAnUnreachableServer(t *testing.T) {
- client := goredis.NewClient(&goredis.Options{
- Addr: "127.0.0.1:1",
- DialTimeout: 200 * time.Millisecond,
- })
- t.Cleanup(func() { _ = client.Close() })
-
- store, err := New(Options{Client: client})
- require.NoError(t, err)
-
- _, err = store.Sync(t.Context(), syncRequest("ns", "a"))
- require.Error(t, err, "an unreachable store must report, so the bandit can fall back")
-
- _, err = store.Window(t.Context(), "ns", 0, 4)
- assert.Error(t, err)
-
- _, err = store.Decide(t.Context(), "ns", 1, ascache.LRU, time.Minute)
- assert.Error(t, err)
-}
-
-func TestStore_RespectsContextCancellation(t *testing.T) {
- store, _, _ := newStore(t)
-
- ctx, cancel := context.WithCancel(t.Context())
- cancel()
-
- _, err := store.Sync(ctx, syncRequest("ns", "a"))
- assert.ErrorIs(t, err, context.Canceled)
-}
-
-func TestParseCountField_RoundTrips(t *testing.T) {
- policies := []ascache.PolicyType{
- ascache.LRU, ascache.LFU, ascache.TwoQueue,
- ascache.ARC, ascache.Random, ascache.TTL, ascache.TinyLFU,
- }
-
- for _, policy := range policies {
- for _, role := range []bandit.Role{bandit.RoleActive, bandit.RoleShadow} {
- for _, hits := range []bool{true, false} {
- field := countField(policy, role, hits)
-
- gotPolicy, gotRole, gotHits, ok := parseCountField(field)
- require.True(t, ok, "field %q did not parse", field)
- assert.Equal(t, policy, gotPolicy)
- assert.Equal(t, role, gotRole)
- assert.Equal(t, hits, gotHits)
- }
- }
- }
-}
-
-func TestParseCountField_RejectsJunk(t *testing.T) {
- for _, field := range []string{"", "1", "1:s", "1:s:h:x", "x:s:h", "1:z:h", "1:s:z"} {
- _, _, _, ok := parseCountField(field)
- assert.False(t, ok, "field %q should not have parsed", field)
- }
-}
diff --git a/bandit/snapshot.go b/bandit/snapshot.go
deleted file mode 100644
index 28523a9..0000000
--- a/bandit/snapshot.go
+++ /dev/null
@@ -1,162 +0,0 @@
-package bandit
-
-import (
- "fmt"
- "sort"
- "strings"
- "time"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-// ArmEvidence is one arm's pooled, decayed evidence as the fleet last reported
-// it.
-type ArmEvidence struct {
- Policy string `json:"policy"`
- // Hits and Misses are weighted sums over the window, so they are
- // fractional and are not request counts. HitRate is what the decision was
- // actually made on.
- Hits float64 `json:"hits"`
- Misses float64 `json:"misses"`
- HitRate float64 `json:"hit_rate"`
-}
-
-// Snapshot is what a distributed bandit is doing, for monitoring.
-//
-// A cache that changes its own eviction policy is only safe to run if you can
-// see what it is doing, and one that takes that decision from other machines
-// needs one more thing visible: whether it is actually hearing from them.
-// Fallback is the field to alert on. It is not an error - the cache keeps
-// working, deciding locally - but a fleet where every replica has quietly
-// fallen back is no longer a fleet, and nothing else about the cache looks any
-// different.
-type Snapshot struct {
- // Selection is the policy this replica's bandit is currently returning.
- Selection string `json:"selection"`
- // Mode is "leader" or "shared-posterior".
- Mode string `json:"mode"`
- // NodeID identifies this replica, and Namespace is the fingerprinted
- // namespace it coordinates under - the plain namespace plus a hash of the
- // cache's arms, capacity and sample rate. Two replicas that should be
- // pooling but are not will differ here, and nowhere else.
- NodeID string `json:"node_id"`
- Namespace string `json:"namespace"`
- // Regime is the human-readable form of what that fingerprint covers.
- Regime string `json:"regime"`
-
- // Fallback reports that the store has been unreachable for longer than
- // FallbackAfter and this replica is deciding on its own evidence.
- Fallback bool `json:"fallback"`
- // LastSyncAge is how long ago the last successful round trip completed.
- LastSyncAge time.Duration `json:"last_sync_age_ns"`
- // LastError is the most recent failure, empty if the last round trip
- // succeeded.
- LastError string `json:"last_error,omitempty"`
- // LastBucket is the coordination bucket of the last successful sync, by
- // the store's clock.
- LastBucket int64 `json:"last_bucket"`
-
- // Syncs and SyncFailures count round trips. Leaderships counts the buckets
- // this replica led, Decisions the fleet decisions it applied, and Rejected
- // the ones it refused because they named a policy it does not have.
- Syncs int64 `json:"syncs"`
- SyncFailures int64 `json:"sync_failures"`
- Leaderships int64 `json:"leaderships"`
- Decisions int64 `json:"decisions"`
- Rejected int64 `json:"rejected"`
-
- // Fleet holds the pooled evidence behind the last decision this replica
- // computed, best hit rate first.
- //
- // It is only populated on a replica that read the window: always under
- // ModeSharedPosterior, and only while leading under ModeLeader. A follower
- // reporting an empty Fleet is working correctly - it is applying a
- // decision, not making one - so read it from whichever replica has
- // Leaderships climbing.
- Fleet []ArmEvidence `json:"fleet"`
-}
-
-// Snapshot reads the bandit's current state. It is safe to call at any time
-// and does not disturb coordination.
-func (d *Distributed) Snapshot() Snapshot {
- d.mu.Lock()
- defer d.mu.Unlock()
-
- mode := "leader"
- if d.cfg.Mode == ModeSharedPosterior {
- mode = "shared-posterior"
- }
-
- snapshot := Snapshot{
- Selection: ascache.PolicyType(d.selection.Load()).String(),
- Mode: mode,
- NodeID: d.cfg.NodeID,
- Namespace: d.namespace,
- Fallback: d.state.fallback,
- LastBucket: int64(d.state.lastBucket),
- Syncs: d.state.syncs,
- SyncFailures: d.state.syncFailures,
- Leaderships: d.state.leaderships,
- Decisions: d.state.decisions,
- Rejected: d.state.rejected,
- Fleet: make([]ArmEvidence, 0, len(d.state.fleet)),
- }
-
- if d.haveShape {
- snapshot.Regime = d.shape.String()
- }
- if d.state.lastErr != nil {
- snapshot.LastError = d.state.lastErr.Error()
- }
- if !d.state.lastSync.IsZero() {
- snapshot.LastSyncAge = d.cfg.Now().Sub(d.state.lastSync)
- }
-
- for policy, arm := range d.state.fleet {
- snapshot.Fleet = append(snapshot.Fleet, ArmEvidence{
- Policy: policy.String(),
- Hits: arm.Hits,
- Misses: arm.Misses,
- HitRate: arm.hitRate(),
- })
- }
-
- // Ties broken by name so an unchanged fleet renders identically on every
- // scrape; ranging a map alone would reorder equal arms on each call.
- sort.SliceStable(snapshot.Fleet, func(i, j int) bool {
- if snapshot.Fleet[i].HitRate != snapshot.Fleet[j].HitRate {
- return snapshot.Fleet[i].HitRate > snapshot.Fleet[j].HitRate
- }
-
- return snapshot.Fleet[i].Policy < snapshot.Fleet[j].Policy
- })
-
- return snapshot
-}
-
-// String renders the snapshot as a short human-readable summary.
-func (s Snapshot) String() string {
- var b strings.Builder
-
- fmt.Fprintf(&b, "%s running %s, %s mode", s.NodeID, s.Selection, s.Mode)
- if s.Fallback {
- fmt.Fprintf(&b, " (FALLBACK: store unreachable for %s)", s.LastSyncAge.Round(time.Millisecond))
- }
- b.WriteString("\n")
-
- fmt.Fprintf(&b, "namespace %s [%s]\n", s.Namespace, s.Regime)
- fmt.Fprintf(&b, "%d syncs, %d failures, %d led, %d decisions applied, %d rejected\n",
- s.Syncs, s.SyncFailures, s.Leaderships, s.Decisions, s.Rejected)
-
- if len(s.Fleet) == 0 {
- return b.String()
- }
-
- fmt.Fprintf(&b, "\n%-10s %9s %14s %14s\n", "policy", "hit rate", "hits", "misses")
- for _, arm := range s.Fleet {
- fmt.Fprintf(&b, "%-10s %8.2f%% %14.0f %14.0f\n",
- arm.Policy, arm.HitRate*100, arm.Hits, arm.Misses)
- }
-
- return b.String()
-}
diff --git a/bandit/store.go b/bandit/store.go
deleted file mode 100644
index 25e6d9b..0000000
--- a/bandit/store.go
+++ /dev/null
@@ -1,163 +0,0 @@
-package bandit
-
-import (
- "context"
- "time"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-// Bucket identifies one coordination epoch fleet-wide. It is derived from the
-// store's clock, never from a replica's, so a machine with a skewed clock
-// cannot write its counts into a window nobody else is reading.
-type Bucket int64
-
-// Role distinguishes how an arm's counts were measured. It matters because the
-// two are not measured the same way: the active arm runs at the cache's full
-// capacity, and every shadow runs on a miniature of it. The rates are
-// comparable by design, but shadows measure 1 to 3 points pessimistic in
-// practice, so pooling one arm's active-role counts with another's shadow-role
-// counts hands the first a systematic advantage.
-type Role uint8
-
-const (
- // RoleShadow is an arm measured as a shadow: miniature capacity, no real
- // values, no traffic served.
- RoleShadow Role = iota + 1
- // RoleActive is the arm that was serving traffic on the reporting replica.
- RoleActive
-)
-
-// String names the role, for keys and for logs.
-func (r Role) String() string {
- if r == RoleActive {
- return "active"
- }
-
- return "shadow"
-}
-
-// ArmCounts is one arm's measurements from one replica, in one role.
-type ArmCounts struct {
- Policy ascache.PolicyType
- Role Role
- Hits int64
- Misses int64
-}
-
-// SyncRequest is the once-per-coordination-epoch call every replica makes.
-//
-// It bundles three things that would otherwise be separate round trips, all of
-// which every replica needs on every tick: publish what I measured, tell me
-// which bucket that landed in, and let me lead this bucket if nobody has
-// claimed it.
-type SyncRequest struct {
- // Namespace scopes every key this call touches. Replicas pool with each
- // other exactly when their namespaces match, which is why it carries a
- // fingerprint of the cache's shape - see Config.Namespace.
- Namespace string
-
- // NodeID identifies the replica, for leadership.
- NodeID string
-
- // Counts is what this replica measured since its last sync. It may be
- // empty on a tick where the cache reported nothing.
- Counts []ArmCounts
-
- // EpochMillis is the coordination epoch length. The store divides its own
- // clock by it to derive the bucket, so every replica agrees on bucket
- // boundaries without their clocks having to agree on anything.
- EpochMillis int64
-
- // CounterTTL is how long a bucket's counters should outlive it. It must
- // comfortably exceed Window * the epoch length, or the leader will read a
- // window with holes in it where buckets have already expired.
- CounterTTL time.Duration
-
- // LeaderTTL is how long a leadership claim lasts. It bounds nothing
- // important: leadership is per-bucket and the decision it produces is
- // written once, so a claim that outlives its usefulness only means the
- // bucket has no leader, and the next bucket gets one.
- LeaderTTL time.Duration
-
- // Lead asks to claim leadership of the current bucket if it is unclaimed.
- // A replica configured for shared-posterior mode never sets it.
- Lead bool
-}
-
-// SyncResult is what the store knew at the moment of the sync.
-type SyncResult struct {
- // Bucket is the bucket the counts were added to, by the store's clock.
- Bucket Bucket
-
- // Leader reports whether this replica claimed leadership of Bucket. At
- // most one replica per bucket ever sees true.
- Leader bool
-
- // Decision is the policy published for Bucket, and HasDecision reports
- // whether one had been published when the sync ran. A replica that syncs
- // before its leader has decided sees no decision and keeps using the
- // previous one, which is the mode's one epoch of built-in staleness.
- Decision ascache.PolicyType
- HasDecision bool
-}
-
-// WindowCounts is the fleet's aggregated measurements for one bucket.
-type WindowCounts struct {
- Bucket Bucket
- // Arms holds the summed counts across every replica that published into
- // the bucket, keyed by policy and role.
- Arms map[ArmKey]ascache.PolicyStats
-}
-
-// ArmKey identifies one arm's counts in one role.
-type ArmKey struct {
- Policy ascache.PolicyType
- Role Role
-}
-
-// Store is the shared state a fleet of caches coordinates through.
-//
-// It is deliberately dumb: it counts, it claims, it reads back. Every decision
-// about what the numbers mean - how far back to look, how much to discount,
-// which arm wins - stays in this package, so backing it with a different store
-// never changes the selection behaviour.
-//
-// Implementations must be safe for concurrent use, and must respect the
-// context: a replica whose store has stopped responding falls back to deciding
-// locally, and it can only do that if these calls actually return.
-type Store interface {
- // Sync publishes one replica's counts and reports the bucket they landed
- // in, whether this replica leads that bucket, and any decision already
- // published for it.
- Sync(ctx context.Context, req SyncRequest) (SyncResult, error)
-
- // Window reads the aggregated counts for buckets first through last
- // inclusive. Buckets that have expired or were never written are omitted
- // rather than reported as zero, so a caller can tell a quiet epoch from a
- // missing one. Only the leader of a bucket calls it.
- Window(ctx context.Context, namespace string, first, last Bucket) ([]WindowCounts, error)
-
- // Decide publishes the policy the fleet should run for a bucket and
- // returns the decision that is actually in force for it.
- //
- // It must not overwrite a decision already published for that bucket: the
- // decision is what replicas act on, and one that changed underneath them
- // would move the fleet mid-epoch for no reason. Publishing to an
- // already-decided bucket is not an error, and returns the decision that
- // was already there - which is why this returns a policy at all. A leader
- // that acted on its own draw while the fleet acted on someone else's would
- // put the one replica making the decisions out of step with every replica
- // following them.
- Decide(
- ctx context.Context,
- namespace string,
- bucket Bucket,
- policy ascache.PolicyType,
- ttl time.Duration,
- ) (ascache.PolicyType, error)
-
- // Close releases whatever the store holds. It does not close a connection
- // the caller supplied.
- Close() error
-}
diff --git a/bandit/window.go b/bandit/window.go
deleted file mode 100644
index 48837f8..0000000
--- a/bandit/window.go
+++ /dev/null
@@ -1,133 +0,0 @@
-package bandit
-
-import (
- "math"
- "math/rand/v2"
- "slices"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-// weighted is one arm's pooled evidence after the window has been discounted.
-// It is fractional because the weighting is, which is also why it is not
-// ascache.PolicyStats.
-type weighted struct {
- Hits float64
- Misses float64
-}
-
-func (w weighted) total() float64 { return w.Hits + w.Misses }
-
-func (w weighted) hitRate() float64 {
- if w.total() == 0 {
- return 0
- }
-
- return w.Hits / w.total()
-}
-
-// aggregate pools a window of per-bucket fleet counts into one posterior per
-// arm, weighting each bucket by decay raised to its age relative to newest.
-//
-// Ageing is done here rather than in the store because the store has many
-// writers. Decaying a shared counter in place would apply the multiplication
-// once per replica per epoch, so a fleet of fifty would forget fifty times
-// faster than a fleet of one, and the same configuration would mean something
-// different at every scale. Plain sums in the store, weighting on read, and
-// the arithmetic is identical whatever the fleet size.
-func aggregate(
- window []WindowCounts,
- newest Bucket,
- decay float64,
- mode EvidenceMode,
-) map[ascache.PolicyType]weighted {
- pooled := make(map[ascache.PolicyType]weighted)
-
- for _, bucket := range window {
- age := int64(newest - bucket.Bucket)
- if age < 0 {
- // A bucket newer than the reference is still being written by the
- // rest of the fleet, so it is a partial count that would weight
- // whichever replicas happen to have arrived already.
- continue
- }
-
- weight := math.Pow(decay, float64(age))
- if weight == 0 {
- continue
- }
-
- for key, stats := range bucket.Arms {
- if mode == EvidenceShadowOnly && key.Role == RoleActive {
- continue
- }
-
- arm := pooled[key.Policy]
- arm.Hits += weight * float64(stats.Hits)
- arm.Misses += weight * float64(stats.Misses)
- pooled[key.Policy] = arm
- }
- }
-
- return pooled
-}
-
-// capEvidence scales each arm's counts down to at most maxEvidence
-// observations, preserving its hit rate exactly.
-//
-// Pooling is what makes this necessary. A Beta posterior's width shrinks with
-// the square root of the evidence behind it, and a fleet supplies evidence in
-// proportion to its size: a thousand replicas hand Thompson sampling a
-// posterior so sharp that every draw returns the same arm, and the bandit
-// stops exploring at exactly the scale where it has the most to gain from
-// noticing a change. Capping the effective sample size keeps the rate estimate
-// and throws away the surplus certainty.
-//
-// At the default cap an arm sitting at a 50% hit rate has a posterior standard
-// deviation of about 0.16 points, so arms half a point apart are reliably
-// separated and arms within a tenth of a point keep being explored.
-func capEvidence(pooled map[ascache.PolicyType]weighted, maxEvidence float64) {
- if maxEvidence <= 0 {
- return
- }
-
- for policy, arm := range pooled {
- total := arm.total()
- if total <= maxEvidence {
- continue
- }
-
- scale := maxEvidence / total
- pooled[policy] = weighted{Hits: arm.Hits * scale, Misses: arm.Misses * scale}
- }
-}
-
-// draw picks an arm by Thompson sampling over the pooled posteriors.
-//
-// Arms are visited in PolicyType order and ties are broken towards the lower
-// PolicyType, so the only nondeterminism is the sampling itself. Ranging the
-// map directly would make a fleet's decisions depend on Go's map seed, which
-// is invisible, unreproducible, and would quietly break the tie-breaking that
-// keeps an unchanged fleet on an unchanged policy.
-func draw(rng *rand.Rand, pooled map[ascache.PolicyType]weighted) ascache.PolicyType {
- arms := make([]ascache.PolicyType, 0, len(pooled))
- for policy := range pooled {
- arms = append(arms, policy)
- }
- slices.Sort(arms)
-
- best := ascache.Undefined
- bestSample := -1.0
-
- for _, policy := range arms {
- arm := pooled[policy]
- // The +1s are a uniform prior: an arm with no evidence is sampled
- // across the whole range rather than pinned at zero and never tried.
- sample := betaSample(rng, 1+arm.Hits, 1+arm.Misses)
- if sample > bestSample {
- best, bestSample = policy, sample
- }
- }
-
- return best
-}
diff --git a/bandit/window_test.go b/bandit/window_test.go
deleted file mode 100644
index 9e7aedd..0000000
--- a/bandit/window_test.go
+++ /dev/null
@@ -1,180 +0,0 @@
-package bandit
-
-import (
- "math"
- "math/rand/v2"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- ascache "github.com/sshaplygin/as-cache"
-)
-
-func TestAggregate_WeightsBucketsByAge(t *testing.T) {
- window := []WindowCounts{
- {Bucket: 8, Arms: map[ArmKey]ascache.PolicyStats{
- {Policy: ascache.LRU, Role: RoleShadow}: {Hits: 100, Misses: 0},
- }},
- {Bucket: 10, Arms: map[ArmKey]ascache.PolicyStats{
- {Policy: ascache.LRU, Role: RoleShadow}: {Hits: 100, Misses: 0},
- }},
- }
-
- pooled := aggregate(window, 10, 0.5, EvidenceAll)
-
- // Bucket 10 is the newest so it is unweighted; bucket 8 is two buckets old
- // and carries 0.5^2 of its counts.
- assert.InDelta(t, 100+25.0, pooled[ascache.LRU].Hits, 1e-9)
-}
-
-func TestAggregate_IgnoresBucketsNewerThanTheReference(t *testing.T) {
- // The current bucket is still being written by the rest of the fleet.
- // Counting it would weight whichever replicas happened to sync first.
- window := []WindowCounts{
- {Bucket: 11, Arms: map[ArmKey]ascache.PolicyStats{
- {Policy: ascache.LRU, Role: RoleShadow}: {Hits: 999, Misses: 0},
- }},
- {Bucket: 10, Arms: map[ArmKey]ascache.PolicyStats{
- {Policy: ascache.LRU, Role: RoleShadow}: {Hits: 5, Misses: 5},
- }},
- }
-
- pooled := aggregate(window, 10, 1, EvidenceAll)
-
- assert.InDelta(t, 5.0, pooled[ascache.LRU].Hits, 1e-9)
- assert.InDelta(t, 5.0, pooled[ascache.LRU].Misses, 1e-9)
-}
-
-func TestAggregate_ShadowOnlyDropsActiveRole(t *testing.T) {
- window := []WindowCounts{
- {Bucket: 1, Arms: map[ArmKey]ascache.PolicyStats{
- {Policy: ascache.LRU, Role: RoleActive}: {Hits: 90, Misses: 10},
- {Policy: ascache.LRU, Role: RoleShadow}: {Hits: 40, Misses: 60},
- {Policy: ascache.TinyLFU, Role: RoleShadow}: {Hits: 50, Misses: 50},
- }},
- }
-
- all := aggregate(window, 1, 1, EvidenceAll)
- assert.InDelta(t, 0.65, all[ascache.LRU].hitRate(), 1e-9,
- "pooling both roles mixes the flattering active measurement in")
-
- shadowOnly := aggregate(window, 1, 1, EvidenceShadowOnly)
- assert.InDelta(t, 0.40, shadowOnly[ascache.LRU].hitRate(), 1e-9)
- assert.InDelta(t, 0.50, shadowOnly[ascache.TinyLFU].hitRate(), 1e-9)
-}
-
-func TestAggregate_EmptyWindowYieldsNoArms(t *testing.T) {
- assert.Empty(t, aggregate(nil, 5, 0.8, EvidenceAll))
-}
-
-func TestCapEvidence_PreservesRateAndBoundsTotal(t *testing.T) {
- pooled := map[ascache.PolicyType]weighted{
- ascache.LRU: {Hits: 9_000_000, Misses: 1_000_000},
- ascache.TinyLFU: {Hits: 50, Misses: 50},
- }
-
- capEvidence(pooled, 1000)
-
- assert.InDelta(t, 0.9, pooled[ascache.LRU].hitRate(), 1e-9, "the rate must survive the cap exactly")
- assert.InDelta(t, 1000.0, pooled[ascache.LRU].total(), 1e-9)
-
- assert.InDelta(t, 100.0, pooled[ascache.TinyLFU].total(), 1e-9,
- "an arm already under the cap must not be touched")
-}
-
-func TestCapEvidence_NegativeCapDisablesIt(t *testing.T) {
- pooled := map[ascache.PolicyType]weighted{
- ascache.LRU: {Hits: 1e9, Misses: 1e9},
- }
-
- capEvidence(pooled, -1)
-
- assert.InDelta(t, 2e9, pooled[ascache.LRU].total(), 1)
-}
-
-// TestCapEvidence_RestoresExploration is the reason the cap exists: pooling
-// multiplies evidence by the size of the fleet, and a posterior built on
-// millions of observations stops moving. Two arms a fifth of a point apart
-// should still both get drawn.
-func TestCapEvidence_RestoresExploration(t *testing.T) {
- fleetScale := func(cap float64) int {
- pooled := map[ascache.PolicyType]weighted{
- ascache.LRU: {Hits: 5_000_000, Misses: 5_000_000},
- ascache.TinyLFU: {Hits: 5_020_000, Misses: 4_980_000},
- }
- capEvidence(pooled, cap)
-
- rng := rand.New(rand.NewPCG(1, 2))
- lruWins := 0
- for range 200 {
- if draw(rng, pooled) == ascache.LRU {
- lruWins++
- }
- }
-
- return lruWins
- }
-
- assert.Zero(t, fleetScale(-1),
- "uncapped, ten million observations make the marginally worse arm unreachable")
- assert.Greater(t, fleetScale(DefaultMaxEvidence), 10,
- "capped, the marginally worse arm is still explored")
-}
-
-func TestDraw_PrefersTheBetterArm(t *testing.T) {
- pooled := map[ascache.PolicyType]weighted{
- ascache.LRU: {Hits: 100, Misses: 900},
- ascache.TinyLFU: {Hits: 900, Misses: 100},
- }
-
- rng := rand.New(rand.NewPCG(42, 42))
- wins := 0
- for range 500 {
- if draw(rng, pooled) == ascache.TinyLFU {
- wins++
- }
- }
-
- assert.Greater(t, wins, 495, "a nine-to-one better arm should win nearly always")
-}
-
-func TestDraw_EmptyEvidenceIsUndefined(t *testing.T) {
- rng := rand.New(rand.NewPCG(1, 1))
- assert.Equal(t, ascache.Undefined, draw(rng, nil))
-}
-
-func TestDraw_DoesNotDependOnMapIterationOrder(t *testing.T) {
- // Two arms with identical evidence must produce the same sequence of draws
- // from the same seed, however Go happens to order the map that run.
- build := func() map[ascache.PolicyType]weighted {
- return map[ascache.PolicyType]weighted{
- ascache.LRU: {Hits: 10, Misses: 10},
- ascache.LFU: {Hits: 10, Misses: 10},
- ascache.TwoQueue: {Hits: 10, Misses: 10},
- ascache.TinyLFU: {Hits: 10, Misses: 10},
- }
- }
-
- sequence := func() []ascache.PolicyType {
- rng := rand.New(rand.NewPCG(7, 7))
- pooled := build()
- out := make([]ascache.PolicyType, 0, 50)
- for range 50 {
- out = append(out, draw(rng, pooled))
- }
-
- return out
- }
-
- first := sequence()
- for range 20 {
- require.Equal(t, first, sequence())
- }
-}
-
-func TestWeighted_HitRateOfNothingIsZero(t *testing.T) {
- var empty weighted
- assert.Zero(t, empty.hitRate())
- assert.False(t, math.IsNaN(empty.hitRate()))
-}
diff --git a/bench/fleet.go b/bench/fleet.go
deleted file mode 100644
index eccfc33..0000000
--- a/bench/fleet.go
+++ /dev/null
@@ -1,354 +0,0 @@
-package bench
-
-import (
- "fmt"
- "strings"
- "sync"
- "time"
-
- ascache "github.com/sshaplygin/as-cache"
- "github.com/sshaplygin/as-cache/bandit"
-)
-
-// Shard splits a workload across n replicas by key, the way a consistent-hash
-// router in front of a fleet would.
-//
-// Sharding by key rather than round-robin is what makes the simulation mean
-// anything: a replica behind a hash router sees a stable slice of the keyspace
-// and can build a working set in it. Round-robin would give every replica the
-// full keyspace at 1/n the density, which is a different workload with
-// different best policies, and would flatter pooling by making every replica's
-// traffic identical.
-func Shard(w Workload, n int) []Workload {
- if n <= 1 {
- return []Workload{w}
- }
-
- shards := make([]Workload, n)
- for i := range shards {
- shards[i] = Workload{
- Name: fmt.Sprintf("%s/shard-%d", w.Name, i),
- Description: w.Description,
- }
- }
-
- for _, k := range w.Keys {
- // n is a positive int, so the remainder is below it and the conversion
- // back is exact.
- shard := int(fnv(k) % uint64(n)) //nolint:gosec // bounded by n on the line above
- shards[shard].Keys = append(shards[shard].Keys, k)
- }
-
- return shards
-}
-
-// Split divides a workload into n consecutive slices, so every replica sees
-// the whole keyspace but only a fraction of the requests.
-//
-// This is the homogeneous case: every replica's traffic has the same shape, so
-// the best policy is the same everywhere and pooling has nothing to reconcile.
-// It is the case pooling should help most, which makes it the fair test of
-// whether it helps at all.
-func Split(w Workload, n int) []Workload {
- if n <= 1 {
- return []Workload{w}
- }
-
- shards := make([]Workload, n)
- for i := range shards {
- shards[i] = Workload{
- Name: fmt.Sprintf("%s/slice-%d", w.Name, i),
- Description: w.Description,
- }
- }
-
- for i, k := range w.Keys {
- shards[i%n].Keys = append(shards[i%n].Keys, k)
- }
-
- return shards
-}
-
-// fnv hashes a key for sharding. Any stable hash does; this one avoids
-// pulling in a dependency and avoids Go's map seed, which changes per process
-// and would make a run unreproducible.
-func fnv(s string) uint64 {
- const (
- offset = 14695981039346656037
- prime = 1099511628211
- )
-
- h := uint64(offset)
- for i := range len(s) {
- h ^= uint64(s[i])
- h *= prime
- }
-
- return h
-}
-
-// ReplayPaced replays a workload at a target request rate rather than as fast
-// as the machine allows.
-//
-// Every other measurement here runs flat out, which is right for comparing
-// policies: it maximises requests per unit of wall clock and keeps runs short.
-// It is wrong for the one question a distributed bandit exists to answer.
-// "Too little traffic to tell the arms apart" means few requests per epoch,
-// and an unpaced replay delivers thousands per epoch however small the
-// workload is - the run just finishes sooner. Pacing is the only way to
-// reproduce the regime the fleet is supposed to help with, and it costs
-// wall-clock time to do it.
-func ReplayPaced(name string, c Cache, w Workload, perSecond float64) Result {
- if perSecond <= 0 {
- return Replay(name, c, w)
- }
-
- result := Result{Policy: name}
- interval := time.Duration(float64(time.Second) / perSecond)
-
- start := time.Now()
- for i, key := range w.Keys {
- // Scheduled against the start rather than by sleeping a fixed interval
- // each time, so the accumulated cost of the cache operations does not
- // drift the rate downwards over a long run.
- if due := start.Add(time.Duration(i) * interval); time.Now().Before(due) {
- time.Sleep(time.Until(due))
- }
-
- if _, ok := c.Get(key); ok {
- result.Hits++
-
- continue
- }
- result.Misses++
- c.Add(key, i)
- }
- result.Duration = time.Since(start)
-
- return result
-}
-
-// FleetSetup describes one way of running a fleet, so the ways can be compared
-// on identical traffic.
-type FleetSetup struct {
- // Name identifies the setup in reports.
- Name string
- // Bandit builds the bandit for replica i. A setup that pools returns
- // bandits sharing one store; one that does not returns independent ones.
- Bandit func(replica int) (ascache.Bandit, func() error, error)
-}
-
-// LocalFleet gives every replica its own Thompson bandit and no shared state.
-// It is the control: what a fleet does today, each replica deciding alone on
-// the fraction of the traffic it happens to see.
-func LocalFleet(discount float64, seed uint64) FleetSetup {
- return FleetSetup{
- Name: "local",
- Bandit: func(replica int) (ascache.Bandit, func() error, error) {
- // replica indexes a slice, so it is non-negative and small.
- return bandit.NewThompson(discount, seed+uint64(replica)), noClose, nil //nolint:gosec // see above
- },
- }
-}
-
-// PooledFleet gives every replica a distributed bandit over one shared
-// in-memory store.
-func PooledFleet(name string, mode bandit.Mode, epoch time.Duration, seed uint64) FleetSetup {
- store := bandit.NewMemStore()
-
- return FleetSetup{
- Name: name,
- Bandit: func(replica int) (ascache.Bandit, func() error, error) {
- b, err := bandit.NewDistributed(bandit.Config{
- Store: store,
- Namespace: "bench",
- NodeID: fmt.Sprintf("r%d", replica),
- CoordinationEpoch: epoch,
- Mode: mode,
- Window: 8,
- // replica indexes a slice, so it is non-negative and small.
- Seed: seed + uint64(replica), //nolint:gosec // see above
- })
- if err != nil {
- return nil, noClose, err
- }
-
- return b, b.Close, nil
- },
- }
-}
-
-func noClose() error { return nil }
-
-// FleetResult is what a whole fleet served, plus how its replicas behaved.
-type FleetResult struct {
- Result
- // Replicas is how many caches served the workload.
- Replicas int
- // Policies counts how many distinct policies the fleet ended on. Under
- // leader election this should be one; a fleet that ends split has either
- // lost its store or is running shared-posterior selection.
- Policies int
- // Ending lists each replica's final active policy.
- Ending []string
-}
-
-// ReplayFleet runs one workload across a fleet of AdaptiveCaches and reports
-// what the fleet as a whole served.
-//
-// Replicas run concurrently, because the coordination this measures only
-// happens in wall-clock time: a serial replay would let one replica finish
-// before another had published anything, and the fleet would never actually
-// overlap.
-func ReplayFleet(
- setup FleetSetup,
- shards []Workload,
- size int,
- settings ascache.Settings,
-) (FleetResult, error) {
- return replayFleet(setup, shards, size, settings, 0)
-}
-
-// ReplayFleetPaced is ReplayFleet with each replica held to perSecond
-// requests, so the fleet actually runs in the thin-traffic regime it exists
-// for rather than merely being given a small workload.
-func ReplayFleetPaced(
- setup FleetSetup,
- shards []Workload,
- size int,
- settings ascache.Settings,
- perSecond float64,
-) (FleetResult, error) {
- return replayFleet(setup, shards, size, settings, perSecond)
-}
-
-func replayFleet(
- setup FleetSetup,
- shards []Workload,
- size int,
- settings ascache.Settings,
- perSecond float64,
-) (FleetResult, error) {
- type outcome struct {
- result Result
- ending string
- }
-
- outcomes := make([]outcome, len(shards))
- closers := make([]func() error, 0, len(shards)*2)
-
- caches := make([]*ascache.AdaptiveCache[string, int], len(shards))
- for i := range shards {
- arms, err := AdaptiveArms(size)
- if err != nil {
- return FleetResult{}, err
- }
-
- policyBandit, closeBandit, err := setup.Bandit(i)
- if err != nil {
- return FleetResult{}, err
- }
- closers = append(closers, closeBandit)
-
- perReplica := settings
- cache, err := ascache.NewAdaptiveCache(arms, policyBandit, &perReplica)
- if err != nil {
- return FleetResult{}, err
- }
- closers = append(closers, cache.Close)
- caches[i] = cache
- }
-
- defer func() {
- for _, close := range closers {
- _ = close()
- }
- }()
-
- start := time.Now()
-
- var wg sync.WaitGroup
- for i := range shards {
- wg.Add(1)
- go func() {
- defer wg.Done()
-
- outcomes[i].result = ReplayPaced(setup.Name, caches[i], shards[i], perSecond)
- outcomes[i].ending = caches[i].ActivePolicy().String()
- }()
- }
- wg.Wait()
-
- fleet := FleetResult{
- Result: Result{Policy: setup.Name, Duration: time.Since(start)},
- Replicas: len(shards),
- Ending: make([]string, 0, len(shards)),
- }
-
- distinct := make(map[string]struct{}, len(shards))
- for _, o := range outcomes {
- fleet.Hits += o.result.Hits
- fleet.Misses += o.result.Misses
- fleet.Ending = append(fleet.Ending, o.ending)
- distinct[o.ending] = struct{}{}
- }
- fleet.Policies = len(distinct)
-
- return fleet, nil
-}
-
-// ReplayFixedFleet runs the same shards against a single fixed policy per
-// replica, which is the baseline every fleet setup has to be compared with:
-// the hit rate a team would get by picking one policy and deploying it
-// everywhere.
-func ReplayFixedFleet(builder PolicyBuilder, shards []Workload, size int) (FleetResult, error) {
- fleet := FleetResult{
- Result: Result{Policy: builder.Name},
- Replicas: len(shards),
- Ending: make([]string, 0, len(shards)),
- }
-
- start := time.Now()
- for _, shard := range shards {
- policy, err := builder.Build(size)
- if err != nil {
- return FleetResult{}, fmt.Errorf("build %s: %w", builder.Name, err)
- }
-
- result := Replay(builder.Name, policyCache{policy}, shard)
- fleet.Hits += result.Hits
- fleet.Misses += result.Misses
- fleet.Ending = append(fleet.Ending, builder.Name)
- }
- fleet.Duration = time.Since(start)
- fleet.Policies = 1
-
- return fleet, nil
-}
-
-// policyCache adapts a bare Policy to the Cache the harness replays against.
-type policyCache struct {
- policy ascache.Policy[string, int]
-}
-
-func (c policyCache) Get(key string) (int, bool) { return c.policy.Get(key) }
-func (c policyCache) Add(key string, value int) bool { return c.policy.Add(key, value) }
-
-// FleetTable renders fleet results as a markdown table, best hit rate first.
-func FleetTable(results []FleetResult) string {
- sorted := make([]FleetResult, len(results))
- copy(sorted, results)
- for i := 1; i < len(sorted); i++ {
- for j := i; j > 0 && sorted[j].HitRate() > sorted[j-1].HitRate(); j-- {
- sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
- }
- }
-
- var b strings.Builder
- b.WriteString("| Setup | Hit rate | Policies in use at the end |\n| --- | --- | --- |\n")
- for _, r := range sorted {
- fmt.Fprintf(&b, "| %s | %.2f%% | %d |\n", r.Policy, r.HitRate()*100, r.Policies)
- }
-
- return b.String()
-}
diff --git a/bench/fleet_test.go b/bench/fleet_test.go
deleted file mode 100644
index 0ac7c75..0000000
--- a/bench/fleet_test.go
+++ /dev/null
@@ -1,368 +0,0 @@
-package bench_test
-
-import (
- "fmt"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- ascache "github.com/sshaplygin/as-cache"
- "github.com/sshaplygin/as-cache/bandit"
- "github.com/sshaplygin/as-cache/bench"
-)
-
-// fleetSettings is tuned the way the real-trace evidence says to tune it: an
-// epoch long enough that the fleet is not spending its time migrating between
-// policies, and warm migration so a switch does not throw the working set
-// away.
-func fleetSettings() ascache.Settings {
- return ascache.Settings{
- EpochDuration: 20 * time.Millisecond,
- EvictPartialCapacityFilling: true,
- MigrationStrategy: ascache.MigrationWarm,
- ShadowSampleRate: 1,
- }
-}
-
-// TestFleet_DoesPoolingBeatDecidingAlone is the question the distributed
-// bandit exists to answer, and it is set up so the answer can come back "no".
-//
-// A fleet is compared four ways on identical traffic: every replica running
-// each fixed policy, every replica deciding alone, and the fleet pooling its
-// evidence under each of the two coordination modes. The comparison that
-// matters is pooled against local, since both are adaptive and only one of
-// them can see the fleet.
-func TestFleet_DoesPoolingBeatDecidingAlone(t *testing.T) {
- if testing.Short() {
- t.Skip("evidence run: replays a fleet of caches, see `make evidence`")
- }
-
- const (
- replicas = 8
- size = 500
- )
-
- workloads := []struct {
- name string
- workload bench.Workload
- split func(bench.Workload, int) []bench.Workload
- }{
- {
- name: "zipf/homogeneous",
- workload: bench.Zipf(400_000, 20_000, 1.1, 11),
- split: bench.Split,
- },
- {
- name: "phase-shift/homogeneous",
- workload: bench.PhaseShift(4, 100_000, 20_000, 3_000, 12),
- split: bench.Split,
- },
- {
- name: "zipf/sharded",
- workload: bench.Zipf(400_000, 20_000, 1.1, 13),
- split: bench.Shard,
- },
- }
-
- for _, w := range workloads {
- t.Run(w.name, func(t *testing.T) {
- shards := w.split(w.workload, replicas)
-
- perReplica := 0
- for _, shard := range shards {
- perReplica += shard.Len()
- }
- perReplica /= len(shards)
-
- results := make([]bench.FleetResult, 0, 12)
-
- for _, builder := range bench.FixedPolicies() {
- fixed, err := bench.ReplayFixedFleet(builder, shards, size)
- require.NoError(t, err)
- results = append(results, fixed)
- }
-
- setups := []bench.FleetSetup{
- bench.LocalFleet(0.7, 21),
- bench.PooledFleet("pooled/leader", bandit.ModeLeader, 50*time.Millisecond, 31),
- bench.PooledFleet("pooled/shared", bandit.ModeSharedPosterior, 50*time.Millisecond, 41),
- }
-
- adaptive := make(map[string]bench.FleetResult, len(setups))
- for _, setup := range setups {
- result, err := bench.ReplayFleet(setup, shards, size, fleetSettings())
- require.NoError(t, err)
- results = append(results, result)
- adaptive[setup.Name] = result
- }
-
- t.Logf("\n%s: %d replicas, %d requests each, cache size %d\n%s",
- w.name, replicas, perReplica, size, bench.FleetTable(results))
-
- best, worst := bestAndWorstFixed(results)
- t.Logf("best fixed policy %s at %.2f%%, worst %s at %.2f%%",
- best.Policy, best.HitRate()*100, worst.Policy, worst.HitRate()*100)
-
- for name, result := range adaptive {
- t.Logf("%s: %.2f%% (%+.2f vs best fixed, %+.2f vs local), %d policies in use at the end",
- name,
- result.HitRate()*100,
- (result.HitRate()-best.HitRate())*100,
- (result.HitRate()-adaptive["local"].HitRate())*100,
- result.Policies)
- }
-
- // The claim this repository is willing to make everywhere else is
- // the one asserted here: adaptive selection bounds the cost of
- // guessing wrong. Beating the best fixed policy is not claimed,
- // and is not asserted.
- for name, result := range adaptive {
- assert.Greater(t, result.HitRate(), worst.HitRate(),
- "%s did worse than the worst policy it could have been given", name)
- }
-
- // The fleet ending on one policy is reported, not asserted.
- // Replicas finish their shards at different moments, so each one's
- // last-applied decision is sampled at a different instant and two
- // of them straddling a fleet-wide switch is a property of when the
- // replay stopped, not of whether coordination worked. The
- // single-policy invariant is asserted where it can be checked
- // deterministically, against a clock the test controls: see
- // TestDistributed_FleetRunsOnePolicyAtATime in the bandit module.
- })
- }
-}
-
-// TestFleet_ThinTrafficIsWherePoolingShouldPay isolates the case the
-// distributed bandit was built for: replicas each seeing too little traffic to
-// rank arms on their own.
-//
-// It reports rather than asserts an improvement. If pooling does not help even
-// here, that is the finding, and it belongs in the README rather than in a
-// failing test.
-func TestFleet_ThinTrafficIsWherePoolingShouldPay(t *testing.T) {
- if testing.Short() {
- t.Skip("evidence run: replays a fleet of caches, see `make evidence`")
- }
-
- const size = 300
-
- for _, replicas := range []int{2, 8, 32} {
- t.Run(fmt.Sprintf("%d-replicas", replicas), func(t *testing.T) {
- // The total traffic is fixed, so more replicas means each sees
- // less of it - which is exactly the axis pooling is supposed to
- // win on.
- shards := bench.Split(bench.Zipf(200_000, 20_000, 1.1, 17), replicas)
-
- local, err := bench.ReplayFleet(bench.LocalFleet(0.7, 5), shards, size, fleetSettings())
- require.NoError(t, err)
-
- pooled, err := bench.ReplayFleet(
- bench.PooledFleet("pooled/leader", bandit.ModeLeader, 50*time.Millisecond, 6),
- shards, size, fleetSettings())
- require.NoError(t, err)
-
- var best bench.FleetResult
- for _, builder := range bench.FixedPolicies() {
- fixed, err := bench.ReplayFixedFleet(builder, shards, size)
- require.NoError(t, err)
- if fixed.HitRate() > best.HitRate() {
- best = fixed
- }
- }
-
- t.Logf("%d replicas, %d requests each: local %.2f%%, pooled %.2f%% (%+.2f), best fixed %s %.2f%%",
- replicas, shards[0].Len(),
- local.HitRate()*100, pooled.HitRate()*100,
- (pooled.HitRate()-local.HitRate())*100,
- best.Policy, best.HitRate()*100)
- })
- }
-}
-
-// TestFleet_PacedThinTraffic is the case the distributed bandit was actually
-// built for, and the only test here that reproduces it.
-//
-// Every other fleet measurement replays flat out, which delivers thousands of
-// requests per epoch however small the workload is - so "thin traffic" never
-// happens, the run just finishes sooner. Here each replica is held to a rate
-// that puts a handful of requests in each cache epoch, which is the regime
-// where a replica genuinely cannot rank its own arms and pooling has something
-// to add.
-//
-// It costs wall-clock time to run, which is the point: the regime cannot be
-// simulated any faster than it happens.
-func TestFleet_PacedThinTraffic(t *testing.T) {
- if testing.Short() {
- t.Skip("evidence run: paced replay, takes tens of seconds")
- }
-
- const (
- replicas = 8
- size = 300
- perSecond = 400 // with a 20ms epoch, about 8 requests per epoch per replica
- requests = 4_000
- )
-
- shards := bench.Split(bench.Zipf(requests*replicas, 20_000, 1.1, 77), replicas)
-
- local, err := bench.ReplayFleetPaced(
- bench.LocalFleet(0.7, 81), shards, size, fleetSettings(), perSecond)
- require.NoError(t, err)
-
- pooled, err := bench.ReplayFleetPaced(
- bench.PooledFleet("pooled/leader", bandit.ModeLeader, 200*time.Millisecond, 91),
- shards, size, fleetSettings(), perSecond)
- require.NoError(t, err)
-
- var best bench.FleetResult
- for _, builder := range bench.FixedPolicies() {
- fixed, err := bench.ReplayFixedFleet(builder, shards, size)
- require.NoError(t, err)
- if fixed.HitRate() > best.HitRate() {
- best = fixed
- }
- }
-
- t.Logf("paced fleet: %d replicas at %d req/s, %d requests each, ~%d requests per cache epoch",
- replicas, perSecond, shards[0].Len(),
- int(float64(perSecond)*fleetSettings().EpochDuration.Seconds()))
- t.Logf(" local %.2f%% across %d policies", local.HitRate()*100, local.Policies)
- t.Logf(" pooled/leader %.2f%% across %d policies (%+.2f vs local)",
- pooled.HitRate()*100, pooled.Policies, (pooled.HitRate()-local.HitRate())*100)
- t.Logf(" best fixed %.2f%% (%s)", best.HitRate()*100, best.Policy)
-}
-
-// TestFleet_HeterogeneousShardsAreWherePoolingShouldHurt is the other side of
-// the argument, and the reason ModeSharedPosterior exists at all.
-//
-// When replicas serve traffic of different shapes, one fleet-wide policy is a
-// compromise, and a replica choosing for itself can do better than the fleet
-// choosing for it. This measures how much that costs.
-func TestFleet_HeterogeneousShardsAreWherePoolingShouldHurt(t *testing.T) {
- if testing.Short() {
- t.Skip("evidence run: replays a fleet of caches, see `make evidence`")
- }
-
- const (
- replicas = 6
- size = 400
- )
-
- // Half the replicas get a looping scan, where recency policies serve
- // nothing; the other half get Zipf, where they do well. No single policy
- // is right for both.
- shards := make([]bench.Workload, 0, replicas)
- loop := bench.Split(bench.Loop(120_000, 2_000), replicas/2)
- zipf := bench.Split(bench.Zipf(120_000, 20_000, 1.1, 23), replicas/2)
- shards = append(shards, loop...)
- shards = append(shards, zipf...)
-
- local, err := bench.ReplayFleet(bench.LocalFleet(0.7, 51), shards, size, fleetSettings())
- require.NoError(t, err)
-
- pooled, err := bench.ReplayFleet(
- bench.PooledFleet("pooled/leader", bandit.ModeLeader, 50*time.Millisecond, 61),
- shards, size, fleetSettings())
- require.NoError(t, err)
-
- shared, err := bench.ReplayFleet(
- bench.PooledFleet("pooled/shared", bandit.ModeSharedPosterior, 50*time.Millisecond, 71),
- shards, size, fleetSettings())
- require.NoError(t, err)
-
- t.Logf("mixed fleet (%d loop replicas, %d zipf replicas):", replicas/2, replicas/2)
- t.Logf(" local %.2f%% across %d policies", local.HitRate()*100, local.Policies)
- t.Logf(" pooled/leader %.2f%% across %d policies (%+.2f vs local)",
- pooled.HitRate()*100, pooled.Policies, (pooled.HitRate()-local.HitRate())*100)
- t.Logf(" pooled/shared %.2f%% across %d policies (%+.2f vs local)",
- shared.HitRate()*100, shared.Policies, (shared.HitRate()-local.HitRate())*100)
-
- // The claim this test is named for. A replica choosing for itself can serve
- // its own shard; one fleet-wide policy is a compromise nobody asked for.
- assert.Less(t, pooled.HitRate(), local.HitRate(),
- "on a fleet whose replicas serve different workloads, pooling must lose to deciding alone")
- assert.Less(t, shared.HitRate(), local.HitRate())
-
- // Leadership concentrates the fleet, but "all replicas on one policy at the
- // final instant" is a claim about convergence speed, not about the
- // mechanism, and it stopped holding exactly when an eighth arm was added:
- // measured over eight runs, seven arms converged to one policy every time,
- // and eight arms end on two roughly one run in five. The leader changes its
- // mind more often with more arms to explore, and a replay is only so many
- // coordination rounds long. Asserting the strict form again would be
- // asserting that this repository never adds another policy.
- assert.LessOrEqual(t, pooled.Policies, 2,
- "leader election must still concentrate the fleet, whatever the arms cost in convergence")
-}
-
-// TestFleet_CoordinationEpochIsTheSettingThatMatters checks whether the gap
-// between pooling and deciding alone is a consequence of how often the fleet
-// gets to change its mind.
-//
-// The single-node evidence found epoch duration to be the setting that decides
-// everything; the fleet has a second, slower clock, and a replay is only so
-// many coordination rounds long. If the gap closes as the coordination epoch
-// shortens, pooling is losing to a lack of decision points. If it does not,
-// pooling is losing for a structural reason and no tuning will fix it.
-func TestFleet_CoordinationEpochIsTheSettingThatMatters(t *testing.T) {
- if testing.Short() {
- t.Skip("evidence run: replays a fleet of caches, see `make evidence`")
- }
-
- const (
- replicas = 8
- size = 500
- )
-
- shards := bench.Split(bench.Zipf(400_000, 20_000, 1.1, 11), replicas)
-
- local, err := bench.ReplayFleet(bench.LocalFleet(0.7, 21), shards, size, fleetSettings())
- require.NoError(t, err)
- t.Logf("local (no coordination): %.2f%%", local.HitRate()*100)
-
- for _, epoch := range []time.Duration{
- 10 * time.Millisecond,
- 25 * time.Millisecond,
- 50 * time.Millisecond,
- 200 * time.Millisecond,
- } {
- pooled, err := bench.ReplayFleet(
- bench.PooledFleet("pooled", bandit.ModeLeader, epoch, 31),
- shards, size, fleetSettings())
- require.NoError(t, err)
-
- t.Logf("coordination epoch %-6s: %.2f%% (%+.2f vs local), %d policies at the end",
- epoch, pooled.HitRate()*100,
- (pooled.HitRate()-local.HitRate())*100, pooled.Policies)
- }
-}
-
-func bestAndWorstFixed(results []bench.FleetResult) (best, worst bench.FleetResult) {
- fixedNames := make(map[string]struct{}, len(bench.FixedPolicies()))
- for _, builder := range bench.FixedPolicies() {
- fixedNames[builder.Name] = struct{}{}
- }
-
- first := true
- for _, result := range results {
- if _, ok := fixedNames[result.Policy]; !ok {
- continue
- }
- if first {
- best, worst, first = result, result, false
-
- continue
- }
- if result.HitRate() > best.HitRate() {
- best = result
- }
- if result.HitRate() < worst.HitRate() {
- worst = result
- }
- }
-
- return best, worst
-}
diff --git a/docker-compose.yml b/docker-compose.yml
deleted file mode 100644
index 02e3357..0000000
--- a/docker-compose.yml
+++ /dev/null
@@ -1,45 +0,0 @@
-# Local servers for the bandit/redis test suite.
-#
-# The suite runs against miniredis by default, which is a fake. The adapter
-# leans on three things a fake can be too permissive about - TIME called inside
-# a Lua script, SET with NX and PX, and HINCRBY on a key the script names
-# itself rather than declaring in KEYS - so the same tests run against a real
-# server here.
-#
-# Both engines are here on purpose. The adapter is documented as needing Redis
-# 7.0 or Valkey 7.2 and above, because deriving the bucket from the server's
-# clock inside a script requires effects replication. Claiming support for two
-# engines and testing one is how that claim quietly stops being true.
-#
-# make redis-up start both
-# make redis-test start both, run the suite against each, stop them
-# make redis-down stop both and remove their data
-#
-# Ports are deliberately off the defaults so this cannot collide with a Redis
-# you are already running for something else.
-
-services:
- valkey:
- image: valkey/valkey:8-alpine
- container_name: as-cache-valkey
- ports:
- - "63799:6379"
- # No persistence: every run starts empty, and nothing here is worth keeping.
- command: ["valkey-server", "--save", "", "--appendonly", "no"]
- healthcheck:
- test: ["CMD", "valkey-cli", "ping"]
- interval: 1s
- timeout: 3s
- retries: 20
-
- redis:
- image: redis:7-alpine
- container_name: as-cache-redis
- ports:
- - "63798:6379"
- command: ["redis-server", "--save", "", "--appendonly", "no"]
- healthcheck:
- test: ["CMD", "redis-cli", "ping"]
- interval: 1s
- timeout: 3s
- retries: 20
diff --git a/docs/design.md b/docs/design.md
index 4aa61d2..88779e8 100644
--- a/docs/design.md
+++ b/docs/design.md
@@ -32,9 +32,9 @@ Do not use it when:
- You cannot give it enough traffic per epoch to measure anything. Arms within
noise of each other reorder run to run, so a cache seeing a handful of
requests per epoch picks essentially at random. `Advice()` reports `Epochs`
- so you can tell whether it has seen enough. If the cause is that your traffic
- is spread across replicas rather than genuinely thin, see [running a
- fleet](fleet.md).
+ so you can tell whether it has seen enough. Traffic spread thin across many
+ replicas is the same problem seen from further away, and this library does
+ not solve it: each replica measures only what it serves.
- Your keyspace fits in the cache. Every policy scores the same when nothing is
ever evicted, and you are paying for shadows that can never tell you anything.
@@ -76,8 +76,7 @@ Then once per epoch:
2. The [bandit](#implementing-the-bandit-interface) receives that evidence and
names the arm for the next epoch. Beta posteriors updated with each arm's
hits and misses, drawn from by Thompson sampling, is the usual choice —
- `bandit.NewThompson` is one — but the interface is yours to implement, and
- `bandit.NewDistributed` pools the evidence across a fleet.
+ `bandit.NewThompson` is one — but the interface is yours to implement.
3. If the named arm is not the active one, [stability
gates](configuration.md#keeping-switches-stable) decide whether the
improvement is worth a switch. On a switch, data moves according to the
@@ -121,9 +120,8 @@ Both methods are called under the cache's write lock, so **an implementation
must not block**. Go's `RWMutex` queues new readers behind a waiting writer, so
a slow bandit stalls every `Get` in the process for its duration.
-Ready-made bandits live in the `bandit` module: `bandit.NewThompson` for a
-single process, [`bandit.NewDistributed`](fleet.md) for a fleet. Both examples
-use the first of those.
+Ready-made bandits live in the `bandit` module: `bandit.NewThompson`, and
+`bandit.NewGreedy` as a control. Both examples use the first.
### Plugging in a third-party bandit
diff --git a/docs/evidence.md b/docs/evidence.md
index 7b21ffe..ab7731c 100644
--- a/docs/evidence.md
+++ b/docs/evidence.md
@@ -407,81 +407,3 @@ Higher rates cost more and buy no better ranking here, so 0.05 is a reasonable
default. Raise it if your keyspace is small enough that 5% of it is only a
handful of keys -- `MinShadowCapacity` guards the degenerate end by raising the
effective rate rather than letting a miniature shrink into noise.
-
-## Does pooling across a fleet help?
-
-**Only in the regime it was built for, and it is worth checking you are in that
-regime before turning it on.** Figures are 8 replicas at cache capacity 300 to
-500 unless the row says otherwise -- the mixed-fleet row below is 6 replicas at
-400 -- and all come from `make evidence`. The mechanism is described in [running a
-fleet](fleet.md).
-
-The case it exists for is a replica that sees too little traffic per epoch to
-rank its own arms. Reproducing that requires holding each replica to a request
-rate — an unpaced replay delivers thousands of requests per epoch however small
-the workload is, it just finishes sooner. Paced to roughly 8 requests per cache
-epoch per replica:
-
-| Setup | Hit rate | Policies in use at the end |
-| --- | --- | --- |
-| pooled, leader-elected | 58.17% | 2 |
-| each replica deciding alone | 54.52% | 7 |
-
-**Pooling gains 3.6 points** over independent replicas. The last column is the
-mechanism: a replica with eight requests an epoch cannot tell nine arms apart,
-so the fleet scatters across seven different policies, several of them poor.
-Pooled, the fleet has 64 requests an epoch of evidence and holds two.
-
-Now the same comparison where replicas are *not* starved — the unpaced replays
-every other measurement here uses:
-
-| Workload | Pooled (leader) | Deciding alone | Best fixed |
-| --- | --- | --- | --- |
-| zipf, split evenly | 68.87% | 69.65% | S3-FIFO 72.73% |
-| zipf, sharded by key | 86.18% | 87.23% | SIEVE 88.22% |
-| phase-shift | 81.91% | 81.58% | 2Q 83.00% |
-| mixed fleet (half loop, half zipf) | 38.28% | 41.33% | — |
-
-**Pooling loses whenever the replicas could already measure for themselves**,
-by about a point on zipf and by 3.1 points on a fleet whose replicas serve
-different workloads. The mixed-fleet row is the clearest: a fleet-wide decision
-is a compromise, and when half your replicas want the policy the other half are
-worst served by, forcing agreement costs more than the disagreement did.
-Phase-shift is the exception that shows the size of the effect — pooling wins
-there by 0.33 points, which is inside the run-to-run noise of these replays.
-
-Fleet size matters more than the direction suggests. Holding the total traffic
-fixed and splitting it more ways:
-
-| Fleet | Requests per replica | Deciding alone | Pooled | Delta |
-| --- | --- | --- | --- | --- |
-| 2 replicas | 100,000 | 67.05% | 66.86% | -0.19 |
-| 8 replicas | 25,000 | 65.55% | 61.65% | -3.89 |
-| 32 replicas | 6,250 | 62.52% | 60.97% | -1.55 |
-
-None of these is starved enough for pooling to pay: even 6,250 requests is
-thousands per epoch. Thin traffic is a rate, not a total, which is why the
-paced replay above is the only one that reproduces the regime.
-
-How fast the fleet coordinates is the setting that decides the rest:
-
-```text
-local (no coordination): 69.47%
-coordination epoch 10ms: 69.72% (+0.25)
-coordination epoch 25ms: 69.94% (+0.47)
-coordination epoch 50ms: 69.01% (-0.46)
-coordination epoch 200ms: 66.95% (-2.52)
-```
-
-Coordinating often enough is worth a fraction of a point; coordinating rarely
-costs 2.5. Note that these replays coordinate through an in-process store, so
-coordination is free in a way it will not be for you: 10ms is where a real
-round trip stops being negligible, and the settings that look best here are the
-ones that cost most to run.
-
-**So the rule is:** pool when your replicas are individually starved of
-traffic, run the same workload shape as each other, and are numerous enough
-that the pooled evidence is meaningfully thicker. Otherwise let each replica
-decide alone — it is simpler, it needs no store, and on this evidence it is
-also better. `Advice()` in [observe-only mode](advisor-mode.md) will tell you
-which case you are in before you deploy anything.
diff --git a/docs/fleet.md b/docs/fleet.md
deleted file mode 100644
index 3c44d8e..0000000
--- a/docs/fleet.md
+++ /dev/null
@@ -1,128 +0,0 @@
-# Running a fleet
-
-One replica of a service sees one replica's traffic. If you run fifty of them
-behind a load balancer, each cache sees a fiftieth of the requests, and the
-"you cannot give it enough traffic per epoch to measure anything" caveat stops
-being about your traffic and starts being about how it was divided.
-
-The `bandit` module pools that evidence back together through Valkey or Redis.
-Each replica publishes its per-epoch counts; one replica per coordination epoch
-reads the fleet's aggregate, chooses, and publishes the choice for the others
-to apply.
-
-```go
-client := goredis.NewClient(&goredis.Options{Addr: "valkey:6379"})
-store, err := redisstore.New(redisstore.Options{Client: client})
-if err != nil {
- return err
-}
-defer store.Close()
-
-b, err := bandit.NewDistributed(bandit.Config{
- Store: store,
- Namespace: "sessions",
- CoordinationEpoch: time.Second,
-})
-if err != nil {
- return err
-}
-defer b.Close()
-
-cache, err := ascache.NewAdaptiveCache(arms, b, &ascache.Settings{
- EpochDuration: 50 * time.Millisecond,
-})
-```
-
-Read [the fleet evidence](evidence.md#does-pooling-across-a-fleet-help) before
-reaching for it. The answer is yes in one specific regime -- replicas
-individually too starved of traffic to rank their own arms -- and no everywhere
-else, and which case you are in is measurable in advance.
-
-**Two clocks, not one.** `EpochDuration` is how often each cache measures;
-`CoordinationEpoch` is how often the fleet decides. They are deliberately
-different scales. Cache epochs are tuned in tens of milliseconds, which is
-below both a round trip to the store and any clock agreement a fleet can be
-assumed to have. Measure on the fast clock, coordinate on the slow one; a
-second is a sensible starting point.
-
-**No replica's clock is ever consulted.** Buckets are derived from the store's
-clock inside a Lua script, so a fleet needs no clock synchronisation at all and
-a machine with a skewed clock cannot write its counts into a window nobody
-reads.
-
-**Nothing touches the network on the cache's path.** The cache calls its bandit
-while holding its write lock, so a round trip there would stall every `Get` in
-the process — and Go's `RWMutex` queues readers behind a waiting writer, so a
-store that hangs would hang the cache. `RecordEpoch` folds numbers into a
-buffer and `SelectPolicy` is an atomic load; all I/O happens on the bandit's
-own goroutine, once per coordination epoch.
-
-**When the store is unreachable**, each replica falls back to a local Thompson
-bandit fed by its own reports, which is exactly the behaviour of a cache that
-was never distributed. Nothing fails and nothing blocks. Counts measured during
-the outage are discarded rather than replayed on recovery — evidence that
-arrives in the wrong window is worse than no evidence. `Snapshot().Fallback` is
-the field to alert on: the cache looks entirely healthy either way.
-
-**Only integers cross the wire.** Per-policy hit and miss counts, a node id,
-and the policy written as its numeric `PolicyType` rather than its name —
-names come from stringer and are a presentation detail, so renaming one would
-silently split a fleet's counters in two. No cache keys and no cache values
-ever leave the process.
-Everything written carries a TTL, so a fleet that stops running leaves nothing
-behind.
-
-Requires Redis 7.0 or Valkey 7.2 and above. `docker-compose.yml` brings up both
-for local testing; `make redis-test` runs the store suite against each.
-
-## Which replicas pool with which
-
-Pooling is only meaningful between caches measuring the same thing. A hit rate
-from a 1000-entry cache says nothing about a 100-entry one, and averaging them
-describes neither — with nothing in the numbers to show it happened.
-
-So `Namespace` is not the whole key. A fingerprint of each cache's measurement
-regime — its arms, its capacity and its sample rate — is appended to it, and
-replicas that share a name without sharing a regime pool separately rather than
-pooling wrongly. `Snapshot().Namespace` and `Snapshot().Regime` are where to
-look when a fleet has unexpectedly split in two. Epoch duration deliberately is
-not part of it: a replica reporting twice as often contributes twice the
-counts at the same rate, and rates are what the comparison is made on.
-
-## The two modes
-
-`ModeLeader` (the default) elects one replica per coordination epoch to decide
-for everyone, so the fleet runs one policy at a time. `ModeSharedPosterior`
-has every replica draw its own selection from the pooled evidence, so no
-election happens and replicas may run different policies indefinitely.
-
-Leader election is the default for a reason that is not obvious. Whenever
-`ShadowSampleRate` is below 1, the active arm on a replica is measured at full
-capacity while every shadow runs on a miniature of it — the two roles are not
-identically measured, even though their rates are comparable by construction.
-Under leader election every replica has the *same* arm in the flattering role,
-so whatever that asymmetry is worth applies uniformly and cancels when the
-counts are summed. Under shared-posterior selection it does not: an arm active
-on most of the fleet is mostly measured in the flattering role, so it
-accumulates an advantage in
-proportion to how widely it is already deployed. `EvidenceShadowOnly` removes
-that feedback by discarding active-role counts, which is why it is available
-under shared-posterior selection and rejected under leader election — where
-the fleet-wide active policy is nobody's shadow and would have no evidence at
-all.
-
-## Pooling changes how much evidence a posterior sees
-
-A Beta posterior narrows with the square root of what it has seen, and a fleet
-supplies evidence in proportion to its size. A thousand replicas produce
-posteriors sharp enough that every Thompson draw returns the same arm — the
-bandit stops exploring precisely at the scale where missing a workload change
-is most expensive. `MaxEvidence` caps the effective sample size, keeping the
-measured rate and discarding the surplus certainty. The default puts an arm's
-posterior standard deviation at about a sixth of a percentage point.
-
-Decay works the same way for the same reason: a shared counter cannot be
-decayed in place, because every replica applying the multiplication would
-compound it once per replica and a fleet of fifty would forget fifty times
-faster than a fleet of one. The store holds plain per-bucket sums and the
-weighting happens on read, so the arithmetic is identical at any fleet size.
diff --git a/docs/getting-started.md b/docs/getting-started.md
index d0d038a..be917e6 100644
--- a/docs/getting-started.md
+++ b/docs/getting-started.md
@@ -84,4 +84,3 @@ full interface definitions are in [design](design.md#implementing-the-bandit-int
- [maypok86/otter](https://github.com/maypok86/otter) — the W-TinyLFU arm
- [scalalang2/golang-fifo](https://github.com/scalalang2/golang-fifo) — the S3-FIFO and SIEVE arms
- [dgraph-io/ristretto](https://github.com/dgraph-io/ristretto) — an early influence on the idea of measuring admission rather than assuming it
-- [redis/go-redis](https://github.com/redis/go-redis) and [Valkey](https://valkey.io/) — the client and store behind the distributed bandit
diff --git a/interfaces.go b/interfaces.go
index 483cd50..c119e51 100644
--- a/interfaces.go
+++ b/interfaces.go
@@ -50,8 +50,7 @@ type Policy[K comparable, V any] interface {
// This package ships no implementation, because the choice of strategy is the
// interesting part and depends on how quickly the traffic changes. Ready-made
// ones live in the companion github.com/sshaplygin/as-cache/bandit module:
-// a local Thompson sampler, and a distributed bandit that pools evidence
-// across a fleet through Valkey or Redis.
+// a local Thompson sampler and a greedy control.
//
// # Implementations must not block
//
diff --git a/scripts/release-check.sh b/scripts/release-check.sh
index b69c16b..c4b0f33 100755
--- a/scripts/release-check.sh
+++ b/scripts/release-check.sh
@@ -33,11 +33,11 @@ VERSION="${1:-vX.Y.Z}"
# Modules intended for publication. bench and examples/* are deliberately
# excluded: they are internal, nothing imports them, and their placeholder
# requires are harmless.
-PUBLISHABLE=(. lfu policies policies/arc policies/fifo policies/tinylfu metrics bandit bandit/redis benchclient)
+PUBLISHABLE=(. lfu policies policies/arc policies/fifo policies/tinylfu metrics bandit benchclient)
# Tagging order. A module cannot require a real version of a sibling until that
# sibling is tagged, so releases go bottom-up through the dependency graph.
-TAG_ORDER=(. lfu policies policies/arc policies/fifo policies/tinylfu metrics bandit bandit/redis benchclient)
+TAG_ORDER=(. lfu policies policies/arc policies/fifo policies/tinylfu metrics bandit benchclient)
fail=0
diff --git a/site/index.html b/site/index.html
index 1e6825f..64f08ee 100644
--- a/site/index.html
+++ b/site/index.html
@@ -403,11 +403,9 @@ Documentation
| Advisor mode |
ObserveOnly, Advice(), and the metrics module |
| Evidence |
- Every measured claim: policy tables, real traces, sampling fidelity, fleets |
+ Every measured claim: policy tables, real traces, sampling fidelity |
| Benchmarking |
Reproducible replays, benchclient, make evidence |
- | Running a fleet |
- Pooling evidence across replicas through Valkey or Redis |