diff --git a/CHANGELOG.md b/CHANGELOG.md index 27b2f92..06c1494 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ diskcache-community forked from DiskCache at upstream commit ebfa37c (3 March 20 ### Docs +- A new benchmarks page, measured on 2026-10-12, compares diskcache-community with diskcache 5.6.3, memcached, Redis, lmdb, sqlitedict, shelve and the Django cache backends, with four charts. The scripts that produce it are in `tests/benchmarks` (`driver.py` runs the stores, `report.py` writes the page and charts from the JSON results). The 2019 pages from upstream stay as the old benchmarks, labelled as such and linked from the new page, and the README header, features and documentation list link to the new page (PR #26). - Shared links show a project card: a social preview image in `.github/`, and Open Graph tags on the documentation through `sphinxext-opengraph` (PR #21). - A logo: the README opens with a centered header (logo, name, tagline, links, badges) in the style of the other community forks, and the documentation shows the logo and a favicon (PR #20). - The README is rewritten in Markdown around install, usage and the fork story; the comparison tables move to a new `comparisons` page in the documentation, which gets its own landing page, and the 2016 testimonials are dropped (PR #19). diff --git a/README.md b/README.md index 6e2ecbd..3f81161 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@

[Tutorial] - [API] +- [Benchmarks] - [Docs] - [Changelog]

@@ -108,9 +109,11 @@ cache = Cache('cache', disk=SafeDisk) - Read-only mode, automatic when the directory is not writable - Opt-in `SafeDisk` for caches whose directory other users can write to - Pure Python on top of the standard library `sqlite3`; no server, no compiler -- Tested on CPython 3.10 to 3.14 on Linux, macOS and Windows, with - [benchmarks](https://diskcache-community.readthedocs.io/en/latest/cache-benchmarks.html) - against the alternatives +- Tested on CPython 3.10 to 3.14 on Linux, macOS and Windows +- Gets in about 2.6 µs and sets in about 19 µs on an Apple M5 Pro in the + [benchmarks](https://diskcache-community.readthedocs.io/en/latest/benchmarks.html) + against memcached, Redis, lmdb, sqlitedict and shelve; sets are 9 to 14 % + faster than in diskcache 5.6.3, and twice as fast for 1 MB values ## Why this fork @@ -171,8 +174,12 @@ The documentation is at - [Tutorial](https://diskcache-community.readthedocs.io/en/latest/tutorial.html) - [API reference](https://diskcache-community.readthedocs.io/en/latest/api.html) -- [Cache benchmarks](https://diskcache-community.readthedocs.io/en/latest/cache-benchmarks.html) - and [DjangoCache benchmarks](https://diskcache-community.readthedocs.io/en/latest/djangocache-benchmarks.html) +- [Benchmarks](https://diskcache-community.readthedocs.io/en/latest/benchmarks.html) + against memcached, Redis, lmdb, sqlitedict and shelve, the Django cache + backends, and diskcache 5.6.3 (measured 2026-10-12) +- Old benchmarks from 2019, measured by upstream for version 4: + [Cache](https://diskcache-community.readthedocs.io/en/latest/cache-benchmarks.html) + and [DjangoCache](https://diskcache-community.readthedocs.io/en/latest/djangocache-benchmarks.html) - [Comparisons](https://diskcache-community.readthedocs.io/en/latest/comparisons.html) with other key-value stores and caching libraries - Case studies: diff --git a/docs/_static/benchmarks/bench-classic.png b/docs/_static/benchmarks/bench-classic.png new file mode 100644 index 0000000..bd4e3a2 Binary files /dev/null and b/docs/_static/benchmarks/bench-classic.png differ diff --git a/docs/_static/benchmarks/bench-django.png b/docs/_static/benchmarks/bench-django.png new file mode 100644 index 0000000..6935809 Binary files /dev/null and b/docs/_static/benchmarks/bench-django.png differ diff --git a/docs/_static/benchmarks/bench-sizes.png b/docs/_static/benchmarks/bench-sizes.png new file mode 100644 index 0000000..13f354a Binary files /dev/null and b/docs/_static/benchmarks/bench-sizes.png differ diff --git a/docs/_static/benchmarks/bench-throughput.png b/docs/_static/benchmarks/bench-throughput.png new file mode 100644 index 0000000..d7c11de Binary files /dev/null and b/docs/_static/benchmarks/bench-throughput.png differ diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst new file mode 100644 index 0000000..7f267c6 --- /dev/null +++ b/docs/benchmarks.rst @@ -0,0 +1,1082 @@ +DiskCache Benchmarks +==================== + +- **Reads:** 2.6 µs median in the classic mix, the fastest store there, 6 times + faster than memcached and 16 times faster than Redis on the same machine. +- **Large working sets:** lmdb reads faster over 100,000 keys (0.9 µs against + 3.2 µs at 95 % hits). +- **Writes:** 19.2 µs median, on par with memcached (16.0 µs to 18.5 µs) and 2 + times faster than Redis. +- **Against diskcache 5.6.3:** reads within 0.6 µs, writes 9 to 14 % faster, + 48 % faster for 1 MB values. +- **Concurrency:** FanoutCache(8) is the fastest setup across processes, + 291,478 operations per second from 8 processes; for 8 threads in one process + use a single Cache (188,028 against 19,715). +- **lmdb** writes take 5.98 ms because its defaults flush to disk on every + write. + +Environment +----------- + +- memcached: memcached 1.6.45 +- Redis: 8.10.2 +- diskcache-community Cache: diskcache-community 5.7.0 +- diskcache-community FanoutCache(8): diskcache-community 5.7.0 +- diskcache 5.6.3 Cache: diskcache 5.6.3 +- memcached, pymemcache: pymemcache 4.0.0 +- memcached, pylibmc: pylibmc 1.6.3, libmemcached 1.0.18 +- Redis, redis-py: redis-py 8.1.0 with hiredis +- lmdb: py-lmdb 3.0.0, liblmdb 0.9.36 +- sqlitedict: sqlitedict 2.1.0 +- shelve (dbm): shelve over dbm.sqlite3, Python 3.14.8 +- Django 6.0.9 + +diskcache-community and diskcache 5.6.3 +--------------------------------------- + +The same workloads, run against the fork and against the last upstream +release in separate environments on the same machine. For latencies a negative +change favours the fork; for operations per second a positive one does. + +.. list-table:: diskcache-community against diskcache 5.6.3 + :header-rows: 1 + :widths: 34 18 20 10 + + * - Measure + - diskcache 5.6.3 + - diskcache-community + - change + * - Classic mix, get p50 + - 2.6 µs + - 2.6 µs + - +0 % + * - Classic mix, set p50 + - 22.3 µs + - 19.2 µs + - -14 % + * - Classic mix, delete p50 + - 20.5 µs + - 18.4 µs + - -11 % + * - Classic mix, ops per second + - 185562 + - 197044 + - +6 % + * - 100-byte values, set p50 + - 24.0 µs + - 21.7 µs + - -10 % + * - 64 KB values, set p50 + - 250.9 µs + - 229.5 µs + - -9 % + * - 1 MB values, set p50 + - 676.4 µs + - 351.3 µs + - -48 % + * - 1 MB values, get p50 + - 55.0 µs + - 54.4 µs + - -1 % + * - 95 % hits, get p50 + - 2.8 µs + - 3.2 µs + - +12 % + * - 8 processes, set p50 + - 22.8 µs + - 20.3 µs + - -11 % + * - 8 threads, ops per second + - 174210 + - 188028 + - +8 % + * - Django, get p50 + - 3.6 µs + - 3.7 µs + - +2 % + * - Django, set p50 + - 24.1 µs + - 24.8 µs + - +3 % + + +The classic mix +--------------- + +The workload of the original 2016 page, kept so the two can be compared: keys +drawn from a space of 100, values of up to about 600 bytes, 90 % gets, 9 % +sets and 1 % deletes, 100,000 operations after a 1,000-operation warm-up, in +one process. The ~1 % miss rate comes from gets after deletes. + +.. image:: _static/benchmarks/bench-classic.png + +.. list-table:: Get latency, classic mix + :header-rows: 1 + :widths: 30 10 18 10 10 10 8 + + * - Store + - p50 + - p50 across runs + - p90 + - p99 + - max + - misses + * - diskcache-community Cache + - 2.6 µs + - 2.6 µs to 2.6 µs + - 3.4 µs + - 3.8 µs + - 36.6 µs + - 8714 + * - diskcache-community FanoutCache(8) + - 2.8 µs + - 2.8 µs to 2.8 µs + - 3.3 µs + - 4.0 µs + - 26.8 µs + - 8714 + * - diskcache 5.6.3 Cache + - 2.6 µs + - 2.5 µs to 2.6 µs + - 3.2 µs + - 3.8 µs + - 27.1 µs + - 8714 + * - memcached, pymemcache + - 17.6 µs + - 17.5 µs to 17.7 µs + - 20.8 µs + - 25.4 µs + - 130.4 µs + - 8714 + * - memcached, pylibmc + - 15.7 µs + - 15.6 µs to 15.7 µs + - 19.0 µs + - 24.0 µs + - 72.6 µs + - 8714 + * - Redis, redis-py + - 40.9 µs + - 40.8 µs to 41.3 µs + - 46.6 µs + - 52.8 µs + - 274.2 µs + - 8714 + * - lmdb + - 3.4 µs + - 3.4 µs to 3.5 µs + - 9.8 µs + - 23.6 µs + - 185.4 µs + - 8714 + * - sqlitedict + - 27.2 µs + - 27.1 µs to 27.3 µs + - 31.0 µs + - 38.7 µs + - 109.8 µs + - 8714 + * - shelve (dbm) + - 3.8 µs + - 3.8 µs to 3.8 µs + - 4.6 µs + - 5.9 µs + - 17.8 µs + - 8714 + +.. list-table:: Set latency, classic mix + :header-rows: 1 + :widths: 30 10 18 10 10 10 8 + + * - Store + - p50 + - p50 across runs + - p90 + - p99 + - max + - misses + * - diskcache-community Cache + - 19.2 µs + - 19.2 µs to 19.9 µs + - 25.4 µs + - 37.3 µs + - 623.2 µs + - 0 + * - diskcache-community FanoutCache(8) + - 20.5 µs + - 20.3 µs to 20.9 µs + - 26.5 µs + - 38.1 µs + - 1.02 ms + - 0 + * - diskcache 5.6.3 Cache + - 22.3 µs + - 22.1 µs to 22.4 µs + - 28.0 µs + - 39.4 µs + - 789.1 µs + - 0 + * - memcached, pymemcache + - 18.5 µs + - 18.3 µs to 18.6 µs + - 21.3 µs + - 25.9 µs + - 60.1 µs + - 0 + * - memcached, pylibmc + - 16.0 µs + - 16.0 µs to 16.1 µs + - 19.4 µs + - 24.3 µs + - 37.2 µs + - 0 + * - Redis, redis-py + - 43.6 µs + - 43.4 µs to 43.7 µs + - 48.5 µs + - 55.7 µs + - 133.5 µs + - 0 + * - lmdb + - 5.98 ms + - 5.97 ms to 5.99 ms + - 6.90 ms + - 8.02 ms + - 14.69 ms + - 0 + * - sqlitedict + - 147.0 µs + - 146.5 µs to 148.6 µs + - 170.5 µs + - 217.3 µs + - 1.22 ms + - 0 + * - shelve (dbm) + - 34.3 µs + - 33.8 µs to 35.0 µs + - 46.7 µs + - 62.5 µs + - 2.29 ms + - 0 + +.. list-table:: Delete latency, classic mix + :header-rows: 1 + :widths: 30 10 18 10 10 10 8 + + * - Store + - p50 + - p50 across runs + - p90 + - p99 + - max + - misses + * - diskcache-community Cache + - 18.4 µs + - 18.2 µs to 18.6 µs + - 21.4 µs + - 40.8 µs + - 569.9 µs + - 111 + * - diskcache-community FanoutCache(8) + - 19.3 µs + - 19.1 µs to 19.4 µs + - 27.2 µs + - 40.2 µs + - 897.4 µs + - 111 + * - diskcache 5.6.3 Cache + - 20.5 µs + - 20.5 µs to 20.6 µs + - 22.5 µs + - 37.1 µs + - 518.0 µs + - 111 + * - memcached, pymemcache + - 16.4 µs + - 16.4 µs to 16.7 µs + - 19.8 µs + - 24.5 µs + - 30.5 µs + - 111 + * - memcached, pylibmc + - 15.6 µs + - 15.5 µs to 15.8 µs + - 19.0 µs + - 24.2 µs + - 29.3 µs + - 111 + * - Redis, redis-py + - 41.4 µs + - 41.0 µs to 41.5 µs + - 46.6 µs + - 52.4 µs + - 74.3 µs + - 111 + * - lmdb + - 5.96 ms + - 5.95 ms to 5.97 ms + - 6.90 ms + - 7.96 ms + - 10.96 ms + - 111 + * - sqlitedict + - 164.3 µs + - 163.5 µs to 164.8 µs + - 188.2 µs + - 230.1 µs + - 466.0 µs + - 111 + * - shelve (dbm) + - 29.0 µs + - 28.8 µs to 29.5 µs + - 39.1 µs + - 54.4 µs + - 118.9 µs + - 111 + +.. list-table:: Throughput and footprint, classic mix + :header-rows: 1 + :widths: 30 14 12 20 + + * - Store + - ops per second + - wall time + - on disk after the run + * - diskcache-community Cache + - 197044 + - 0.50 s + - 4 MB + * - diskcache-community FanoutCache(8) + - 182771 + - 0.54 s + - 32 MB + * - diskcache 5.6.3 Cache + - 185562 + - 0.53 s + - 4 MB + * - memcached, pymemcache + - 53497 + - 1.85 s + - server + * - memcached, pylibmc + - 58268 + - 1.70 s + - server + * - Redis, redis-py + - 23519 + - 4.21 s + - server + * - lmdb + - 1592 + - 62.17 s + - 192 KB + * - sqlitedict + - 24246 + - 4.08 s + - 64 KB + * - shelve (dbm) + - 131608 + - 0.75 s + - 4 MB + + +Value sizes +----------- + +Sets followed by gets at four value sizes. The 64 KB and 1 MB tiers cross +DiskCache's default 32 KB threshold above which values live in files next to +the database; memcached was started with its item size limit raised to 4 MB +for the 1 MB tier. + +.. image:: _static/benchmarks/bench-sizes.png + +.. list-table:: 100-byte values, 20,000 sets then 20,000 gets over 1,000 keys + :header-rows: 1 + :widths: 30 12 12 12 12 + + * - Store + - set p50 + - set p99 + - get p50 + - get p99 + * - diskcache-community Cache + - 21.7 µs + - 37.0 µs + - 2.5 µs + - 3.3 µs + * - diskcache-community FanoutCache(8) + - 19.2 µs + - 36.4 µs + - 2.6 µs + - 3.4 µs + * - diskcache 5.6.3 Cache + - 24.0 µs + - 40.9 µs + - 2.4 µs + - 3.0 µs + * - memcached, pymemcache + - 16.7 µs + - 24.0 µs + - 13.8 µs + - 23.3 µs + * - memcached, pylibmc + - 15.7 µs + - 22.0 µs + - 15.5 µs + - 21.8 µs + * - Redis, redis-py + - 43.2 µs + - 52.7 µs + - 41.0 µs + - 50.5 µs + * - lmdb + - 6.01 ms + - 8.04 ms + - 1.5 µs + - 3.2 µs + * - sqlitedict + - 149.5 µs + - 232.5 µs + - 26.6 µs + - 36.6 µs + * - shelve (dbm) + - 33.5 µs + - 56.4 µs + - 3.7 µs + - 5.1 µs + +.. list-table:: 8 KB values, 20,000 sets then 20,000 gets over 1,000 keys + :header-rows: 1 + :widths: 30 12 12 12 12 + + * - Store + - set p50 + - set p99 + - get p50 + - get p99 + * - diskcache-community Cache + - 23.2 µs + - 44.0 µs + - 3.2 µs + - 4.5 µs + * - diskcache-community FanoutCache(8) + - 20.1 µs + - 41.2 µs + - 3.2 µs + - 4.7 µs + * - diskcache 5.6.3 Cache + - 25.8 µs + - 48.0 µs + - 2.9 µs + - 4.2 µs + * - memcached, pymemcache + - 18.0 µs + - 25.7 µs + - 19.2 µs + - 25.7 µs + * - memcached, pylibmc + - 16.8 µs + - 23.4 µs + - 17.2 µs + - 23.5 µs + * - Redis, redis-py + - 42.4 µs + - 54.0 µs + - 40.3 µs + - 51.7 µs + * - lmdb + - 6.02 ms + - 7.99 ms + - 1.8 µs + - 6.7 µs + * - sqlitedict + - 174.4 µs + - 292.0 µs + - 28.4 µs + - 39.2 µs + * - shelve (dbm) + - 51.1 µs + - 82.2 µs + - 6.2 µs + - 7.8 µs + +.. list-table:: 64 KB values, 4,000 sets then 4,000 gets over 500 keys + :header-rows: 1 + :widths: 30 12 12 12 12 + + * - Store + - set p50 + - set p99 + - get p50 + - get p99 + * - diskcache-community Cache + - 229.5 µs + - 399.1 µs + - 17.4 µs + - 31.6 µs + * - diskcache-community FanoutCache(8) + - 296.9 µs + - 429.2 µs + - 17.8 µs + - 31.1 µs + * - diskcache 5.6.3 Cache + - 250.9 µs + - 419.8 µs + - 17.2 µs + - 30.8 µs + * - memcached, pymemcache + - 27.7 µs + - 36.2 µs + - 33.0 µs + - 44.3 µs + * - memcached, pylibmc + - 27.6 µs + - 37.8 µs + - 29.2 µs + - 37.5 µs + * - Redis, redis-py + - 57.5 µs + - 67.6 µs + - 53.9 µs + - 65.5 µs + * - lmdb + - 6.02 ms + - 8.05 ms + - 4.2 µs + - 21.1 µs + * - sqlitedict + - 282.2 µs + - 449.3 µs + - 40.8 µs + - 51.9 µs + * - shelve (dbm) + - 110.9 µs + - 1.90 ms + - 21.4 µs + - 27.6 µs + +.. list-table:: 1 MB values, 400 sets then 400 gets over 50 keys + :header-rows: 1 + :widths: 30 12 12 12 12 + + * - Store + - set p50 + - set p99 + - get p50 + - get p99 + * - diskcache-community Cache + - 351.3 µs + - 483.9 µs + - 54.4 µs + - 79.7 µs + * - diskcache-community FanoutCache(8) + - 352.3 µs + - 484.6 µs + - 54.2 µs + - 81.3 µs + * - diskcache 5.6.3 Cache + - 676.4 µs + - 834.5 µs + - 55.0 µs + - 81.7 µs + * - memcached, pymemcache + - 122.2 µs + - 135.9 µs + - 220.2 µs + - 238.4 µs + * - memcached, pylibmc + - 301.8 µs + - 319.7 µs + - 154.8 µs + - 168.2 µs + * - Redis, redis-py + - 161.2 µs + - 180.4 µs + - 166.8 µs + - 192.1 µs + * - lmdb + - 6.99 ms + - 9.04 ms + - 34.7 µs + - 211.6 µs + * - sqlitedict + - 2.14 ms + - 2.99 ms + - 190.1 µs + - 221.6 µs + * - shelve (dbm) + - 1.24 ms + - 4.35 ms + - 264.8 µs + - 307.8 µs + + +Hit ratio +--------- + +Gets over a working set of 100,000 keys with 256-byte values, 100,000 +operations, at a 95 % and a 50 % hit ratio. Misses are requests for keys that +were never stored. + +.. list-table:: Get latency by hit ratio + :header-rows: 1 + :widths: 30 12 12 12 12 12 + + * - Store + - 95 % hits p50 + - 95 % hits p99 + - 50 % hits p50 + - 50 % hits p99 + - on disk + * - diskcache-community Cache + - 3.2 µs + - 4.3 µs + - 2.6 µs + - 4.0 µs + - 68 MB + * - diskcache-community FanoutCache(8) + - 3.1 µs + - 4.4 µs + - 2.8 µs + - 4.3 µs + - 68 MB + * - diskcache 5.6.3 Cache + - 2.8 µs + - 4.0 µs + - 2.5 µs + - 4.0 µs + - 69 MB + * - memcached, pymemcache + - 17.5 µs + - 24.5 µs + - 16.8 µs + - 24.1 µs + - server + * - memcached, pylibmc + - 15.8 µs + - 22.6 µs + - 15.6 µs + - 22.4 µs + - server + * - Redis, redis-py + - 40.0 µs + - 50.6 µs + - 40.7 µs + - 50.8 µs + - server + * - lmdb + - 0.9 µs + - 1.8 µs + - 0.7 µs + - 1.9 µs + - 49 MB + * - sqlitedict + - 27.9 µs + - 42.7 µs + - 27.4 µs + - 41.3 µs + - 30 MB + * - shelve (dbm) + - 5.0 µs + - 6.7 µs + - 3.5 µs + - 6.3 µs + - 33 MB + + +Concurrency +----------- + +The classic mix from 4 and 8 processes sharing one store, and from 8 threads +in one process, each worker running the full 100,000 operations. sqlitedict and +shelve are single-process stores by design and are left out of the process +workloads; shelve's dbm objects are bound to the thread that opened them, so it +is left out of the thread workload too. The memcached clients are not +thread-safe, so each thread gets its own connection, as pymemcache's +PooledClient and pylibmc's ThreadMappedPool hand out. Cells show get p50 / +set p50. + +.. image:: _static/benchmarks/bench-throughput.png + +.. list-table:: Latency under concurrency, get p50 / set p50 + :header-rows: 1 + :widths: 30 16 16 16 16 + + * - Store + - 1 process + - 4 processes + - 8 processes + - 8 threads + * - diskcache-community Cache + - 2.6 µs / 19.2 µs + - 2.6 µs / 20.3 µs + - 2.6 µs / 20.3 µs + - 2.5 µs / 20.1 µs + * - diskcache-community FanoutCache(8) + - 2.8 µs / 20.5 µs + - 3.3 µs / 28.8 µs + - 3.5 µs / 32.9 µs + - 216.6 µs / 1.26 ms + * - diskcache 5.6.3 Cache + - 2.6 µs / 22.3 µs + - 2.5 µs / 23.0 µs + - 2.5 µs / 22.8 µs + - 2.6 µs / 23.8 µs + * - memcached, pymemcache + - 17.6 µs / 18.5 µs + - 23.3 µs / 23.8 µs + - 38.3 µs / 38.6 µs + - 88.8 µs / 87.8 µs + * - memcached, pylibmc + - 15.7 µs / 16.0 µs + - 20.6 µs / 20.8 µs + - 33.3 µs / 33.6 µs + - 45.9 µs / 46.3 µs + * - Redis, redis-py + - 40.9 µs / 43.6 µs + - 51.8 µs / 55.9 µs + - 70.8 µs / 75.1 µs + - 223.1 µs / 224.3 µs + * - lmdb + - 3.4 µs / 5.98 ms + - 3.2 µs / 23.96 ms + - 3.3 µs / 48.03 ms + - 4.9 µs / 49.00 ms + * - sqlitedict + - 27.2 µs / 147.0 µs + - not run + - not run + - 404.8 µs / 525.9 µs + * - shelve (dbm) + - 3.8 µs / 34.3 µs + - not run + - not run + - not run + +.. list-table:: Operations per second, all workers together + :header-rows: 1 + :widths: 30 16 16 16 16 + + * - Store + - 1 process + - 4 processes + - 8 processes + - 8 threads + * - diskcache-community Cache + - 197044 + - 179441 + - 184652 + - 188028 + * - diskcache-community FanoutCache(8) + - 182771 + - 233447 + - 291478 + - 19715 + * - diskcache 5.6.3 Cache + - 185562 + - 173486 + - 176978 + - 174210 + * - memcached, pymemcache + - 53497 + - 152049 + - 181101 + - 77380 + * - memcached, pylibmc + - 58268 + - 169042 + - 211748 + - 146576 + * - Redis, redis-py + - 23519 + - 71765 + - 101483 + - 32954 + * - lmdb + - 1592 + - 1636 + - 1625 + - 1600 + * - sqlitedict + - 24246 + - not run + - not run + - 18344 + * - shelve (dbm) + - 131608 + - not run + - not run + - not run + + +Django backends +--------------- + +The classic mix through Django's cache API with each backend in turn, which +adds Django's key building and versioning on top of every operation. +``DatabaseCache`` uses SQLite; ``RedisCache`` is Django's own backend. + +.. image:: _static/benchmarks/bench-django.png + +.. list-table:: Get latency, Django backends + :header-rows: 1 + :widths: 30 10 18 10 10 10 8 + + * - Store + - p50 + - p50 across runs + - p90 + - p99 + - max + - misses + * - LocMemCache + - 1.7 µs + - 1.7 µs to 1.7 µs + - 1.8 µs + - 1.9 µs + - 27.1 µs + - 8714 + * - DjangoCache, diskcache-community + - 3.7 µs + - 3.7 µs to 3.7 µs + - 4.2 µs + - 5.1 µs + - 30.5 µs + - 8714 + * - DjangoCache, diskcache 5.6.3 + - 3.6 µs + - 3.5 µs to 3.6 µs + - 4.1 µs + - 5.0 µs + - 36.9 µs + - 8714 + * - PyMemcacheCache + - 26.3 µs + - 26.2 µs to 26.5 µs + - 30.2 µs + - 35.4 µs + - 97.2 µs + - 8714 + * - PyLibMCCache + - 20.7 µs + - 20.7 µs to 20.8 µs + - 23.9 µs + - 28.0 µs + - 70.9 µs + - 8714 + * - RedisCache + - 75.5 µs + - 75.1 µs to 76.0 µs + - 82.5 µs + - 89.1 µs + - 247.8 µs + - 8714 + * - FileBasedCache + - 14.9 µs + - 14.8 µs to 15.1 µs + - 19.6 µs + - 26.8 µs + - 61.0 µs + - 8714 + * - DatabaseCache (SQLite) + - 16.5 µs + - 16.4 µs to 16.6 µs + - 20.9 µs + - 27.3 µs + - 51.0 µs + - 8714 + +.. list-table:: Set latency, Django backends + :header-rows: 1 + :widths: 30 10 18 10 10 10 8 + + * - Store + - p50 + - p50 across runs + - p90 + - p99 + - max + - misses + * - LocMemCache + - 2.0 µs + - 2.0 µs to 2.0 µs + - 2.0 µs + - 2.2 µs + - 20.5 µs + - 0 + * - DjangoCache, diskcache-community + - 24.8 µs + - 24.7 µs to 24.8 µs + - 31.7 µs + - 48.1 µs + - 713.0 µs + - 0 + * - DjangoCache, diskcache 5.6.3 + - 24.1 µs + - 24.1 µs to 24.1 µs + - 31.2 µs + - 46.4 µs + - 1.47 ms + - 0 + * - PyMemcacheCache + - 27.0 µs + - 27.0 µs to 27.1 µs + - 30.9 µs + - 36.2 µs + - 75.5 µs + - 0 + * - PyLibMCCache + - 21.7 µs + - 21.7 µs to 21.7 µs + - 24.5 µs + - 28.8 µs + - 55.3 µs + - 0 + * - RedisCache + - 79.2 µs + - 78.9 µs to 80.0 µs + - 86.2 µs + - 94.0 µs + - 277.0 µs + - 0 + * - FileBasedCache + - 193.8 µs + - 193.7 µs to 196.0 µs + - 231.7 µs + - 309.6 µs + - 1.52 ms + - 0 + * - DatabaseCache (SQLite) + - 223.8 µs + - 222.7 µs to 227.8 µs + - 248.3 µs + - 308.2 µs + - 2.80 ms + - 0 + +.. list-table:: Delete latency, Django backends + :header-rows: 1 + :widths: 30 10 18 10 10 10 8 + + * - Store + - p50 + - p50 across runs + - p90 + - p99 + - max + - misses + * - LocMemCache + - 1.5 µs + - 1.5 µs to 1.6 µs + - 1.6 µs + - 2.0 µs + - 8.3 µs + - 111 + * - DjangoCache, diskcache-community + - 23.2 µs + - 23.0 µs to 23.2 µs + - 32.7 µs + - 46.5 µs + - 756.9 µs + - 111 + * - DjangoCache, diskcache 5.6.3 + - 22.3 µs + - 22.2 µs to 22.4 µs + - 31.9 µs + - 48.5 µs + - 842.8 µs + - 111 + * - PyMemcacheCache + - 25.8 µs + - 25.7 µs to 25.8 µs + - 29.5 µs + - 34.9 µs + - 45.4 µs + - 111 + * - PyLibMCCache + - 20.7 µs + - 20.7 µs to 20.8 µs + - 23.6 µs + - 27.5 µs + - 34.9 µs + - 111 + * - RedisCache + - 74.6 µs + - 74.2 µs to 75.3 µs + - 81.4 µs + - 89.7 µs + - 103.3 µs + - 111 + * - FileBasedCache + - 29.7 µs + - 29.2 µs to 29.8 µs + - 39.3 µs + - 47.9 µs + - 141.4 µs + - 111 + * - DatabaseCache (SQLite) + - 207.7 µs + - 207.7 µs to 208.2 µs + - 236.5 µs + - 289.8 µs + - 522.6 µs + - 111 + +.. list-table:: Throughput and footprint, Django backends + :header-rows: 1 + :widths: 30 14 12 20 + + * - Store + - ops per second + - wall time + - on disk after the run + * - LocMemCache + - 457085 + - 0.22 s + - in memory + * - DjangoCache, diskcache-community + - 147952 + - 0.67 s + - 32 MB + * - DjangoCache, diskcache 5.6.3 + - 150672 + - 0.66 s + - 32 MB + * - PyMemcacheCache + - 35724 + - 2.77 s + - server + * - PyLibMCCache + - 45486 + - 2.18 s + - server + * - RedisCache + - 12930 + - 7.66 s + - server + * - FileBasedCache + - 30159 + - 3.28 s + - 5 KB + * - DatabaseCache (SQLite) + - 25504 + - 3.88 s + - 80 KB + + +Running the benchmarks +---------------------- + +The scripts need memcached and redis-server on the path, a venv with +diskcache-community, pymemcache, pylibmc, redis, sqlitedict, lmdb and Django, +and a second venv with diskcache 5.6.3 and Django for the mainline column:: + + python tests/benchmarks/driver.py --fork-python /bin/python \ + --mainline-python /bin/python --out results/ + python tests/benchmarks/driver.py --django --fork-python /bin/python \ + --mainline-python /bin/python --out results/ + python tests/benchmarks/report.py --results results/ --docs docs/ + +The driver starts and stops the servers, runs one store at a time, and the +report script writes this page and its charts. diff --git a/docs/cache-benchmarks.rst b/docs/cache-benchmarks.rst index f55f5b9..d763ec2 100644 --- a/docs/cache-benchmarks.rst +++ b/docs/cache-benchmarks.rst @@ -1,5 +1,13 @@ -DiskCache Cache Benchmarks -========================== +Old Cache Benchmarks (2019) +=========================== + +.. note:: + + These are the old benchmarks, measured by upstream DiskCache for version 4 + in 2019 with the scripts ``tests/benchmark_core.py`` and + ``tests/benchmark_djangocache.py``. They are kept for reference. The current + measurements, with diskcache-community, diskcache 5.6.3 and today's + versions of the other stores, are on :doc:`benchmarks`. Accurately measuring performance is a difficult task. The benchmarks on this page are synthetic in the sense that they were designed to stress getting, diff --git a/docs/comparisons.rst b/docs/comparisons.rst index b858f0f..f2fbca2 100644 --- a/docs/comparisons.rst +++ b/docs/comparisons.rst @@ -69,7 +69,7 @@ Integrations? Django None None None None **Timings** -These are rough measurements. See :doc:`cache-benchmarks` for more +These are rough measurements. See :doc:`benchmarks` for more rigorous data. ================ ============= ========= ========= ============ ============ diff --git a/docs/development.rst b/docs/development.rst index 0329e8c..50c447c 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -62,7 +62,7 @@ Install development dependencies with `pip `_:: All packages for running tests and the lint checks will be installed. Additional packages like ``pylibmc`` and ``redis`` along with their server -counterparts are necessary for some benchmarks. +counterparts are necessary for the benchmarks. Testing ------- @@ -140,8 +140,15 @@ let it run overnight. Stress testing will stop at the first failure. Running Benchmarks ------------------ -Running and plotting benchmarks is a two step process. Each is a Python script -in the tests directory. Benchmark scripts are prefixed with ``benchmark_``. For +The :doc:`benchmarks` page is written by the scripts in ``tests/benchmarks``. +``driver.py`` starts memcached and Redis, runs each store in its own process +from a virtual environment with diskcache-community and one with diskcache +5.6.3, and stores JSON results; ``report.py`` turns the results into the page +and its charts. The page's last section lists the commands. + +The scripts below produced the :doc:`old benchmarks (2019) `. +Running and plotting them is a two step process. Each is a Python script in +the tests directory. Benchmark scripts are prefixed with ``benchmark_``. For example: :: diff --git a/docs/djangocache-benchmarks.rst b/docs/djangocache-benchmarks.rst index 6319e31..24352f0 100644 --- a/docs/djangocache-benchmarks.rst +++ b/docs/djangocache-benchmarks.rst @@ -1,5 +1,13 @@ -DiskCache DjangoCache Benchmarks -================================ +Old DjangoCache Benchmarks (2019) +================================= + +.. note:: + + These are the old benchmarks, measured by upstream DiskCache for version 4 + in 2019 with the scripts ``tests/benchmark_core.py`` and + ``tests/benchmark_djangocache.py``. They are kept for reference. The current + measurements, with diskcache-community, diskcache 5.6.3 and today's + versions of the other stores, are on :doc:`benchmarks`. :doc:`DiskCache ` provides a Django-compatible cache API in :class:`diskcache_community.DjangoCache`. A discussion of its options and diff --git a/docs/index.rst b/docs/index.rst index c2fc223..a6c0be2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,13 +21,16 @@ Install it with pip and import ``diskcache_community`` where you imported $ pip install diskcache-community -Start with the :doc:`tutorial` and keep the :doc:`api` at hand. +Start with the :doc:`tutorial` and keep the :doc:`api` at hand. The +:doc:`benchmarks` compare DiskCache with memcached, Redis, lmdb and other +stores, and the fork with diskcache 5.6.3. .. toctree:: :maxdepth: 1 :titlesonly: tutorial + benchmarks cache-benchmarks djangocache-benchmarks case-study-web-crawler diff --git a/tests/benchmarks/django_bench.py b/tests/benchmarks/django_bench.py new file mode 100644 index 0000000..94ad66c --- /dev/null +++ b/tests/benchmarks/django_bench.py @@ -0,0 +1,173 @@ +"""The classic mix through Django's cache API, one backend per process. + + python django_bench.py --backend diskcache --runs 5 --out result.json + +Backends: locmem, filebased, db (SQLite), pymemcache, pylibmc, redis +(Django's own backend), diskcache (diskcache-community if the interpreter holds +it, else upstream diskcache). +""" + +import argparse +import json +import os +import shutil +import sys +import tempfile +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import runner # noqa: E402 +from stores import diskcache_package # noqa: E402 + + +def configure(backend, workdir): + import django + from django.conf import settings + + caches = { + 'locmem': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'LOCATION': 'bench', + }, + 'filebased': { + 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + 'LOCATION': os.path.join(workdir, 'filebased'), + }, + 'db': { + 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', + 'LOCATION': 'bench_cache_table', + }, + 'pymemcache': { + 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', + 'LOCATION': '127.0.0.1:11211', + }, + 'pylibmc': { + 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', + 'LOCATION': '127.0.0.1:11211', + }, + 'redis': { + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + 'LOCATION': 'redis://127.0.0.1:6379/2', + }, + 'diskcache': { + 'BACKEND': diskcache_package().__name__ + '.DjangoCache', + 'LOCATION': os.path.join(workdir, 'diskcache'), + }, + } + settings.configure( + CACHES={'default': caches[backend]}, + DATABASES={ + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(workdir, 'db.sqlite3'), + } + }, + INSTALLED_APPS=[], + USE_TZ=True, + ) + django.setup() + if backend == 'db': + from django.core.management import call_command + + call_command('createcachetable', verbosity=0) + from django.core.cache import cache + + return cache + + +class DjangoStore: + """Adapter so the classic mix can drive Django's cache API.""" + + def __init__(self, cache): + self.cache = cache + + def get(self, key): + return self.cache.get(key.decode()) + + def set(self, key, value): + self.cache.set(key.decode(), value) + return True + + def delete(self, key): + return self.cache.delete(key.decode()) + + +def one_run(backend): + workdir = tempfile.mkdtemp(prefix='bench-django-') + cache = configure(backend, workdir) + cache.clear() + store = DjangoStore(cache) + runner.prepopulate(store) + t0 = time.perf_counter() + timings, misses = runner.run_classic(store, seed=0) + elapsed = time.perf_counter() - t0 + footprint = runner.stores.tree_size(workdir) + cache.clear() + try: + cache.close() + except Exception: # noqa: BLE001 + pass + shutil.rmtree(workdir, ignore_errors=True) + return runner.summarise(timings, misses, elapsed, footprint) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--backend', required=True) + parser.add_argument('--runs', type=int, default=5) + parser.add_argument('--warmup-runs', type=int, default=1) + parser.add_argument('--out', required=True) + args = parser.parse_args() + # Django can be configured once per process, so every run is a child process. + if os.environ.get('BENCH_DJANGO_CHILD'): + print(json.dumps(one_run(args.backend))) + return + import subprocess + + runs = [] + env = dict(os.environ, BENCH_DJANGO_CHILD='1') + for i in range(args.warmup_runs + args.runs): + proc = subprocess.run( + [ + sys.executable, + '-I', + __file__, + '--backend', + args.backend, + '--out', + args.out, + ], + env=env, + capture_output=True, + text=True, + ) + if proc.returncode != 0: + sys.stderr.write(proc.stderr) + raise SystemExit(proc.returncode) + if i >= args.warmup_runs: + runs.append(json.loads(proc.stdout.strip().splitlines()[-1])) + print('run', i, 'done', file=sys.stderr) + import django + + version = '' + if args.backend == 'diskcache': + diskcache = diskcache_package() + + version = '%s %s' % (diskcache.__title__, diskcache.__version__) + doc = { + 'store': 'django-' + args.backend, + 'store_name': 'Django %s %s' % (django.get_version(), args.backend), + 'store_version': version, + 'workload': 'django', + 'runs': runs, + 'environment': runner.environment(), + 'ops': runner.OPS, + 'warmup': runner.WARMUP, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, 'w') as writer: + json.dump(doc, writer, indent=1) + + +if __name__ == '__main__': + main() diff --git a/tests/benchmarks/driver.py b/tests/benchmarks/driver.py new file mode 100644 index 0000000..e15a40b --- /dev/null +++ b/tests/benchmarks/driver.py @@ -0,0 +1,282 @@ +"""Run the whole benchmark matrix. + + python driver.py --fork-python /bin/python \ + --mainline-python /bin/python \ + --out results/ [--stores ...] [--workloads ...] [--runs 5] + +The fork interpreter needs diskcache-community plus pymemcache, pylibmc, +redis, sqlitedict and lmdb; the mainline interpreter needs diskcache 5.6.3. +memcached and redis-server must be on PATH; they are started on their default +ports for the duration of the run, restarted if they die, and stopped +afterwards. Every item runs in its own process group with a timeout, so a +hung store cannot stall the matrix. Finished items are skipped on rerun. +""" + +import argparse +import os +import signal +import socket +import subprocess +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +ALL_STORES = [ + 'cache', + 'fanout', + 'cache563', + 'pymemcache', + 'pylibmc', + 'redis', + 'sqlitedict', + 'shelve', + 'lmdb', +] +ALL_WORKLOADS = [ + 'classic', + 'sizes', + 'hit95', + 'hit50', + 'procs4', + 'procs8', + 'threads8', +] +CONCURRENT = {'procs4', 'procs8', 'threads8'} +# single-process stores by design: one writer, no cross-process locking +SINGLE_PROCESS = {'sqlitedict', 'shelve'} +# shelve's dbm objects are bound to the thread that opened them +SINGLE_THREAD = {'shelve'} +SERVER_OF = { + 'pymemcache': 'memcached', + 'pylibmc': 'memcached', + 'redis': 'redis', +} +PORTS = {'memcached': 11211, 'redis': 6379} + + +def log(*parts): + print(time.strftime('%H:%M:%S'), *parts, flush=True) + + +def port_open(port): + with socket.socket() as sock: + sock.settimeout(0.5) + return sock.connect_ex(('127.0.0.1', port)) == 0 + + +def wait_port(port, timeout=15): + deadline = time.time() + timeout + while time.time() < deadline: + if port_open(port): + return True + time.sleep(0.1) + return False + + +class Servers: + COMMANDS = { + # memcached: default memory limit raised and item size raised to 4 MB for the 1 MB tier + 'memcached': ['memcached', '-p', '11211', '-m', '1024', '-I', '4m'], + # redis: no persistence, so the comparison is memory-only like memcached + 'redis': [ + 'redis-server', + '--port', + '6379', + '--save', + '', + '--appendonly', + 'no', + '--loglevel', + 'warning', + ], + } + + def __init__(self): + self.procs = {} + + def start(self, name): + self.procs[name] = subprocess.Popen( + self.COMMANDS[name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if not wait_port(PORTS[name]): + raise SystemExit('%s did not start' % name) + + def start_all(self): + for name in self.COMMANDS: + self.start(name) + + def ensure(self, name): + """Restart the server if it died; return True when a restart happened.""" + proc = self.procs.get(name) + if proc is not None and proc.poll() is None and port_open(PORTS[name]): + return False + log( + 'restart', + name, + '(exited with %s)' % (proc.poll() if proc else 'never started'), + ) + self.start(name) + return True + + def stop(self): + for proc in self.procs.values(): + proc.terminate() + for proc in self.procs.values(): + try: + proc.wait(10) + except subprocess.TimeoutExpired: + proc.kill() + self.procs = {} + + +def run_item(cmd, timeout): + """Run one benchmark process in its own process group; kill the group on timeout.""" + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + _out, err = proc.communicate(timeout=timeout) + return proc.returncode, err + except subprocess.TimeoutExpired: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + _out, err = proc.communicate() + return ( + 'timeout', + (err or '') + '\nkilled after %d s timeout\n' % timeout, + ) + + +def run_one(label, name, cmd, out, timeout): + t0 = time.time() + code, err = run_item(cmd, timeout) + status = 'ok' if code == 0 else 'FAILED (%s)' % code + log('%-9s %-20s %-18s %7.1fs' % (label, name, status, time.time() - t0)) + if code != 0: + with open(out + '.err', 'w') as writer: + writer.write(err) + print(err[-600:], flush=True) + + +DJANGO_BACKENDS = [ + 'locmem', + 'filebased', + 'db', + 'pymemcache', + 'pylibmc', + 'redis', + 'diskcache', +] + + +def run_django(args, servers): + jobs = [ + (backend, args.fork_python, 'django__' + backend) + for backend in DJANGO_BACKENDS + ] + jobs.append(('diskcache', args.mainline_python, 'django__diskcache563')) + for backend, python, name in jobs: + out = os.path.join(args.out, name + '.json') + if os.path.exists(out): + log('skip', name, '(exists)') + continue + if backend in SERVER_OF: + servers.ensure(SERVER_OF[backend]) + cmd = [ + python, + '-I', + os.path.join(HERE, 'django_bench.py'), + '--backend', + backend, + '--runs', + str(args.django_runs), + '--out', + out, + ] + run_one('django', name, cmd, out, args.item_timeout) + + +def run_matrix(args, servers): + for workload in args.workloads.split(','): + for store in args.stores.split(','): + if store in SINGLE_PROCESS and workload.startswith('procs'): + log('skip', workload, store, '(single-process store)') + continue + if store in SINGLE_THREAD and workload.startswith('threads'): + log('skip', workload, store, '(single-thread store)') + continue + python = ( + args.mainline_python + if store == 'cache563' + else args.fork_python + ) + out = os.path.join(args.out, '%s__%s.json' % (workload, store)) + if os.path.exists(out): + log('skip', workload, store, '(exists)') + continue + if store in SERVER_OF: + servers.ensure(SERVER_OF[store]) + runs = ( + args.concurrent_runs if workload in CONCURRENT else args.runs + ) + cmd = [ + python, + '-I', + os.path.join(HERE, 'runner.py'), + '--store', + store, + '--workload', + workload, + '--runs', + str(runs), + '--out', + out, + ] + run_one(workload, store, cmd, out, args.item_timeout) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--fork-python', required=True) + parser.add_argument('--mainline-python', required=True) + parser.add_argument('--out', required=True) + parser.add_argument('--stores', default=','.join(ALL_STORES)) + parser.add_argument('--workloads', default=','.join(ALL_WORKLOADS)) + parser.add_argument('--runs', type=int, default=5) + parser.add_argument('--concurrent-runs', type=int, default=3) + parser.add_argument( + '--item-timeout', + type=int, + default=7200, + help='seconds before one store/workload is killed', + ) + parser.add_argument( + '--django', + action='store_true', + help='run the Django backend matrix instead', + ) + parser.add_argument('--django-runs', type=int, default=3) + args = parser.parse_args() + os.makedirs(args.out, exist_ok=True) + servers = Servers() + servers.start_all() + log('servers up') + try: + if args.django: + run_django(args, servers) + else: + run_matrix(args, servers) + finally: + servers.stop() + log('servers down') + + +if __name__ == '__main__': + main() diff --git a/tests/benchmarks/report.py b/tests/benchmarks/report.py new file mode 100644 index 0000000..e15da79 --- /dev/null +++ b/tests/benchmarks/report.py @@ -0,0 +1,953 @@ +"""Turn the benchmark JSON results into reST tables, charts and the docs page. + +python report.py --results results/ --docs ../../docs +""" + +import argparse +import glob +import json +import os +import re +import statistics +import textwrap + +STORE_ORDER = [ + 'cache', + 'fanout', + 'cache563', + 'pymemcache', + 'pylibmc', + 'redis', + 'lmdb', + 'sqlitedict', + 'shelve', +] +LABELS = { + 'cache': 'diskcache-community Cache', + 'fanout': 'diskcache-community FanoutCache(8)', + 'cache563': 'diskcache 5.6.3 Cache', + 'pymemcache': 'memcached, pymemcache', + 'pylibmc': 'memcached, pylibmc', + 'redis': 'Redis, redis-py', + 'lmdb': 'lmdb', + 'sqlitedict': 'sqlitedict', + 'shelve': 'shelve (dbm)', +} +DJANGO_ORDER = [ + 'django-locmem', + 'django-diskcache', + 'django-diskcache563', + 'django-pymemcache', + 'django-pylibmc', + 'django-redis', + 'django-filebased', + 'django-db', +] +DJANGO_LABELS = { + 'django-locmem': 'LocMemCache', + 'django-diskcache': 'DjangoCache, diskcache-community', + 'django-diskcache563': 'DjangoCache, diskcache 5.6.3', + 'django-pymemcache': 'PyMemcacheCache', + 'django-pylibmc': 'PyLibMCCache', + 'django-redis': 'RedisCache', + 'django-filebased': 'FileBasedCache', + 'django-db': 'DatabaseCache (SQLite)', +} +COLORS = { + 'cache': '#0F766E', + 'fanout': '#2DD4BF', + 'cache563': '#94A3B8', + 'pymemcache': '#F59E0B', + 'pylibmc': '#D97706', + 'redis': '#DC2626', + 'lmdb': '#7C3AED', + 'sqlitedict': '#2563EB', + 'shelve': '#64748B', +} + + +def load(results_dir): + """Results keyed by (workload, store) from the file name, which is the + driver's name for the item: the two Django diskcache files share a store + name inside.""" + docs = {} + for path in sorted(glob.glob(os.path.join(results_dir, '*.json'))): + with open(path) as reader: + doc = json.load(reader) + workload, store = os.path.basename(path)[: -len('.json')].split( + '__', 1 + ) + if workload == 'django': + store = 'django-' + store + docs[(workload, store)] = doc + return docs + + +def med(values): + values = [v for v in values if v is not None] + return statistics.median(values) if values else None + + +def stat(doc, action, field, tier=None): + """Median across runs of one statistic.""" + values = [] + for run in doc['runs']: + block = run['tiers'][tier][action] if tier else run['ops'].get(action) + if block: + values.append(block[field]) + return med(values) + + +def spread(doc, action, field): + values = [ + run['ops'][action][field] + for run in doc['runs'] + if action in run['ops'] + ] + return (min(values), max(values)) if values else (None, None) + + +def fmt_ns(value): + if value is None: + return 'n/a' + us = value / 1000 + if us >= 1000: + return '%.2f ms' % (us / 1000) + return '%.1f µs' % us + + +def fmt_bytes(value): + if value is None: + return 'n/a' + if value == 0: + return 'server' + for unit in ('B', 'KB', 'MB', 'GB'): + if value < 1024: + return '%.0f %s' % (value, unit) + value /= 1024 + return '%.1f TB' % value + + +def list_table(title, header, rows, widths=None): + out = ['.. list-table:: %s' % title, ' :header-rows: 1'] + if widths: + out.append(' :widths: %s' % ' '.join(str(w) for w in widths)) + out.append('') + for i, row in enumerate([header] + rows): + for j, cell in enumerate(row): + prefix = ' * - ' if j == 0 else ' - ' + out.append(prefix + str(cell)) + out.append('') + return '\n'.join(out) + + +def op_table(docs, workload, action, title, stores=STORE_ORDER, labels=LABELS): + rows = [] + for store in stores: + doc = docs.get((workload, store)) + if not doc: + continue + p50 = stat(doc, action, 'p50') + if p50 is None: + continue + lo, hi = spread(doc, action, 'p50') + rows.append( + [ + labels[store], + fmt_ns(p50), + '%s to %s' % (fmt_ns(lo), fmt_ns(hi)), + fmt_ns(stat(doc, action, 'p90')), + fmt_ns(stat(doc, action, 'p99')), + fmt_ns(stat(doc, action, 'max')), + '%d' % stat(doc, action, 'misses'), + ] + ) + return list_table( + title, + ['Store', 'p50', 'p50 across runs', 'p90', 'p99', 'max', 'misses'], + rows, + [30, 10, 18, 10, 10, 10, 8], + ) + + +def summary_table(docs, workload, title, stores=STORE_ORDER, labels=LABELS): + rows = [] + for store in stores: + doc = docs.get((workload, store)) + if not doc: + continue + footprint = fmt_bytes(med([r['footprint_bytes'] for r in doc['runs']])) + if store == 'django-locmem': + footprint = 'in memory' + rows.append( + [ + labels[store], + '%.0f' % med([r['throughput_ops_s'] for r in doc['runs']]), + '%.2f s' % med([r['elapsed_s'] for r in doc['runs']]), + footprint, + ] + ) + return list_table( + title, + ['Store', 'ops per second', 'wall time', 'on disk after the run'], + rows, + [30, 14, 12, 20], + ) + + +def sizes_table(docs, tier, title): + rows = [] + for store in STORE_ORDER: + doc = docs.get(('sizes', store)) + if not doc or tier not in doc['runs'][0]['tiers']: + continue + rows.append( + [ + LABELS[store], + fmt_ns(stat(doc, 'set', 'p50', tier)), + fmt_ns(stat(doc, 'set', 'p99', tier)), + fmt_ns(stat(doc, 'get', 'p50', tier)), + fmt_ns(stat(doc, 'get', 'p99', tier)), + ] + ) + return list_table( + title, + ['Store', 'set p50', 'set p99', 'get p50', 'get p99'], + rows, + [30, 12, 12, 12, 12], + ) + + +def hit_table(docs, title): + rows = [] + for store in STORE_ORDER: + d95, d50 = docs.get(('hit95', store)), docs.get(('hit50', store)) + if not (d95 and d50): + continue + rows.append( + [ + LABELS[store], + fmt_ns(stat(d95, 'get', 'p50')), + fmt_ns(stat(d95, 'get', 'p99')), + fmt_ns(stat(d50, 'get', 'p50')), + fmt_ns(stat(d50, 'get', 'p99')), + fmt_bytes(med([r['footprint_bytes'] for r in d95['runs']])), + ] + ) + return list_table( + title, + [ + 'Store', + '95 % hits p50', + '95 % hits p99', + '50 % hits p50', + '50 % hits p99', + 'on disk', + ], + rows, + [30, 12, 12, 12, 12, 12], + ) + + +def concurrency_table(docs, title): + rows = [] + for store in STORE_ORDER: + cells = [LABELS[store]] + found = False + for workload in ('classic', 'procs4', 'procs8', 'threads8'): + doc = docs.get((workload, store)) + if doc: + found = True + cells.append( + '%s / %s' + % ( + fmt_ns(stat(doc, 'get', 'p50')), + fmt_ns(stat(doc, 'set', 'p50')), + ) + ) + else: + cells.append('not run') + if found: + rows.append(cells) + return list_table( + title, + ['Store', '1 process', '4 processes', '8 processes', '8 threads'], + rows, + [30, 16, 16, 16, 16], + ) + + +def throughput_table(docs, title): + rows = [] + for store in STORE_ORDER: + cells = [LABELS[store]] + found = False + for workload in ('classic', 'procs4', 'procs8', 'threads8'): + doc = docs.get((workload, store)) + if doc: + found = True + cells.append( + '%.0f' % med([r['throughput_ops_s'] for r in doc['runs']]) + ) + else: + cells.append('not run') + if found: + rows.append(cells) + return list_table( + title, + ['Store', '1 process', '4 processes', '8 processes', '8 threads'], + rows, + [30, 16, 16, 16, 16], + ) + + +# Rows of the fork-against-5.6.3 table: label, workload, the two store keys, +# what to compare, the action and the size tier. +FORK_ROWS = [ + ( + 'Classic mix, get p50', + 'classic', + 'cache', + 'cache563', + 'p50', + 'get', + None, + ), + ( + 'Classic mix, set p50', + 'classic', + 'cache', + 'cache563', + 'p50', + 'set', + None, + ), + ( + 'Classic mix, delete p50', + 'classic', + 'cache', + 'cache563', + 'p50', + 'delete', + None, + ), + ( + 'Classic mix, ops per second', + 'classic', + 'cache', + 'cache563', + 'throughput', + None, + None, + ), + ( + '100-byte values, set p50', + 'sizes', + 'cache', + 'cache563', + 'p50', + 'set', + '100 B', + ), + ( + '64 KB values, set p50', + 'sizes', + 'cache', + 'cache563', + 'p50', + 'set', + '64 KB', + ), + ( + '1 MB values, set p50', + 'sizes', + 'cache', + 'cache563', + 'p50', + 'set', + '1 MB', + ), + ( + '1 MB values, get p50', + 'sizes', + 'cache', + 'cache563', + 'p50', + 'get', + '1 MB', + ), + ('95 % hits, get p50', 'hit95', 'cache', 'cache563', 'p50', 'get', None), + ( + '8 processes, set p50', + 'procs8', + 'cache', + 'cache563', + 'p50', + 'set', + None, + ), + ( + '8 threads, ops per second', + 'threads8', + 'cache', + 'cache563', + 'throughput', + None, + None, + ), + ( + 'Django, get p50', + 'django', + 'django-diskcache', + 'django-diskcache563', + 'p50', + 'get', + None, + ), + ( + 'Django, set p50', + 'django', + 'django-diskcache', + 'django-diskcache563', + 'p50', + 'set', + None, + ), +] + + +def fork_table(docs, title): + rows = [] + for label, workload, new, old, kind, action, tier in FORK_ROWS: + new_doc, old_doc = docs.get((workload, new)), docs.get((workload, old)) + if not (new_doc and old_doc): + continue + if kind == 'throughput': + values = [ + med([r['throughput_ops_s'] for r in doc['runs']]) + for doc in (old_doc, new_doc) + ] + cells = ['%.0f' % value for value in values] + else: + values = [ + stat(doc, action, kind, tier) for doc in (old_doc, new_doc) + ] + cells = [fmt_ns(value) for value in values] + if None in values or not values[0]: + continue + change = (values[1] - values[0]) / values[0] * 100 + rows.append([label] + cells + ['%+.0f %%' % change]) + return list_table( + title, + ['Measure', 'diskcache 5.6.3', 'diskcache-community', 'change'], + rows, + [34, 18, 20, 10], + ) + + +# ---------------------------------------------------------------- charts + + +def chart_classic(docs, out_dir, workload='classic', name='classic'): + import matplotlib + + matplotlib.use('Agg') + import matplotlib.pyplot as plt + + actions = ['get', 'set', 'delete'] + fig, axes = plt.subplots(1, 3, figsize=(13, 4.2), sharey=False) + for ax, action in zip(axes, actions): + stores = [ + s + for s in STORE_ORDER + if (workload, s) in docs + and stat(docs[(workload, s)], action, 'p50') + ] + y = range(len(stores)) + for offset, field, alpha in ( + (-0.27, 'p50', 1.0), + (0.0, 'p90', 0.7), + (0.27, 'p99', 0.45), + ): + values = [ + stat(docs[(workload, s)], action, field) / 1000 for s in stores + ] + ax.barh( + [i + offset for i in y], + values, + height=0.26, + color=[COLORS[s] for s in stores], + alpha=alpha, + label=field, + ) + ax.set_yticks(list(y)) + ax.set_yticklabels( + [LABELS[s] for s in stores] + if ax is axes[0] + else [''] * len(stores), + fontsize=8, + ) + ax.invert_yaxis() + ax.set_xscale('log') + ax.set_xlabel('%s latency, µs (log), p50 dark to p99 light' % action) + ax.grid(axis='x', alpha=0.3) + fig.tight_layout() + path = os.path.join(out_dir, 'bench-%s.png' % name) + fig.savefig(path, dpi=130) + plt.close(fig) + return path + + +def chart_sizes(docs, out_dir): + import matplotlib + + matplotlib.use('Agg') + import matplotlib.pyplot as plt + + tiers = ['100 B', '8 KB', '64 KB', '1 MB'] + fig, axes = plt.subplots(1, 2, figsize=(11, 4.2)) + for ax, action in zip(axes, ('set', 'get')): + for store in STORE_ORDER: + doc = docs.get(('sizes', store)) + if not doc: + continue + values = [stat(doc, action, 'p50', t) for t in tiers] + if any(v is None for v in values): + continue + ax.plot( + tiers, + [v / 1000 for v in values], + marker='o', + color=COLORS[store], + label=LABELS[store], + ) + ax.set_yscale('log') + ax.set_ylabel('%s p50, µs (log)' % action) + ax.set_xlabel('value size') + ax.grid(alpha=0.3) + axes[1].legend(fontsize=7, loc='upper left') + fig.tight_layout() + path = os.path.join(out_dir, 'bench-sizes.png') + fig.savefig(path, dpi=130) + plt.close(fig) + return path + + +def chart_throughput(docs, out_dir): + import matplotlib + + matplotlib.use('Agg') + import matplotlib.pyplot as plt + + workloads = [ + ('classic', '1 process'), + ('procs4', '4 processes'), + ('procs8', '8 processes'), + ('threads8', '8 threads'), + ] + stores = [s for s in STORE_ORDER if ('classic', s) in docs] + fig, ax = plt.subplots(figsize=(11, 4.2)) + width = 0.8 / len(workloads) + for j, (workload, label) in enumerate(workloads): + values = [] + for s in stores: + doc = docs.get((workload, s)) + values.append( + med([r['throughput_ops_s'] for r in doc['runs']]) if doc else 0 + ) + ax.bar( + [i + (j - 1.5) * width for i in range(len(stores))], + values, + width=width, + label=label, + alpha=0.5 + 0.15 * j, + color=[COLORS[s] for s in stores], + ) + ax.set_xticks(range(len(stores))) + ax.set_xticklabels( + [LABELS[s] for s in stores], rotation=25, ha='right', fontsize=8 + ) + ax.set_ylabel('operations per second, all workers') + ax.set_yscale('log') + ax.grid(axis='y', alpha=0.3) + ax.legend(fontsize=8) + fig.tight_layout() + path = os.path.join(out_dir, 'bench-throughput.png') + fig.savefig(path, dpi=130) + plt.close(fig) + return path + + +def chart_django(docs, out_dir): + import matplotlib + + matplotlib.use('Agg') + import matplotlib.pyplot as plt + + backends = [b for b in DJANGO_ORDER if ('django', b) in docs] + fig, axes = plt.subplots(1, 3, figsize=(13, 4.2)) + for ax, action in zip(axes, ('get', 'set', 'delete')): + y = range(len(backends)) + for offset, field, alpha in ( + (-0.27, 'p50', 1.0), + (0.0, 'p90', 0.7), + (0.27, 'p99', 0.45), + ): + values = [ + stat(docs[('django', b)], action, field) / 1000 + for b in backends + ] + ax.barh( + [i + offset for i in y], + values, + height=0.26, + color='#0F766E', + alpha=alpha, + ) + ax.set_yticks(list(y)) + ax.set_yticklabels( + [DJANGO_LABELS[b] for b in backends] + if ax is axes[0] + else [''] * len(backends), + fontsize=8, + ) + ax.invert_yaxis() + ax.set_xscale('log') + ax.set_xlabel('%s latency, µs (log), p50 dark to p99 light' % action) + ax.grid(axis='x', alpha=0.3) + fig.tight_layout() + path = os.path.join(out_dir, 'bench-django.png') + fig.savefig(path, dpi=130) + plt.close(fig) + return path + + +# ------------------------------------------------------------------ page + + +def summary_section(docs): + """The conclusions at the top of the page, each computed from the results + and left out when its results are missing.""" + + def p50(workload, store, action, tier=None): + doc = docs.get((workload, store)) + return stat(doc, action, 'p50', tier) if doc else None + + def ops(workload, store): + doc = docs.get((workload, store)) + if not doc: + return None + return med([r['throughput_ops_s'] for r in doc['runs']]) + + lines = [] + get, put = p50('classic', 'cache', 'get'), p50('classic', 'cache', 'set') + memcached_get = [ + p50('classic', s, 'get') for s in ('pymemcache', 'pylibmc') + ] + memcached_set = [ + p50('classic', s, 'set') for s in ('pymemcache', 'pylibmc') + ] + redis_get = p50('classic', 'redis', 'get') + redis_set = p50('classic', 'redis', 'set') + lmdb_set = p50('classic', 'lmdb', 'set') + if get and None not in memcached_get and redis_get: + others = [p50('classic', s, 'get') for s in STORE_ORDER[3:]] + fastest = all(other is None or get <= other for other in others) + lines.append( + '- **Reads:** %s median in the classic mix%s, %.0f times faster' + ' than memcached and %.0f times faster than Redis on the same' + ' machine.' + % ( + fmt_ns(get), + ', the fastest store there' if fastest else '', + min(memcached_get) / get, + redis_get / get, + ) + ) + hit_cache, hit_lmdb = p50('hit95', 'cache', 'get'), p50( + 'hit95', 'lmdb', 'get' + ) + if hit_cache and hit_lmdb and hit_lmdb < hit_cache: + lines.append( + '- **Large working sets:** lmdb reads faster over 100,000 keys' + ' (%s against %s at 95 %% hits).' + % (fmt_ns(hit_lmdb), fmt_ns(hit_cache)) + ) + if put and None not in memcached_set and redis_set: + lines.append( + '- **Writes:** %s median, on par with memcached (%s to %s) and %.0f' + ' times faster than Redis.' + % ( + fmt_ns(put), + fmt_ns(min(memcached_set)), + fmt_ns(max(memcached_set)), + redis_set / put, + ) + ) + reads, writes = [], [] + for _label, workload, new, old, kind, action, tier in FORK_ROWS: + if kind != 'p50': + continue + values = ( + p50(workload, old, action, tier), + p50(workload, new, action, tier), + ) + if None in values: + continue + if action == 'get': + reads.append(abs(values[1] - values[0])) + elif workload != 'django': + writes.append(((values[0] - values[1]) / values[0] * 100, tier)) + if reads and writes: + large = [change for change, tier in writes if tier == '1 MB'] + small = [change for change, tier in writes if tier != '1 MB'] + text = ( + '- **Against diskcache 5.6.3:** reads within %s, writes %.0f to %.0f %%' + ' faster' % (fmt_ns(max(reads)), min(small), max(small)) + ) + if large: + text += ', %.0f %% faster for 1 MB values' % large[0] + lines.append(text + '.') + fanout8, cache_threads, fanout_threads = ( + ops('procs8', 'fanout'), + ops('threads8', 'cache'), + ops('threads8', 'fanout'), + ) + if fanout8 and cache_threads and fanout_threads: + lines.append( + '- **Concurrency:** FanoutCache(8) is the fastest setup across' + ' processes, {:,.0f} operations per second from 8 processes; for 8' + ' threads in one process use a single Cache ({:,.0f} against' + ' {:,.0f}).'.format(fanout8, cache_threads, fanout_threads) + ) + if lmdb_set: + lines.append( + '- **lmdb** writes take %s because its defaults flush to disk on' + ' every write.' % fmt_ns(lmdb_set) + ) + return '\n'.join( + textwrap.fill( + line.replace(' %', '\u00a0%'), 79, subsequent_indent=' ' + ) + for line in lines + ) + + +def environment_section(docs): + env = next(iter(docs.values()))['environment'] + versions = [] + for store in STORE_ORDER: + doc = docs.get(('classic', store)) + if doc: + versions.append( + '- %s: %s' + % (LABELS[store], doc['store_version'] or 'standard library') + ) + django = docs.get(('django', 'django-locmem')) + if django: + versions.append('- %s' % django['store_name'].replace(' locmem', '')) + redis = re.search(r'v=([\d.]+)', env['redis'] or '') + return '\n'.join( + [ + '- memcached: %s' % env['memcached'].splitlines()[0] + if env['memcached'] + else '- memcached: n/a', + '- Redis: %s' % (redis.group(1) if redis else 'n/a'), + ] + + versions + ) + + +PAGE = """DiskCache Benchmarks +==================== + +{summary} + +Environment +----------- + +{environment} + +diskcache-community and diskcache 5.6.3 +--------------------------------------- + +The same workloads, run against the fork and against the last upstream +release in separate environments on the same machine. For latencies a negative +change favours the fork; for operations per second a positive one does. + +{fork} + +The classic mix +--------------- + +The workload of the original 2016 page, kept so the two can be compared: keys +drawn from a space of 100, values of up to about 600 bytes, 90 % gets, 9 % +sets and 1 % deletes, 100,000 operations after a 1,000-operation warm-up, in +one process. The ~1 % miss rate comes from gets after deletes. + +.. image:: _static/benchmarks/bench-classic.png + +{classic_get} +{classic_set} +{classic_delete} +{classic_summary} + +Value sizes +----------- + +Sets followed by gets at four value sizes. The 64 KB and 1 MB tiers cross +DiskCache's default 32 KB threshold above which values live in files next to +the database; memcached was started with its item size limit raised to 4 MB +for the 1 MB tier. + +.. image:: _static/benchmarks/bench-sizes.png + +{sizes_100b} +{sizes_8kb} +{sizes_64kb} +{sizes_1mb} + +Hit ratio +--------- + +Gets over a working set of 100,000 keys with 256-byte values, 100,000 +operations, at a 95 % and a 50 % hit ratio. Misses are requests for keys that +were never stored. + +{hit} + +Concurrency +----------- + +The classic mix from 4 and 8 processes sharing one store, and from 8 threads +in one process, each worker running the full 100,000 operations. sqlitedict and +shelve are single-process stores by design and are left out of the process +workloads; shelve's dbm objects are bound to the thread that opened them, so it +is left out of the thread workload too. The memcached clients are not +thread-safe, so each thread gets its own connection, as pymemcache's +PooledClient and pylibmc's ThreadMappedPool hand out. Cells show get p50 / +set p50. + +.. image:: _static/benchmarks/bench-throughput.png + +{concurrency} +{throughput} + +Django backends +--------------- + +The classic mix through Django's cache API with each backend in turn, which +adds Django's key building and versioning on top of every operation. +``DatabaseCache`` uses SQLite; ``RedisCache`` is Django's own backend. + +.. image:: _static/benchmarks/bench-django.png + +{django_get} +{django_set} +{django_delete} +{django_summary} + +Running the benchmarks +---------------------- + +The scripts need memcached and redis-server on the path, a venv with +diskcache-community, pymemcache, pylibmc, redis, sqlitedict, lmdb and Django, +and a second venv with diskcache 5.6.3 and Django for the mainline column:: + + python tests/benchmarks/driver.py --fork-python /bin/python \\ + --mainline-python /bin/python --out results/ + python tests/benchmarks/driver.py --django --fork-python /bin/python \\ + --mainline-python /bin/python --out results/ + python tests/benchmarks/report.py --results results/ --docs docs/ + +The driver starts and stops the servers, runs one store at a time, and the +report script writes this page and its charts. +""" + + +def build_page(docs): + def dj(action, title): + return op_table( + docs, 'django', action, title, DJANGO_ORDER, DJANGO_LABELS + ) + + return PAGE.format( + summary=summary_section(docs), + environment=environment_section(docs), + fork=fork_table(docs, 'diskcache-community against diskcache 5.6.3'), + classic_get=op_table( + docs, 'classic', 'get', 'Get latency, classic mix' + ), + classic_set=op_table( + docs, 'classic', 'set', 'Set latency, classic mix' + ), + classic_delete=op_table( + docs, 'classic', 'delete', 'Delete latency, classic mix' + ), + classic_summary=summary_table( + docs, 'classic', 'Throughput and footprint, classic mix' + ), + sizes_100b=sizes_table( + docs, + '100 B', + '100-byte values, 20,000 sets then 20,000 gets over 1,000 keys', + ), + sizes_8kb=sizes_table( + docs, + '8 KB', + '8 KB values, 20,000 sets then 20,000 gets over 1,000 keys', + ), + sizes_64kb=sizes_table( + docs, + '64 KB', + '64 KB values, 4,000 sets then 4,000 gets over 500 keys', + ), + sizes_1mb=sizes_table( + docs, '1 MB', '1 MB values, 400 sets then 400 gets over 50 keys' + ), + hit=hit_table(docs, 'Get latency by hit ratio'), + concurrency=concurrency_table( + docs, 'Latency under concurrency, get p50 / set p50' + ), + throughput=throughput_table( + docs, 'Operations per second, all workers together' + ), + django_get=dj('get', 'Get latency, Django backends'), + django_set=dj('set', 'Set latency, Django backends'), + django_delete=dj('delete', 'Delete latency, Django backends'), + django_summary=summary_table( + docs, + 'django', + 'Throughput and footprint, Django backends', + DJANGO_ORDER, + DJANGO_LABELS, + ), + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--results', required=True) + parser.add_argument('--docs', required=True) + args = parser.parse_args() + docs = load(args.results) + charts_dir = os.path.join(args.docs, '_static', 'benchmarks') + os.makedirs(charts_dir, exist_ok=True) + chart_classic(docs, charts_dir) + chart_sizes(docs, charts_dir) + chart_throughput(docs, charts_dir) + if any(w == 'django' for w, _ in docs): + chart_django(docs, charts_dir) + page = build_page(docs) + with open(os.path.join(args.docs, 'benchmarks.rst'), 'w') as writer: + writer.write(page) + print( + 'wrote', + os.path.join(args.docs, 'benchmarks.rst'), + 'and charts in', + charts_dir, + ) + + +if __name__ == '__main__': + main() diff --git a/tests/benchmarks/run_overnight.sh b/tests/benchmarks/run_overnight.sh new file mode 100755 index 0000000..89e22fc --- /dev/null +++ b/tests/benchmarks/run_overnight.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Run the whole benchmark matrix unattended on a quiet machine. +# +# tests/benchmarks/run_overnight.sh +# +# Keeps the Mac from idle-sleeping while it runs (the lid must stay open or +# an external display attached), logs to /driver.log, and writes +# /DONE when finished. Safe to rerun: finished stores are skipped. +set -uo pipefail +FORK=${1:?fork venv python} +MAIN=${2:?mainline venv python} +OUT=${3:?results dir} +HERE="$(cd "$(dirname "$0")" && pwd)" +mkdir -p "$OUT" +rm -f "$OUT/DONE" +{ + echo "start $(date '+%Y-%m-%d %H:%M:%S')" + caffeinate -i -s python3 "$HERE/driver.py" --fork-python "$FORK" --mainline-python "$MAIN" --out "$OUT" --runs 5 --concurrent-runs 3 + caffeinate -i -s python3 "$HERE/driver.py" --django --fork-python "$FORK" --mainline-python "$MAIN" --out "$OUT" --django-runs 3 + echo "end $(date '+%Y-%m-%d %H:%M:%S')" +} >> "$OUT/driver.log" 2>&1 +touch "$OUT/DONE" diff --git a/tests/benchmarks/runner.py b/tests/benchmarks/runner.py new file mode 100644 index 0000000..b3b6ae4 --- /dev/null +++ b/tests/benchmarks/runner.py @@ -0,0 +1,373 @@ +"""Run one store through one workload and write a JSON result. + + python runner.py --store cache --workload classic --runs 5 --out result.json + +Workloads + classic the 2016 mix: keys from a space of 100, values up to ~600 B, + 90 % get, 9 % set, 1 % delete, 100k ops, 1k warm-up + sizes set then get at 100 B, 8 KB, 64 KB, 1 MB + hit95 gets over a 100k-key working set with 95 % hits (the set is + loaded in bulk, unmeasured) + hit50 the same with 50 % hits + procs4 the classic mix from 4 processes on one shared store + procs8 the classic mix from 8 processes on one shared store + threads8 the classic mix from 8 threads in one process (memcached + clients get one connection per thread, as their pools do) + +Every measured operation records its latency in nanoseconds; the result +holds the percentiles, the throughput, the miss rate and the disk footprint +of every run, plus the environment. +""" + +import argparse +import datetime +import json +import multiprocessing +import os +import platform +import random +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import threading +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import stores # noqa: E402 + +OPS = int(os.environ.get('BENCH_OPS', 100_000)) +WARMUP = min(1_000, OPS // 10) +RANGE = 100 +SIZES = [ + ('100 B', 100, 20_000, 1_000), + ('8 KB', 8 * 1024, 20_000, 1_000), + ('64 KB', 64 * 1024, 4_000, 500), + ('1 MB', 1024 * 1024, 400, 50), +] +HIT_KEYS = int(os.environ.get('BENCH_HIT_KEYS', 100_000)) + + +def percentiles(values): + if not values: + return None + values = sorted(values) + n = len(values) + + def pct(p): + return values[min(n - 1, max(0, int(n * p) - 1))] + + return { + 'count': n, + 'p50': pct(0.50), + 'p90': pct(0.90), + 'p99': pct(0.99), + 'max': values[-1], + 'total_ns': sum(values), + } + + +def classic_ops(seed, ops=OPS): + """The 2016 operation sequence, one deterministic stream per seed.""" + rnd = random.Random(seed) + for count in range(ops): + key = str(rnd.randrange(RANGE)).encode() + value = str(count).encode() * rnd.randrange(1, 100) + choice = rnd.random() + yield count, key, value, ( + 'get' if choice < 0.9 else 'set' if choice < 0.99 else 'delete' + ) + + +def run_classic(store, seed, ops=OPS, warmup=WARMUP): + timings = {'get': [], 'set': [], 'delete': []} + misses = {'get': 0, 'set': 0, 'delete': 0} + perf = time.perf_counter_ns + for count, key, value, action in classic_ops(seed, ops): + if action == 'get': + start = perf() + result = store.get(key) + end = perf() + miss = result is None + elif action == 'set': + start = perf() + result = store.set(key, value) + end = perf() + miss = result is False + else: + start = perf() + result = store.delete(key) + end = perf() + miss = result is False + if count >= warmup: + timings[action].append(end - start) + if miss: + misses[action] += 1 + return timings, misses + + +def prepopulate(store): + for key in range(RANGE): + store.set(str(key).encode(), str(key).encode()) + + +def workload_classic(cls, workdir): + store = cls(workdir) + prepopulate(store) + t0 = time.perf_counter() + timings, misses = run_classic(store, seed=0) + elapsed = time.perf_counter() - t0 + footprint = store.footprint() + store.close() + return summarise(timings, misses, elapsed, footprint) + + +def summarise(timings, misses, elapsed, footprint, extra=None): + out = {'ops': {}, 'elapsed_s': elapsed, 'footprint_bytes': footprint} + total = 0 + for action, values in timings.items(): + if not values: + continue + stats = percentiles(values) + stats['misses'] = misses.get(action, 0) + out['ops'][action] = stats + total += len(values) + out['throughput_ops_s'] = total / elapsed if elapsed else None + if extra: + out.update(extra) + return out + + +def workload_sizes(cls, workdir): + store = cls(workdir) + rnd = random.Random(1) + result = {'tiers': {}} + perf = time.perf_counter_ns + t0 = time.perf_counter() + for label, size, ops, keys in SIZES: + payload = os.urandom(size) + key_list = [ + ('k%d-%s' % (i, label.replace(' ', ''))).encode() + for i in range(keys) + ] + set_t, get_t, misses = [], [], 0 + for i in range(ops): + key = key_list[i % keys] + start = perf() + store.set(key, payload) + set_t.append(perf() - start) + for i in range(ops): + key = key_list[rnd.randrange(keys)] + start = perf() + value = store.get(key) + get_t.append(perf() - start) + if value is None: + misses += 1 + result['tiers'][label] = { + 'size': size, + 'set': percentiles(set_t), + 'get': percentiles(get_t), + 'get_misses': misses, + } + store.clear() + elapsed = time.perf_counter() - t0 + result['elapsed_s'] = elapsed + result['footprint_bytes'] = store.footprint() + store.close() + return result + + +def workload_hit(cls, workdir, hit_ratio): + store = cls(workdir) + payload = os.urandom(256) + keys = [('h%d' % i).encode() for i in range(HIT_KEYS)] + # setup is not measured, so load the working set in bulk + store.set_many([(key, payload) for key in keys]) + rnd = random.Random(2) + perf = time.perf_counter_ns + get_t, misses = [], 0 + t0 = time.perf_counter() + for i in range(OPS): + if rnd.random() < hit_ratio: + key = keys[rnd.randrange(HIT_KEYS)] + else: + key = ('miss%d' % rnd.randrange(HIT_KEYS)).encode() + start = perf() + value = store.get(key) + get_t.append(perf() - start) + if value is None: + misses += 1 + elapsed = time.perf_counter() - t0 + footprint = store.footprint() + store.close() + return summarise( + {'get': get_t}, + {'get': misses}, + elapsed, + footprint, + {'hit_ratio_target': hit_ratio, 'working_set_keys': HIT_KEYS}, + ) + + +def _proc_worker(store_key, workdir, seed, queue): + cls = stores.STORES[store_key] + time.sleep(0.05) + store = cls(workdir) + timings, misses = run_classic(store, seed=seed) + store.close() + queue.put((timings, misses)) + + +def workload_procs(cls, workdir, nprocs, store_key): + store = cls(workdir) + prepopulate(store) + store.close() + ctx = multiprocessing.get_context('spawn') + queue = ctx.Queue() + procs = [ + ctx.Process( + target=_proc_worker, args=(store_key, workdir, seed, queue) + ) + for seed in range(nprocs) + ] + t0 = time.perf_counter() + for p in procs: + p.start() + results = [queue.get() for _ in procs] + for p in procs: + p.join() + elapsed = time.perf_counter() - t0 + timings = {'get': [], 'set': [], 'delete': []} + misses = {'get': 0, 'set': 0, 'delete': 0} + for t, m in results: + for action in timings: + timings[action].extend(t[action]) + misses[action] += m[action] + store = cls(workdir) + footprint = store.footprint() + store.close() + return summarise( + timings, misses, elapsed, footprint, {'processes': nprocs} + ) + + +def workload_threads(cls, workdir, nthreads=8): + store = cls(workdir) + prepopulate(store) + results = [None] * nthreads + errors = [] + + def worker(ident): + try: + handle = store.for_thread() + try: + results[ident] = run_classic(handle, seed=ident) + finally: + if handle is not store: + handle.close() + except BaseException as exc: # noqa: BLE001 + errors.append((ident, exc)) + + threads = [ + threading.Thread(target=worker, args=(i,)) for i in range(nthreads) + ] + t0 = time.perf_counter() + for t in threads: + t.start() + for t in threads: + t.join() + elapsed = time.perf_counter() - t0 + if errors: + ident, exc = errors[0] + raise RuntimeError('thread %d failed: %r' % (ident, exc)) from exc + timings = {'get': [], 'set': [], 'delete': []} + misses = {'get': 0, 'set': 0, 'delete': 0} + for t, m in results: + for action in timings: + timings[action].extend(t[action]) + misses[action] += m[action] + footprint = store.footprint() + store.close() + return summarise( + timings, misses, elapsed, footprint, {'threads': nthreads} + ) + + +WORKLOADS = { + 'classic': lambda cls, d, k: workload_classic(cls, d), + 'sizes': lambda cls, d, k: workload_sizes(cls, d), + 'hit95': lambda cls, d, k: workload_hit(cls, d, 0.95), + 'hit50': lambda cls, d, k: workload_hit(cls, d, 0.50), + 'procs4': lambda cls, d, k: workload_procs(cls, d, 4, k), + 'procs8': lambda cls, d, k: workload_procs(cls, d, 8, k), + 'threads8': lambda cls, d, k: workload_threads(cls, d, 8), +} + + +def environment(): + def cmd(*args): + try: + return subprocess.run( + args, capture_output=True, text=True, timeout=10 + ).stdout.strip() + except Exception: # noqa: BLE001 + return '' + + return { + 'date': datetime.date.today().isoformat(), + 'python': sys.version.split()[0], + 'implementation': platform.python_implementation(), + 'platform': platform.platform(), + 'machine': platform.machine(), + 'cpu': cmd('sysctl', '-n', 'machdep.cpu.brand_string') + or platform.processor(), + 'cores': os.cpu_count(), + 'sqlite': sqlite3.sqlite_version, + 'memcached': cmd('memcached', '-V'), + 'redis': cmd('redis-server', '--version'), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--store', required=True, choices=sorted(stores.STORES) + ) + parser.add_argument('--workload', required=True, choices=sorted(WORKLOADS)) + parser.add_argument('--runs', type=int, default=5) + parser.add_argument('--warmup-runs', type=int, default=1) + parser.add_argument('--out', required=True) + args = parser.parse_args() + cls = stores.STORES[args.store] + workload = WORKLOADS[args.workload] + runs = [] + for i in range(args.warmup_runs + args.runs): + workdir = tempfile.mkdtemp(prefix='bench-%s-' % args.store) + if cls.server: + cls(workdir).clear() + try: + result = workload(cls, workdir, args.store) + finally: + shutil.rmtree(workdir, ignore_errors=True) + if i >= args.warmup_runs: + runs.append(result) + print('run', i, 'done', file=sys.stderr) + doc = { + 'store': args.store, + 'store_name': cls.name, + 'store_version': cls.version(), + 'workload': args.workload, + 'runs': runs, + 'environment': environment(), + 'ops': OPS, + 'warmup': WARMUP, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, 'w') as writer: + json.dump(doc, writer, indent=1) + + +if __name__ == '__main__': + multiprocessing.freeze_support() + main() diff --git a/tests/benchmarks/stores.py b/tests/benchmarks/stores.py new file mode 100644 index 0000000..a53588c --- /dev/null +++ b/tests/benchmarks/stores.py @@ -0,0 +1,440 @@ +"""Store adapters with one small interface: get, set, delete, close, clear, +footprint, version. Keys and values are bytes. + +Each adapter wraps the library the way its own documentation shows, with +default settings unless the comment says otherwise. +""" + +import os +import sys + + +def diskcache_package(): + """Return the diskcache under test: the fork if installed, else upstream. + + The driver runs the same store in two interpreters, one holding + diskcache-community (imported as diskcache_community) and one holding + only diskcache 5.6.3 (imported as diskcache). + """ + try: + import diskcache_community as package + except ImportError: + import diskcache as package + return package + + +def chunks(items, size): + items = list(items) + for start in range(0, len(items), size): + yield items[start : start + size] + + +def tree_size(path): + total = 0 + for root, _dirs, files in os.walk(path): + for name in files: + try: + total += os.path.getsize(os.path.join(root, name)) + except OSError: + pass + return total + + +class Store: + name = 'store' + server = None # 'memcached' or 'redis' when a server process is needed + needs_fork_venv = False # True for the diskcache-community adapters + + def __init__(self, workdir): + self.workdir = workdir + + def get(self, key): + raise NotImplementedError + + def set(self, key, value): + raise NotImplementedError + + def set_many(self, items): + """Bulk load for unmeasured setup; the default is one set per item.""" + for key, value in items: + self.set(key, value) + + def delete(self, key): + raise NotImplementedError + + def close(self): + pass + + def clear(self): + raise NotImplementedError + + def footprint(self): + return tree_size(self.workdir) + + def for_thread(self): + """The handle a worker thread should use; the shared store by default. + + Clients that are not thread-safe return a fresh connection, which is + what their pooled variants hand out per thread. + """ + return self + + @classmethod + def version(cls): + return '' + + +# ------------------------------------------------------------ diskcache + + +class DiskCacheCache(Store): + name = 'diskcache.Cache' + + def __init__(self, workdir): + diskcache = diskcache_package() + + super().__init__(workdir) + self.cache = diskcache.Cache(workdir) + + def get(self, key): + return self.cache.get(key) + + def set(self, key, value): + return self.cache.set(key, value) + + def set_many(self, items): + with self.cache.transact(): + for key, value in items: + self.cache.set(key, value) + + def delete(self, key): + return self.cache.delete(key) + + def close(self): + self.cache.close() + + def clear(self): + self.cache.clear() + + @classmethod + def version(cls): + diskcache = diskcache_package() + + return '%s %s' % (diskcache.__title__, diskcache.__version__) + + +class DiskCacheFanout(DiskCacheCache): + name = 'diskcache.FanoutCache(shards=8, timeout=0.010)' + + def __init__(self, workdir): + diskcache = diskcache_package() + + Store.__init__(self, workdir) + self.cache = diskcache.FanoutCache(workdir, shards=8, timeout=0.010) + + +# ------------------------------------------------------------ memcached + + +class PyMemcache(Store): + name = 'memcached (pymemcache)' + server = 'memcached' + + def __init__(self, workdir): + import pymemcache.client.base + + super().__init__(workdir) + # wait for the server's reply on every command, as the other clients do + self.client = pymemcache.client.base.Client( + ('127.0.0.1', 11211), no_delay=True, default_noreply=False + ) + + def get(self, key): + return self.client.get(key) + + def set(self, key, value): + return self.client.set(key, value) + + def set_many(self, items): + for chunk in chunks(items, 1000): + self.client.set_many(dict(chunk), noreply=False) + + def for_thread(self): + # pymemcache.Client is not thread-safe; PooledClient would do the same + return PyMemcache(self.workdir) + + def delete(self, key): + return self.client.delete(key, noreply=False) + + def close(self): + self.client.close() + + def clear(self): + self.client.flush_all() + + def footprint(self): + return 0 + + @classmethod + def version(cls): + import pymemcache + + return 'pymemcache %s' % pymemcache.__version__ + + +class PyLibMC(Store): + name = 'memcached (pylibmc)' + server = 'memcached' + + def __init__(self, workdir): + import pylibmc + + super().__init__(workdir) + # the settings the 2016 benchmark used + self.client = pylibmc.Client( + ['127.0.0.1'], + binary=True, + behaviors={'tcp_nodelay': True, 'ketama': True}, + ) + + def get(self, key): + return self.client.get(key) + + def set(self, key, value): + return self.client.set(key, value) + + def set_many(self, items): + for chunk in chunks(items, 1000): + self.client.set_multi(dict(chunk)) + + def for_thread(self): + # pylibmc.Client is not thread-safe; ThreadMappedPool clones one per thread + return PyLibMC(self.workdir) + + def delete(self, key): + return self.client.delete(key) + + def close(self): + self.client.disconnect_all() + + def clear(self): + self.client.flush_all() + + def footprint(self): + return 0 + + @classmethod + def version(cls): + import pylibmc + + return 'pylibmc %s, libmemcached %s' % ( + pylibmc.__version__, + pylibmc.libmemcached_version, + ) + + +# ---------------------------------------------------------------- redis + + +class Redis(Store): + name = 'redis (redis-py)' + server = 'redis' + + def __init__(self, workdir): + import redis + + super().__init__(workdir) + self.client = redis.Redis(host='127.0.0.1', port=6379, db=1) + + def get(self, key): + return self.client.get(key) + + def set(self, key, value): + return self.client.set(key, value) + + def set_many(self, items): + for chunk in chunks(items, 10000): + self.client.mset(dict(chunk)) + + def delete(self, key): + return bool(self.client.delete(key)) + + def close(self): + self.client.close() + + def clear(self): + self.client.flushdb() + + def footprint(self): + return 0 + + @classmethod + def version(cls): + import redis + + hiredis = ( + 'with hiredis' + if redis.utils.HIREDIS_AVAILABLE + else 'without hiredis' + ) + return 'redis-py %s %s' % (redis.__version__, hiredis) + + +# ------------------------------------------------------ embedded stores + + +class SqliteDict(Store): + name = 'sqlitedict (autocommit)' + + def __init__(self, workdir): + import sqlitedict + + super().__init__(workdir) + os.makedirs(workdir, exist_ok=True) + self.db = sqlitedict.SqliteDict( + os.path.join(workdir, 'store.sqlite'), autocommit=True + ) + + def get(self, key): + # UserDict.get checks membership first, so a concurrent delete from + # another thread can surface as KeyError here + try: + return self.db[key.decode()] + except KeyError: + return None + + def set(self, key, value): + self.db[key.decode()] = value + return True + + def set_many(self, items): + self.db.update((key.decode(), value) for key, value in items) + + def delete(self, key): + try: + del self.db[key.decode()] + except KeyError: + return False + return True + + def close(self): + self.db.close() + + def clear(self): + self.db.clear() + + @classmethod + def version(cls): + import sqlitedict + + return 'sqlitedict %s' % sqlitedict.__version__ + + +class Shelve(Store): + name = 'shelve (dbm)' + + def __init__(self, workdir): + import shelve + + super().__init__(workdir) + os.makedirs(workdir, exist_ok=True) + self.db = shelve.open(os.path.join(workdir, 'shelf')) + + def get(self, key): + return self.db.get(key.decode()) + + def set(self, key, value): + self.db[key.decode()] = value + return True + + def delete(self, key): + try: + del self.db[key.decode()] + except KeyError: + return False + return True + + def close(self): + self.db.close() + + def clear(self): + self.db.clear() + + @classmethod + def version(cls): + import dbm + import shelve + import shutil + import tempfile + + # open a throwaway shelf to see which dbm module this Python picks + probe = tempfile.mkdtemp(prefix='bench-shelve-probe-') + try: + shelve.open(os.path.join(probe, 'shelf')).close() + backend = dbm.whichdb(os.path.join(probe, 'shelf')) or 'unknown' + finally: + shutil.rmtree(probe, ignore_errors=True) + return 'shelve over %s, Python %s' % (backend, sys.version.split()[0]) + + +class Lmdb(Store): + name = 'lmdb (py-lmdb)' + + def __init__(self, workdir): + import lmdb + + super().__init__(workdir) + os.makedirs(workdir, exist_ok=True) + self.env = lmdb.open(workdir, map_size=4 * 2**30) + + def get(self, key): + with self.env.begin() as txn: + return txn.get(key) + + def set(self, key, value): + with self.env.begin(write=True) as txn: + return txn.put(key, value) + + def set_many(self, items): + with self.env.begin(write=True) as txn: + for key, value in items: + txn.put(key, value) + + def delete(self, key): + with self.env.begin(write=True) as txn: + return txn.delete(key) + + def close(self): + self.env.close() + + def clear(self): + with self.env.begin(write=True) as txn: + txn.drop(self.env.open_db(), delete=False) + + def footprint(self): + # the data file is sparse up to map_size; count the used pages + info = self.env.info() + stat = self.env.stat() + return stat['psize'] * (info['last_pgno'] + 1) + + @classmethod + def version(cls): + import lmdb + + return 'py-lmdb %s, liblmdb %s' % ( + lmdb.__version__, + '.'.join(map(str, lmdb.version())), + ) + + +STORES = { + 'cache': DiskCacheCache, + 'fanout': DiskCacheFanout, + 'cache563': DiskCacheCache, # run under the venv holding diskcache 5.6.3 + 'pymemcache': PyMemcache, + 'pylibmc': PyLibMC, + 'redis': Redis, + 'sqlitedict': SqliteDict, + 'shelve': Shelve, + 'lmdb': Lmdb, +} diff --git a/tox.ini b/tox.ini index cdd093b..982e604 100644 --- a/tox.ini +++ b/tox.ini @@ -73,6 +73,7 @@ addopts= --ignore docs/case-study-web-crawler.rst --ignore docs/sf-python-2017-meetup-talk.rst --ignore tests/benchmark_core.py + --ignore tests/benchmarks --ignore tests/benchmark_djangocache.py --ignore tests/benchmark_glob.py --ignore tests/issue_85.py