From 448d363e31de2898e1d8284f36837872f888931f Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Fri, 7 Aug 2026 14:16:08 -0400 Subject: [PATCH] MDEV-40669 HEAP MIN_ROWS pre-sizes blocks past max_heap_table_size `init_block()` captures `requested_min_records` from the caller's `min_records` before the `min_records= MY_MIN(min_records, max_records)` clamp, then restores that raw value when the block allocation cap added by MDEV-40447 fires. `max_records` is derived from `max_heap_table_size` / `tmp_memory_table_size` and is the most rows the table can ever hold, so that clamp is what has always bounded a HEAP table's allocations. Restoring the pre-clamp value undoes it, and an unreachable `MIN_ROWS` pre-sizes the record block and every hash key block past the ceiling. With `max_heap_table_size=64M` and `MIN_ROWS=20000000` a one-row table allocates 1GB where it used to allocate 96MB; `MIN_ROWS=4294967295` at the shipped default 16MB ceiling allocates 4GB, bounded only by the `INT_MAX32` clamp on `memory_needed`. A few such tables exhaust memory. Fix: clamp `requested_min_records` to `max_records` as well. `MY_MIN` keeps 0 at 0, so "no `min_records` requested" stays distinguishable from an explicit `MIN_ROWS`, and the cap keeps ignoring the defaulted 1000-row heuristic. The cap and the clamp together give four regimes, and only the last one changes: 1. No `MIN_ROWS`: capped, sizing comes from the ceiling alone. 2. `MIN_ROWS` below the cap: capped. 3. `MIN_ROWS` above the cap but within `max_records`: pre-sizes to `MIN_ROWS`, past the cap, as MDEV-40447 intends. 4. `MIN_ROWS` at or above `max_records`: unreachable, so it degrades to plain ceiling-derived sizing. In case 4 the cap branch becomes a no-op, because `records_in_block` already equals `max_records`, so sizing returns to exactly what it was before MDEV-40447. `init_block()` no longer defaults `max_records` itself. The block sizing ceiling is derived once in `heap_create()` and the parameter is `const`, so the caller's value reaches `share->max_records` unchanged: 0 there means "no row limit", and `hp_alloc_from_tail()` skips the limit check only while it is 0. Tests: - `storage/heap/hp_test_block_size-t.c`: a four-case boundary walk asserting the exact `alloc_size` of each regime above at one ceiling-derived `max_records`, and a keyed case asserting that an unreachable `MIN_ROWS` clamps the hash key block as well as the record block (`sizeof(HASH_INFO)` gives that block its own `recbuffer` and its own cap). A `max_records=0` case covers the derived ceiling: the block is sized from it while `share->max_records` stays 0, and the table accepts far more rows than that default. - `mysql-test/suite/heap/min_rows_alloc.test`: the same regimes end to end across three ceilings, plus `MIN_ROWS` at the .frm maximum under the shipped default ceiling. --- mysql-test/suite/heap/min_rows_alloc.result | 89 ++++++++++ mysql-test/suite/heap/min_rows_alloc.test | 95 ++++++++++ storage/heap/hp_create.c | 37 +++- storage/heap/hp_test_block_size-t.c | 181 +++++++++++++++++++- 4 files changed, 393 insertions(+), 9 deletions(-) create mode 100644 mysql-test/suite/heap/min_rows_alloc.result create mode 100644 mysql-test/suite/heap/min_rows_alloc.test diff --git a/mysql-test/suite/heap/min_rows_alloc.result b/mysql-test/suite/heap/min_rows_alloc.result new file mode 100644 index 0000000000000..c06920a1f8503 --- /dev/null +++ b/mysql-test/suite/heap/min_rows_alloc.result @@ -0,0 +1,89 @@ +# +# A ceiling large enough for the block cap to be live. +# (a INT, b INT, KEY(a)) rows are 16 bytes, hash entries 24. +# +SET SESSION max_heap_table_size= 256*1024*1024; +# +# No MIN_ROWS: sizing comes from the ceiling alone and is capped. +# +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +4194272 4194272 +DROP TABLE t1; +# +# MIN_ROWS below the cap does not raise the block above the cap. +# +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +4194272 4194272 +DROP TABLE t1; +# +# MIN_ROWS above the cap but within the ceiling still pre-sizes +# past the cap: it is a real row count expectation. +# +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +16777184 33554400 +DROP TABLE t1; +# +# MIN_ROWS beyond the ceiling is unreachable and must clamp to it +# instead of reserving memory the table can never use. Both of +# these hold more rows than 256M can fit, so both must produce the +# same, ceiling-derived, sizing. +# +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +134217696 268435424 +DROP TABLE t1; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=40000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +134217696 268435424 +DROP TABLE t1; +# +# A smaller ceiling clamps harder, and MIN_ROWS at the .frm +# maximum stays bounded even at the shipped default ceiling. +# +SET SESSION max_heap_table_size= 64*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +33554400 67108832 +DROP TABLE t1; +SET SESSION max_heap_table_size= 16*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=4294967295; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +8388576 16777184 +DROP TABLE t1; +# +# The clamp is a ceiling clamp, not a MIN_ROWS ban: raising the +# ceiling lets the same MIN_ROWS pre-size further. +# +SET SESSION max_heap_table_size= 1024*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=8000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +134217696 268435424 +DROP TABLE t1; +SET SESSION max_heap_table_size= DEFAULT; diff --git a/mysql-test/suite/heap/min_rows_alloc.test b/mysql-test/suite/heap/min_rows_alloc.test new file mode 100644 index 0000000000000..6c67781d5f42a --- /dev/null +++ b/mysql-test/suite/heap/min_rows_alloc.test @@ -0,0 +1,95 @@ +# MIN_ROWS pre-sizes a MEMORY table's blocks, but it must never pre-size +# them past max_heap_table_size: max_records is derived from that ceiling +# and is the most rows the table can ever hold, so a larger MIN_ROWS is +# unreachable. Block allocations are also capped (see +# main.tmp_table_heap_alloc) when they come from the ceiling rather than +# from a caller row-count expectation. The two interact, so walk all of +# the regimes: no MIN_ROWS, MIN_ROWS below the cap, MIN_ROWS above the +# cap but reachable, and MIN_ROWS beyond the ceiling. +# +# Sizes are exact byte counts and depend on pointer width. + +--source include/have_64bit.inc + +--echo # +--echo # A ceiling large enough for the block cap to be live. +--echo # (a INT, b INT, KEY(a)) rows are 16 bytes, hash entries 24. +--echo # +SET SESSION max_heap_table_size= 256*1024*1024; + +--echo # +--echo # No MIN_ROWS: sizing comes from the ceiling alone and is capped. +--echo # +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +--echo # +--echo # MIN_ROWS below the cap does not raise the block above the cap. +--echo # +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +--echo # +--echo # MIN_ROWS above the cap but within the ceiling still pre-sizes +--echo # past the cap: it is a real row count expectation. +--echo # +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +--echo # +--echo # MIN_ROWS beyond the ceiling is unreachable and must clamp to it +--echo # instead of reserving memory the table can never use. Both of +--echo # these hold more rows than 256M can fit, so both must produce the +--echo # same, ceiling-derived, sizing. +--echo # +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=40000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +--echo # +--echo # A smaller ceiling clamps harder, and MIN_ROWS at the .frm +--echo # maximum stays bounded even at the shipped default ceiling. +--echo # +SET SESSION max_heap_table_size= 64*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +SET SESSION max_heap_table_size= 16*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=4294967295; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +--echo # +--echo # The clamp is a ceiling clamp, not a MIN_ROWS ban: raising the +--echo # ceiling lets the same MIN_ROWS pre-size further. +--echo # +SET SESSION max_heap_table_size= 1024*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=8000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +SET SESSION max_heap_table_size= DEFAULT; diff --git a/storage/heap/hp_create.c b/storage/heap/hp_create.c index 8bdc08bb36908..012584927b112 100644 --- a/storage/heap/hp_create.c +++ b/storage/heap/hp_create.c @@ -19,7 +19,7 @@ static int keys_compare(void *heap_rb, const void *key1, const void *key2); static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records, - ulong max_records); + const ulong max_records); /* @@ -63,6 +63,15 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, ulong min_records= create_info->min_records; ulong max_records= create_info->max_records; uint visible_offset; + /* + max_records is the table's row limit and 0 means "no limit"; it is + stored as such in share->max_records, where hp_alloc_from_tail() + relies on 0 disabling the check. Block sizing needs a concrete row + count instead, so derive that ceiling here and leave max_records + itself untouched. + */ + ulong block_max_records= (max_records ? max_records : + MY_MAX(min_records, 1000)); DBUG_ENTER("heap_create"); if (!create_info->internal_table) @@ -243,7 +252,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, share->blob_count= create_info->blob_count; } init_block(&share->block, hp_memory_needed_per_row(reclength), - min_records, max_records); + min_records, block_max_records); /* Fix keys */ memcpy(share->keydef, keydef, (size_t) (sizeof(keydef[0]) * keys)); for (i= 0, keyinfo= share->keydef; i < keys; i++, keyinfo++) @@ -272,7 +281,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, else { init_block(&keyinfo->block, sizeof(HASH_INFO), min_records, - max_records); + block_max_records); keyinfo->delete_key= hp_delete_key; keyinfo->write_key= hp_write_key; keyinfo->hash_buckets= 0; @@ -375,7 +384,7 @@ ha_rows hp_rows_in_memory(size_t reclength, size_t index_size, static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records, - ulong max_records) + const ulong max_records) { ulong i,records_in_block,cap_records; ulong recbuffer= (ulong) MY_ALIGN(reclength, sizeof(uchar*)); @@ -385,13 +394,22 @@ static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records, size_t alloc_size; /* - If not min_records and max_records are given, optimize for 1000 rows + If no min_records is given, optimize for 1000 rows. max_records is + the caller's sizing ceiling and is never changed here. */ if (!min_records) min_records= MY_MIN(1000, max_records / heap_allocation_parts); - if (!max_records) - max_records= MY_MAX(min_records, 1000); min_records= MY_MIN(min_records, max_records); + /* + An explicit min_records may override the cap below, but only as far + as the ceiling reaches: max_records is the most rows the table's + memory ceiling (max_heap_table_size / tmp_memory_table_size) can + ever hold, so a larger MIN_ROWS is unreachable and pre-sizing for it + would reserve memory the table can never use. MY_MIN keeps 0 at 0, + so "no min_records requested" stays distinguishable from an explicit + one. + */ + requested_min_records= MY_MIN(requested_min_records, max_records); /* We don't want too few records_in_block as otherwise the overhead of @@ -417,7 +435,10 @@ static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records, heap_max_allocation_block. Only an explicit min_records from the caller (CREATE TABLE ... MIN_ROWS) is a real row count expectation and may pre-size beyond the cap; the 1000-row default min_records - is a heuristic and must not override the cap. + is a heuristic and must not override the cap. That override is + bounded by max_records (clamped above), so an unreachable MIN_ROWS + degrades to plain ceiling-derived sizing instead of reserving a + block the ceiling can never fill. */ cap_records= (heap_max_allocation_block - extra) / recbuffer; if (records_in_block > cap_records) diff --git a/storage/heap/hp_test_block_size-t.c b/storage/heap/hp_test_block_size-t.c index 58111f600b3b9..f900e534b02d7 100644 --- a/storage/heap/hp_test_block_size-t.c +++ b/storage/heap/hp_test_block_size-t.c @@ -16,7 +16,12 @@ that memory allocators recycle without returning pages to the OS; - an explicit min_records pre-sizing hint (CREATE TABLE ... MIN_ROWS) still overrides the cap; + - an explicit min_records below the cap does not defeat the cap; + - an explicit min_records above max_records is unreachable and is + clamped to the ceiling instead of pre-sizing past it; - small tables are sized exactly as before; + - max_records=0 keeps meaning "no row limit" even though block + sizing needs a concrete row count; - a capped table remains fully functional past the first block. */ @@ -407,11 +412,176 @@ static void test_explicit_min_records_wide_row(void) } +/* + Walk the whole min_records / cap interaction at one ceiling-derived + max_records, so the boundaries between the regimes are pinned down + together rather than one at a time. + + max_records 1M with 104-byte rows puts max_records/heap_allocation_parts + (62500 records) above cap_records (40319), so the cap is live for every + case here and only min_records decides the outcome: + + - no min_records -> capped (the heuristic must not override) + - min_records below cap -> capped (too small to raise the block) + - cap < min_records < max -> pre-sized to min_records, past the cap + - min_records >= max -> unreachable, clamped to max_records +*/ + +#define BOUNDARY_MAX_RECORDS 1000000UL + +static void test_min_records_cap_boundary(void) +{ + static const struct + { + const char *name; + ulong min_records; + size_t expected_alloc; + const char *what; + } cases[]= + { + { "test_block_bnd_none", 0UL, + 4194272, "no min_records is capped" }, + { "test_block_bnd_small", 10000UL, + 4194272, "min_records below the cap is capped" }, + { "test_block_bnd_mid", 200000UL, + 33554400, "min_records above the cap pre-sizes past it" }, + { "test_block_bnd_over", 100000000UL, + 134217696, "min_records above max_records clamps to the ceiling" } + }; + uint i; + + for (i= 0; i < array_elements(cases); i++) + { + HP_SHARE *share; + HP_INFO *info; + + if (create_table(cases[i].name, 0, REC_LENGTH, cases[i].min_records, + BOUNDARY_MAX_RECORDS, NULL, &share)) + { + ok(0, "setup failed for %s: %d", cases[i].what, my_errno); + continue; + } + + ok(share->block.alloc_size == cases[i].expected_alloc, + "%s (got %zu, expected %zu)", cases[i].what, + share->block.alloc_size, cases[i].expected_alloc); + + if ((info= heap_open(cases[i].name, 2))) + { + heap_drop_table(info); + heap_close(info); + } + } +} + + +/* + Test: the ceiling clamp applies to the hash key block too. An + unreachable MIN_ROWS inflates DATA_LENGTH and INDEX_LENGTH alike, + because both blocks run through init_block(). sizeof(HASH_INFO) is + 24 on LP64, so the key block has its own recbuffer, its own + cap_records and its own expected size -- assert both blocks, not just + the record one. +*/ + +static void test_min_records_above_max_records_keyed(void) +{ + HP_SHARE *share; + HP_INFO *info; + HP_KEYDEF keydef; + HA_KEYSEG keyseg; + + init_int_keydef(&keydef, &keyseg); + + if (create_table("test_block_bnd_keyed", 1, REC_LENGTH, 100000000UL, + BOUNDARY_MAX_RECORDS, &keydef, &share)) + { + ok(0, "setup failed: %d", my_errno); + skip(2, "setup failed"); + return; + } + + ok(share->block.alloc_size == 134217696, + "record block clamped to the ceiling (got %zu)", + share->block.alloc_size); + + /* 1M * sizeof(HASH_INFO) + extra, rounded up to 32MB */ + ok(share->keydef[0].block.alloc_size == 33554400, + "hash key block clamped to the ceiling (got %zu)", + share->keydef[0].block.alloc_size); + + if ((info= heap_open("test_block_bnd_keyed", 2))) + { + heap_drop_table(info); + heap_close(info); + } +} + + +/* + Test: max_records=0 means "no row limit", not "1000 rows". + + Block sizing needs a concrete row count, so heap_create() derives one + when the caller passes max_records=0. That derived value must not + reach share->max_records: hp_alloc_from_tail() only skips the row + limit check while max_records is 0, so storing the derived default + would turn an unlimited table into one that reports + HA_ERR_RECORD_FILE_FULL a few blocks in. +*/ + +static void test_no_max_records_is_unlimited(void) +{ + HP_SHARE *share; + HP_INFO *info; + uchar rec[REC_LENGTH]; + ulong i; + ulong write_failures= 0; + const ulong rows= 5000; + + if (create_table("test_block_nomax", 0, REC_LENGTH, 0, 0, NULL, &share)) + { + ok(0, "setup failed: %d", my_errno); + skip(3, "setup failed"); + return; + } + ok(1, "created keyless table with max_records=0"); + + ok(share->max_records == 0, + "max_records stays 0, meaning no row limit (got %lu)", + share->max_records); + + ok(share->block.alloc_size == 16352, + "block sized from the derived default (got %zu, expected 16352)", + share->block.alloc_size); + + if (!(info= heap_open("test_block_nomax", 2))) + { + ok(0, "heap_open failed: %d", my_errno); + skip(1, "open failed"); + return; + } + heap_extra(info, HA_EXTRA_NO_READCHECK); + + for (i= 0; i < rows; i++) + { + build_record(rec, (int32) i); + if (heap_write(info, rec)) + write_failures++; + } + ok(write_failures == 0, + "wrote %lu rows, well past the derived default, without error " + "(%lu failures)", rows, write_failures); + + heap_drop_table(info); + heap_close(info); +} + + int main(int argc __attribute__((unused)), char **argv __attribute__((unused))) { MY_INIT("hp_test_block_size"); - plan(24); + plan(34); diag("Test 1: ceiling-derived max_records capped (keyed table)"); test_huge_max_records_capped(); @@ -437,6 +607,15 @@ int main(int argc __attribute__((unused)), diag("Test 8: explicit min_records still overrides cap for wide rows"); test_explicit_min_records_wide_row(); + diag("Test 9: min_records / cap boundary walk"); + test_min_records_cap_boundary(); + + diag("Test 10: unreachable min_records clamps record and key blocks"); + test_min_records_above_max_records_keyed(); + + diag("Test 11: max_records=0 stays unlimited"); + test_no_max_records_is_unlimited(); + my_end(0); return exit_status(); }