diff --git a/mysql-test/suite/heap/min_rows_alloc.result b/mysql-test/suite/heap/min_rows_alloc.result new file mode 100644 index 0000000000000..c06920a1f8503 --- /dev/null +++ b/mysql-test/suite/heap/min_rows_alloc.result @@ -0,0 +1,89 @@ +# +# A ceiling large enough for the block cap to be live. +# (a INT, b INT, KEY(a)) rows are 16 bytes, hash entries 24. +# +SET SESSION max_heap_table_size= 256*1024*1024; +# +# No MIN_ROWS: sizing comes from the ceiling alone and is capped. +# +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +4194272 4194272 +DROP TABLE t1; +# +# MIN_ROWS below the cap does not raise the block above the cap. +# +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +4194272 4194272 +DROP TABLE t1; +# +# MIN_ROWS above the cap but within the ceiling still pre-sizes +# past the cap: it is a real row count expectation. +# +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +16777184 33554400 +DROP TABLE t1; +# +# MIN_ROWS beyond the ceiling is unreachable and must clamp to it +# instead of reserving memory the table can never use. Both of +# these hold more rows than 256M can fit, so both must produce the +# same, ceiling-derived, sizing. +# +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +134217696 268435424 +DROP TABLE t1; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=40000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +134217696 268435424 +DROP TABLE t1; +# +# A smaller ceiling clamps harder, and MIN_ROWS at the .frm +# maximum stays bounded even at the shipped default ceiling. +# +SET SESSION max_heap_table_size= 64*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +33554400 67108832 +DROP TABLE t1; +SET SESSION max_heap_table_size= 16*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=4294967295; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +8388576 16777184 +DROP TABLE t1; +# +# The clamp is a ceiling clamp, not a MIN_ROWS ban: raising the +# ceiling lets the same MIN_ROWS pre-size further. +# +SET SESSION max_heap_table_size= 1024*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=8000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DATA_LENGTH INDEX_LENGTH +134217696 268435424 +DROP TABLE t1; +SET SESSION max_heap_table_size= DEFAULT; diff --git a/mysql-test/suite/heap/min_rows_alloc.test b/mysql-test/suite/heap/min_rows_alloc.test new file mode 100644 index 0000000000000..6c67781d5f42a --- /dev/null +++ b/mysql-test/suite/heap/min_rows_alloc.test @@ -0,0 +1,95 @@ +# MIN_ROWS pre-sizes a MEMORY table's blocks, but it must never pre-size +# them past max_heap_table_size: max_records is derived from that ceiling +# and is the most rows the table can ever hold, so a larger MIN_ROWS is +# unreachable. Block allocations are also capped (see +# main.tmp_table_heap_alloc) when they come from the ceiling rather than +# from a caller row-count expectation. The two interact, so walk all of +# the regimes: no MIN_ROWS, MIN_ROWS below the cap, MIN_ROWS above the +# cap but reachable, and MIN_ROWS beyond the ceiling. +# +# Sizes are exact byte counts and depend on pointer width. + +--source include/have_64bit.inc + +--echo # +--echo # A ceiling large enough for the block cap to be live. +--echo # (a INT, b INT, KEY(a)) rows are 16 bytes, hash entries 24. +--echo # +SET SESSION max_heap_table_size= 256*1024*1024; + +--echo # +--echo # No MIN_ROWS: sizing comes from the ceiling alone and is capped. +--echo # +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +--echo # +--echo # MIN_ROWS below the cap does not raise the block above the cap. +--echo # +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +--echo # +--echo # MIN_ROWS above the cap but within the ceiling still pre-sizes +--echo # past the cap: it is a real row count expectation. +--echo # +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=1000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +--echo # +--echo # MIN_ROWS beyond the ceiling is unreachable and must clamp to it +--echo # instead of reserving memory the table can never use. Both of +--echo # these hold more rows than 256M can fit, so both must produce the +--echo # same, ceiling-derived, sizing. +--echo # +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=40000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +--echo # +--echo # A smaller ceiling clamps harder, and MIN_ROWS at the .frm +--echo # maximum stays bounded even at the shipped default ceiling. +--echo # +SET SESSION max_heap_table_size= 64*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=20000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +SET SESSION max_heap_table_size= 16*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=4294967295; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +--echo # +--echo # The clamp is a ceiling clamp, not a MIN_ROWS ban: raising the +--echo # ceiling lets the same MIN_ROWS pre-size further. +--echo # +SET SESSION max_heap_table_size= 1024*1024*1024; +CREATE TABLE t1 (a INT, b INT, KEY(a)) ENGINE=MEMORY MIN_ROWS=8000000; +INSERT INTO t1 VALUES (1,1),(2,2); +SELECT DATA_LENGTH, INDEX_LENGTH FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; + +SET SESSION max_heap_table_size= DEFAULT; diff --git a/storage/heap/hp_create.c b/storage/heap/hp_create.c index 8bdc08bb36908..012584927b112 100644 --- a/storage/heap/hp_create.c +++ b/storage/heap/hp_create.c @@ -19,7 +19,7 @@ static int keys_compare(void *heap_rb, const void *key1, const void *key2); static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records, - ulong max_records); + const ulong max_records); /* @@ -63,6 +63,15 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, ulong min_records= create_info->min_records; ulong max_records= create_info->max_records; uint visible_offset; + /* + max_records is the table's row limit and 0 means "no limit"; it is + stored as such in share->max_records, where hp_alloc_from_tail() + relies on 0 disabling the check. Block sizing needs a concrete row + count instead, so derive that ceiling here and leave max_records + itself untouched. + */ + ulong block_max_records= (max_records ? max_records : + MY_MAX(min_records, 1000)); DBUG_ENTER("heap_create"); if (!create_info->internal_table) @@ -243,7 +252,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, share->blob_count= create_info->blob_count; } init_block(&share->block, hp_memory_needed_per_row(reclength), - min_records, max_records); + min_records, block_max_records); /* Fix keys */ memcpy(share->keydef, keydef, (size_t) (sizeof(keydef[0]) * keys)); for (i= 0, keyinfo= share->keydef; i < keys; i++, keyinfo++) @@ -272,7 +281,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, else { init_block(&keyinfo->block, sizeof(HASH_INFO), min_records, - max_records); + block_max_records); keyinfo->delete_key= hp_delete_key; keyinfo->write_key= hp_write_key; keyinfo->hash_buckets= 0; @@ -375,7 +384,7 @@ ha_rows hp_rows_in_memory(size_t reclength, size_t index_size, static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records, - ulong max_records) + const ulong max_records) { ulong i,records_in_block,cap_records; ulong recbuffer= (ulong) MY_ALIGN(reclength, sizeof(uchar*)); @@ -385,13 +394,22 @@ static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records, size_t alloc_size; /* - If not min_records and max_records are given, optimize for 1000 rows + If no min_records is given, optimize for 1000 rows. max_records is + the caller's sizing ceiling and is never changed here. */ if (!min_records) min_records= MY_MIN(1000, max_records / heap_allocation_parts); - if (!max_records) - max_records= MY_MAX(min_records, 1000); min_records= MY_MIN(min_records, max_records); + /* + An explicit min_records may override the cap below, but only as far + as the ceiling reaches: max_records is the most rows the table's + memory ceiling (max_heap_table_size / tmp_memory_table_size) can + ever hold, so a larger MIN_ROWS is unreachable and pre-sizing for it + would reserve memory the table can never use. MY_MIN keeps 0 at 0, + so "no min_records requested" stays distinguishable from an explicit + one. + */ + requested_min_records= MY_MIN(requested_min_records, max_records); /* We don't want too few records_in_block as otherwise the overhead of @@ -417,7 +435,10 @@ static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records, heap_max_allocation_block. Only an explicit min_records from the caller (CREATE TABLE ... MIN_ROWS) is a real row count expectation and may pre-size beyond the cap; the 1000-row default min_records - is a heuristic and must not override the cap. + is a heuristic and must not override the cap. That override is + bounded by max_records (clamped above), so an unreachable MIN_ROWS + degrades to plain ceiling-derived sizing instead of reserving a + block the ceiling can never fill. */ cap_records= (heap_max_allocation_block - extra) / recbuffer; if (records_in_block > cap_records) diff --git a/storage/heap/hp_test_block_size-t.c b/storage/heap/hp_test_block_size-t.c index 58111f600b3b9..04b1fffa05443 100644 --- a/storage/heap/hp_test_block_size-t.c +++ b/storage/heap/hp_test_block_size-t.c @@ -16,7 +16,12 @@ that memory allocators recycle without returning pages to the OS; - an explicit min_records pre-sizing hint (CREATE TABLE ... MIN_ROWS) still overrides the cap; + - an explicit min_records below the cap does not defeat the cap; + - an explicit min_records above max_records is unreachable and is + clamped to the ceiling instead of pre-sizing past it; - small tables are sized exactly as before; + - max_records=0 keeps meaning "no row limit" even though block + sizing needs a concrete row count; - a capped table remains fully functional past the first block. */ @@ -407,11 +412,182 @@ static void test_explicit_min_records_wide_row(void) } +/* + Walk the whole min_records / cap interaction at one ceiling-derived + max_records, so the boundaries between the regimes are pinned down + together rather than one at a time. + + max_records 1M with 104-byte rows puts max_records/heap_allocation_parts + (62500 records) above cap_records (40319), so the cap is live for every + case here and only min_records decides the outcome: + + - no min_records -> capped (the heuristic must not override) + - min_records below cap -> capped (too small to raise the block) + - cap < min_records < max -> pre-sized to min_records, past the cap + - min_records >= max -> unreachable, clamped to max_records +*/ + +#define BOUNDARY_MAX_RECORDS 1000000UL + +static void test_min_records_cap_boundary(void) +{ + static const struct + { + const char *name; + ulong min_records; + size_t expected_alloc; + const char *what; + } cases[]= + { + { "test_block_bnd_none", 0UL, + 4194272, "no min_records is capped" }, + { "test_block_bnd_small", 10000UL, + 4194272, "min_records below the cap is capped" }, + { "test_block_bnd_mid", 200000UL, + 33554400, "min_records above the cap pre-sizes past it" }, + { "test_block_bnd_over", 100000000UL, + 134217696, "min_records above max_records clamps to the ceiling" } + }; + uint i; + + for (i= 0; i < array_elements(cases); i++) + { + HP_SHARE *share; + HP_INFO *info; + + if (create_table(cases[i].name, 0, REC_LENGTH, cases[i].min_records, + BOUNDARY_MAX_RECORDS, NULL, &share)) + { + ok(0, "setup failed for %s: %d", cases[i].what, my_errno); + continue; + } + + ok(share->block.alloc_size == cases[i].expected_alloc, + "%s (got %zu, expected %zu)", cases[i].what, + share->block.alloc_size, cases[i].expected_alloc); + + if ((info= heap_open(cases[i].name, 2))) + { + heap_drop_table(info); + heap_close(info); + } + } +} + + +/* + Test: the ceiling clamp applies to the hash key block too. An + unreachable MIN_ROWS inflates DATA_LENGTH and INDEX_LENGTH alike, + because both blocks run through init_block(). sizeof(HASH_INFO) is + 24 on LP64 and 12 on ILP32, so the key block has its own recbuffer, + its own cap_records and its own expected size -- assert both blocks, + not just the record one. +*/ + +static void test_min_records_above_max_records_keyed(void) +{ + HP_SHARE *share; + HP_INFO *info; + HP_KEYDEF keydef; + HA_KEYSEG keyseg; + + init_int_keydef(&keydef, &keyseg); + + if (create_table("test_block_bnd_keyed", 1, REC_LENGTH, 100000000UL, + BOUNDARY_MAX_RECORDS, &keydef, &share)) + { + ok(0, "setup failed: %d", my_errno); + skip(2, "setup failed"); + return; + } + + ok(share->block.alloc_size == 134217696, + "record block clamped to the ceiling (got %zu)", + share->block.alloc_size); + + /* + 1M * sizeof(HASH_INFO) + extra, rounded up to the next power of two: + 32MB on LP64, 16MB on ILP32. The record block above rounds to the + same size on both, but halving the key record halves memory_needed, + which lands a whole power of two lower. + */ + ok(share->keydef[0].block.alloc_size == + (SIZEOF_CHARP == 8 ? 33554400 : 16777184), + "hash key block clamped to the ceiling (got %zu)", + share->keydef[0].block.alloc_size); + + if ((info= heap_open("test_block_bnd_keyed", 2))) + { + heap_drop_table(info); + heap_close(info); + } +} + + +/* + Test: max_records=0 means "no row limit", not "1000 rows". + + Block sizing needs a concrete row count, so heap_create() derives one + when the caller passes max_records=0. That derived value must not + reach share->max_records: hp_alloc_from_tail() only skips the row + limit check while max_records is 0, so storing the derived default + would turn an unlimited table into one that reports + HA_ERR_RECORD_FILE_FULL a few blocks in. +*/ + +static void test_no_max_records_is_unlimited(void) +{ + HP_SHARE *share; + HP_INFO *info; + uchar rec[REC_LENGTH]; + ulong i; + ulong write_failures= 0; + const ulong rows= 5000; + + if (create_table("test_block_nomax", 0, REC_LENGTH, 0, 0, NULL, &share)) + { + ok(0, "setup failed: %d", my_errno); + skip(3, "setup failed"); + return; + } + ok(1, "created keyless table with max_records=0"); + + ok(share->max_records == 0, + "max_records stays 0, meaning no row limit (got %lu)", + share->max_records); + + ok(share->block.alloc_size == 16352, + "block sized from the derived default (got %zu, expected 16352)", + share->block.alloc_size); + + if (!(info= heap_open("test_block_nomax", 2))) + { + ok(0, "heap_open failed: %d", my_errno); + skip(1, "open failed"); + return; + } + heap_extra(info, HA_EXTRA_NO_READCHECK); + + for (i= 0; i < rows; i++) + { + build_record(rec, (int32) i); + if (heap_write(info, rec)) + write_failures++; + } + ok(write_failures == 0, + "wrote %lu rows, well past the derived default, without error " + "(%lu failures)", rows, write_failures); + + heap_drop_table(info); + heap_close(info); +} + + int main(int argc __attribute__((unused)), char **argv __attribute__((unused))) { MY_INIT("hp_test_block_size"); - plan(24); + plan(34); diag("Test 1: ceiling-derived max_records capped (keyed table)"); test_huge_max_records_capped(); @@ -437,6 +613,15 @@ int main(int argc __attribute__((unused)), diag("Test 8: explicit min_records still overrides cap for wide rows"); test_explicit_min_records_wide_row(); + diag("Test 9: min_records / cap boundary walk"); + test_min_records_cap_boundary(); + + diag("Test 10: unreachable min_records clamps record and key blocks"); + test_min_records_above_max_records_keyed(); + + diag("Test 11: max_records=0 stays unlimited"); + test_no_max_records_is_unlimited(); + my_end(0); return exit_status(); }