Repository navigation
nbtree: opt-in leaf key prefix compression #53
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -50,6 +50,7 @@ tests += { | |
| 't/004_verify_nbtree_unique.pl', | ||
| 't/005_pitr.pl', | ||
| 't/006_verify_gin.pl', | ||
| 't/007_prefix.pl', | ||
| ], | ||
| }, | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -183,6 +183,35 @@ INSERT INTO varlena_bug VALUES (repeat('Test', 250)); | |
| ALTER TABLE varlena_bug ALTER COLUMN v SET STORAGE extended; | ||
| SELECT bt_index_check('varlena_bug_idx', true); | ||
|
|
||
| -- Check a prefix-compressed (prefix_compression=on) index: the data items are | ||
| -- stored coded against a per-page PREFIX item, so amcheck must decode them and | ||
| -- validate the BTP_PREFIX structural invariants. Use keys with a long shared | ||
| -- leading prefix (so pages actually compress) plus some that share nothing and | ||
| -- a NULL (stored uncoded), and enough rows to force splits and deduplication. | ||
| CREATE TABLE prefix_bug (v text COLLATE "C"); | ||
| INSERT INTO prefix_bug | ||
| SELECT 'https://example.com/path/to/resource/' || lpad(g::text, 8, '0') | ||
| FROM generate_series(1, 20000) g; | ||
| INSERT INTO prefix_bug | ||
| SELECT 'https://example.com/path/to/resource/' || lpad((g % 100)::text, 8, '0') | ||
| FROM generate_series(1, 20000) g; | ||
|
Comment on lines
+192
to
+197
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 40,000 inserts (two 20k generate_series) is a heavy addition to the amcheck regression suite's runtime. Prefix compression only kicks in on full-page splits, so a large dataset is needed to reach the compressed-page paths, but it's worth confirming on the -hackers thread whether this volume is the minimum required to reliably force splits + dedup + BTP_PREFIX formation (narrower keys / a tighter fill would reduce the row count needed). Low-severity; flagging because regression-suite cost is a common review point. |
||
| INSERT INTO prefix_bug VALUES ('zzz-no-shared-prefix'), (NULL); | ||
| CREATE INDEX prefix_bug_idx ON prefix_bug (v) WITH (prefix_compression = on); | ||
|
Comment on lines
+198
to
+199
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The new prefix-compression test index is non-unique, so it never exercises |
||
| SELECT bt_index_check('prefix_bug_idx', heapallindexed => true); | ||
| SELECT bt_index_parent_check('prefix_bug_idx', heapallindexed => true, rootdescend => true); | ||
|
Comment on lines
+199
to
+201
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Test-coverage gap: this exercises only a non-unique prefix-compressed index. The uniqueness-checking path in verify_nbtree.c (gated on state->indexinfo->ii_Unique) is never reached here, yet that path is where prefix decoding interacts most dangerously with amcheck's cross-row state (lVis retains a pointer into the per-item BTItemBuf decode workspace). Add a UNIQUE prefix-compressed index case -- including adjacent equal keys / posting lists on the same leaf -- so bt_entry_unique_check()/bt_report_duplicate() run against decoded items. A healthy-index check will not surface a wrong-TID report, so also consider a corruption-injection or at least a duplicate-key scenario that drives the duplicate-reporting branch. |
||
|
|
||
| -- The final leaf's decoded split budget includes its last posting tuple, | ||
| -- not the smaller high key that would replace it on a non-rightmost leaf. | ||
| DROP INDEX prefix_bug_idx; | ||
| TRUNCATE prefix_bug; | ||
| INSERT INTO prefix_bug | ||
| SELECT repeat('x', 115) || lpad(g::text, 4, '0') | ||
| FROM generate_series(1, 106) g; | ||
| INSERT INTO prefix_bug | ||
| SELECT repeat('x', 115) || '0107' FROM generate_series(1, 113); | ||
| CREATE INDEX prefix_bug_idx ON prefix_bug (v) WITH (prefix_compression = on); | ||
| SELECT bt_index_parent_check('prefix_bug_idx', heapallindexed => true, rootdescend => true); | ||
|
|
||
| -- cleanup | ||
| DROP TABLE bttest_a; | ||
| DROP TABLE bttest_b; | ||
|
|
@@ -194,3 +223,4 @@ DROP TABLE bttest_unique_nulls; | |
| DROP OWNED BY regress_bttest_role; -- permissions | ||
| DROP ROLE regress_bttest_role; | ||
| DROP TABLE varlena_bug; | ||
| DROP TABLE prefix_bug; | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,253 @@ | ||
| # Copyright (c) 2026, PostgreSQL Global Development Group | ||
|
|
||
| # Check prefix metadata before amcheck decodes tuples or recounts their sizes. | ||
| use strict; | ||
| use warnings FATAL => 'all'; | ||
|
|
||
| use PostgreSQL::Test::Cluster; | ||
| use PostgreSQL::Test::Utils; | ||
| use Test::More; | ||
|
|
||
| my $node = PostgreSQL::Test::Cluster->new('prefix'); | ||
| $node->init(no_data_checksums => 1); | ||
| $node->append_conf('postgresql.conf', 'autovacuum = off'); | ||
| $node->start; | ||
| $node->safe_psql( | ||
| 'postgres', q( | ||
| CREATE EXTENSION amcheck; | ||
| CREATE TABLE prefix_data (u text COLLATE "C"); | ||
| INSERT INTO prefix_data | ||
| SELECT repeat('prefix/', 12) || lpad(g::text, 4, '0') | ||
| FROM generate_series(1, 20) g; | ||
| CREATE TABLE prefix_posting AS | ||
| SELECT u FROM prefix_data CROSS JOIN generate_series(1, 4); | ||
| )); | ||
| my $blksize = $node->safe_psql('postgres', 'SHOW block_size'); | ||
| my $align = $node->safe_psql('postgres', | ||
| 'SELECT max_data_alignment FROM pg_control_init()'); | ||
| my $little_endian = pack('L', 1) eq "\x01\x00\x00\x00"; | ||
|
|
||
| # ItemIdData is a native-endian bitfield; tuple metadata uses native uint16. | ||
| sub itemid | ||
| { | ||
| my ($page, $offnum) = @_; | ||
| my $lp = unpack('L', substr($page, 24 + 4 * ($offnum - 1), 4)); | ||
| return $little_endian | ||
| ? ($lp & 0x7fff, $lp >> 17) | ||
| : ($lp >> 17, $lp & 0x7fff); | ||
| } | ||
|
|
||
| sub put_short | ||
| { | ||
| my ($page, $offset, $value) = @_; | ||
| substr($$page, $offset, 2) = pack('S', $value); | ||
| return; | ||
| } | ||
|
|
||
| # Each case changes one field in its own index. Include a later member so a | ||
| # comparison with the next item cannot decode it before it has been checked. | ||
| my @cases = ( | ||
| [ | ||
| 'group', 0, | ||
| sub { my ($p, $d, $m) = @_; put_short($p, $m + 8, 65535); } | ||
| ], | ||
| [ | ||
| 'shared length', | ||
| 0, sub { my ($p, $d, $m) = @_; put_short($p, $m + 10, 65535); } | ||
| ], | ||
| [ | ||
| 'original size', | ||
| 0, sub { my ($p, $d, $m) = @_; put_short($p, $m + 12, 0); } | ||
| ], | ||
| [ | ||
| 'residual extent', | ||
| 0, sub { my ($p, $d, $m) = @_; put_short($p, $m + 10, 0); } | ||
| ], | ||
| [ | ||
| 'directory group count', | ||
| 0, sub { my ($p, $d) = @_; put_short($p, $d + 8, 65535); } | ||
| ], | ||
| [ | ||
| 'anchor offset', | ||
| 0, sub { my ($p, $d) = @_; put_short($p, $d + 14, 65535); } | ||
| ], | ||
| [ | ||
| 'anchor size', | ||
| 0, | ||
| sub { | ||
| my ($p, $d) = @_; | ||
| my $a = unpack('S', substr($$p, $d + 14, 2)); | ||
| put_short($p, $d + $a, 65535); | ||
| } | ||
| ], | ||
| [ | ||
| 'anchor tuple size', | ||
| 0, | ||
| sub { | ||
| my ($p, $d) = @_; | ||
| my $a = unpack('S', substr($$p, $d + 14, 2)); | ||
| put_short($p, $d + $a + $align + 6, 0); | ||
| } | ||
| ], | ||
| [ | ||
| 'decoded size cache', | ||
| 0, sub { my ($p, $d) = @_; put_short($p, $d + 10, 0); } | ||
| ], | ||
| [ | ||
| 'short code header', | ||
| 0, | ||
| sub { | ||
| my ($p, $d, $m) = @_; | ||
| my $lp = unpack('L', substr($$p, 32, 4)); | ||
| $lp = | ||
| $little_endian | ||
| ? (($lp & 0x1ffff) | (8 << 17)) | ||
| : (($lp & 0xffff8000) | 8); | ||
| substr($$p, 32, 4) = pack('L', $lp); | ||
| put_short($p, $m + 6, 8); | ||
| } | ||
| ], | ||
| [ | ||
| 'posting offset', | ||
| 1, | ||
| sub { | ||
| my ($p, $d, $m) = @_; | ||
| put_short($p, $m, 0); | ||
| put_short($p, $m + 2, 65535); | ||
| } | ||
| ], | ||
| [ | ||
| 'posting count', | ||
| 1, | ||
| sub { | ||
| my ($p, $d, $m) = @_; | ||
| put_short($p, $m + 4, 0x2000 | 0x0fff); | ||
| } | ||
| ], | ||
| [ | ||
| 'pivot member', | ||
| 0, | ||
| sub { | ||
| my ($p) = @_; | ||
| my $maxoff = (unpack('S', substr($$p, 12, 2)) - 24) / 4; | ||
| my ($m) = itemid($$p, 2 + int(($maxoff - 1) / 2)); | ||
| my $info = unpack('S', substr($$p, $m + 6, 2)); | ||
| put_short($p, $m + 6, $info | 0x2000); | ||
| } | ||
| ], | ||
| [ | ||
| 'member line pointer', | ||
| 0, | ||
| sub { | ||
| my ($p) = @_; | ||
| my $maxoff = (unpack('S', substr($$p, 12, 2)) - 24) / 4; | ||
| my $lp_off = 24 + 4 * (1 + int(($maxoff - 1) / 2)); | ||
| my $lp = unpack('L', substr($$p, $lp_off, 4)); | ||
| $lp = $little_endian | ||
| ? (($lp & 0xffff8000) | 32760) | ||
| : (($lp & 0x1ffff) | (32760 << 17)); | ||
| substr($$p, $lp_off, 4) = pack('L', $lp); | ||
| } | ||
| ],); | ||
|
|
||
| my @paths; | ||
| for my $i (0 .. $#cases) | ||
| { | ||
| my $table = $cases[$i][1] ? 'prefix_posting' : 'prefix_data'; | ||
| $node->safe_psql('postgres', | ||
| "CREATE TABLE prefix_case_$i AS TABLE $table; | ||
| CREATE INDEX prefix_idx_$i ON prefix_case_$i (u) | ||
| WITH (prefix_compression = on)" | ||
| ); | ||
| is( $node->safe_psql( | ||
| 'postgres', | ||
| "SELECT bt_index_parent_check('prefix_idx_$i', true, true)"), | ||
| '', | ||
| "$cases[$i][0]: clean index passes"); | ||
| push @paths, | ||
| $node->data_dir . '/' | ||
| . $node->safe_psql('postgres', | ||
| "SELECT pg_relation_filepath('prefix_idx_$i')"); | ||
| } | ||
|
|
||
| # Stop before opening relation files, avoiding stale buffers and background | ||
| # writes. Checksums are disabled so the errors come from prefix validation. | ||
| $node->stop; | ||
| for my $i (0 .. $#cases) | ||
| { | ||
| open(my $fh, '+<', $paths[$i]) or BAIL_OUT("open: $!"); | ||
| binmode $fh; | ||
| defined(sysseek($fh, $blksize, 0)) or BAIL_OUT("seek: $!"); | ||
| my $page; | ||
| sysread($fh, $page, $blksize) == $blksize or BAIL_OUT("read: $!"); | ||
| my $special = unpack('S', substr($page, 16, 2)); | ||
| my $flags = unpack('S', substr($page, $special + 12, 2)); | ||
| ($flags & 0x201) == 0x201 or BAIL_OUT('fixture is not a coded leaf'); | ||
| unpack('L', substr($page, $special + 4, 4)) == 0 | ||
| or BAIL_OUT('fixture leaf is not rightmost'); | ||
| my ($directory) = itemid($page, 1); | ||
| my ($member, $membersz) = itemid($page, 3); | ||
| my $info = unpack('S', substr($page, $member + 6, 2)); | ||
| ($info & 0x1fff) == $membersz or BAIL_OUT('fixture tuple size mismatch'); | ||
|
|
||
| if ($cases[$i][1]) | ||
| { | ||
| ($info & 0x2000) | ||
| && (unpack('S', substr($page, $member + 4, 2)) & 0x2000) | ||
| or BAIL_OUT('fixture member is not a posting tuple'); | ||
| } | ||
| $cases[$i][2]->(\$page, $directory, $member); | ||
| defined(sysseek($fh, $blksize, 0)) or BAIL_OUT("seek: $!"); | ||
| syswrite($fh, $page) == $blksize or BAIL_OUT("write: $!"); | ||
| close($fh) or BAIL_OUT("close: $!"); | ||
| } | ||
| $node->start; | ||
|
|
||
| for my $i (0 .. $#cases) | ||
| { | ||
| for my $check ('bt_index_check', 'bt_index_parent_check') | ||
| { | ||
| my ($ret, $out, $err) = $node->psql('postgres', | ||
| "\\set VERBOSITY verbose\nSELECT $check('prefix_idx_$i');"); | ||
| is($ret, 3, "$cases[$i][0]: $check reports an error"); | ||
| like($err, qr/ERROR:\s+XX002:.*prefix/i, | ||
| "$cases[$i][0]: prefix corruption diagnostic"); | ||
| unlike( | ||
| $err, | ||
| qr/server closed the connection|terminated by signal/i, | ||
| "$cases[$i][0]: no backend crash"); | ||
| } | ||
| } | ||
|
|
||
| # Each table has just one index. Exercise the shared decoder without the | ||
| # whole-page validation performed by amcheck. The new header cases damage | ||
| # the first binary-search midpoint, before any member can be decoded. | ||
| my $scan_settings = q( | ||
| SET enable_seqscan = off; | ||
| SET enable_bitmapscan = off; | ||
| SET enable_indexonlyscan = off; | ||
| ); | ||
| for my $i (0 .. $#cases) | ||
| { | ||
| next unless $cases[$i][0] =~ | ||
| /^(group|shared length|short code header|posting offset|posting count|pivot member|member line pointer)$/; | ||
| my $query = "SELECT u FROM prefix_case_$i | ||
| WHERE u >= repeat('prefix/', 12) || '0001'"; | ||
| my $plan = $node->safe_psql('postgres', | ||
| "$scan_settings EXPLAIN (COSTS OFF) $query"); | ||
| like($plan, qr/Index Scan using prefix_idx_$i on prefix_case_$i\b/, | ||
| "$cases[$i][0]: direct index scan plan"); | ||
| note($plan); | ||
| my ($ret, $out, $err) = $node->psql('postgres', | ||
| "\\set VERBOSITY verbose\n$scan_settings $query"); | ||
| is($ret, 3, "$cases[$i][0]: direct scan reports an error"); | ||
| like($err, qr/ERROR:\s+XX002:.*prefix/i, | ||
| "$cases[$i][0]: direct scan prefix diagnostic"); | ||
| unlike($err, qr/server closed the connection|terminated by signal/i, | ||
| "$cases[$i][0]: direct scan does not crash"); | ||
| } | ||
|
|
||
| is($node->safe_psql('postgres', 'SELECT 1'), | ||
| '1', 'node remains usable after corruption checks'); | ||
| $node->stop; | ||
| done_testing(); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The prefix check inserts 40,000 rows and then runs both a
heapallindexedbt_index_checkand abt_index_parent_checkwithrootdescend => true.rootdescendperforms a full root-to-leaf descent per leaf tuple, making this by far the most expensive check in this file (the existing 100k-row checks onbttest_a/bdo not userootdescend). Confirm 20,000+20,000 is the minimum needed to force splits + dedup + prefix coding; if a smaller count still exercises multi-level splits and posting-list dedup, prefer it to keepmake check/buildfarm time down. (low confidence on the exact threshold)