diff --git a/bin/installcheck b/bin/installcheck index b54f6335..18e73ef7 100755 --- a/bin/installcheck +++ b/bin/installcheck @@ -30,7 +30,7 @@ rm -rf "$TMPDIR" # Initialize: setting PGUSER as the owner initdb --no-locale --encoding=UTF8 --nosync -U "$PGUSER" # Start the server -pg_ctl start -o "-F -c listen_addresses=\"\" -c log_min_messages=WARNING -k $PGDATA" +pg_ctl start -o "-F -c listen_addresses=\"\" -c log_min_messages=WARNING -c wal_level=logical -k $PGDATA" # Create the test db createdb contrib_regression @@ -52,7 +52,7 @@ else fi # Execute the test fixtures -psql -v ON_ERROR_STOP= -f test/fixtures.sql -f lints/0001*.sql -f lints/0002*.sql -f lints/0003*.sql -f lints/0004*.sql -f lints/0005*.sql -f lints/0006*.sql -f lints/0007*.sql -f lints/0008*.sql -f lints/0009*.sql -f lints/0010*.sql -f lints/0011*.sql -f lints/0013*.sql -f lints/0014*.sql -f lints/0015*.sql -f lints/0016*.sql -f lints/0017*.sql -f lints/0018*.sql -f lints/0019*.sql -f lints/0020*.sql -f lints/0021*.sql -f lints/0022*.sql -f lints/0023*.sql -f lints/0024*.sql -f lints/0025*.sql -f lints/0026*.sql -f lints/0027*.sql -f lints/0028*.sql -f lints/0029*.sql -d contrib_regression +psql -v ON_ERROR_STOP= -f test/fixtures.sql -f lints/0001*.sql -f lints/0002*.sql -f lints/0003*.sql -f lints/0004*.sql -f lints/0005*.sql -f lints/0006*.sql -f lints/0007*.sql -f lints/0008*.sql -f lints/0009*.sql -f lints/0010*.sql -f lints/0011*.sql -f lints/0013*.sql -f lints/0014*.sql -f lints/0015*.sql -f lints/0016*.sql -f lints/0017*.sql -f lints/0018*.sql -f lints/0019*.sql -f lints/0020*.sql -f lints/0021*.sql -f lints/0022*.sql -f lints/0023*.sql -f lints/0024*.sql -f lints/0025*.sql -f lints/0026*.sql -f lints/0027*.sql -f lints/0028*.sql -f lints/0029*.sql -f lints/0030*.sql -d contrib_regression # Run tests ${REGRESS} --use-existing --dbname=contrib_regression --inputdir=${TESTDIR} ${TESTS} diff --git a/docs/0030_unused_replication_slot.md b/docs/0030_unused_replication_slot.md new file mode 100644 index 00000000..76c04d15 --- /dev/null +++ b/docs/0030_unused_replication_slot.md @@ -0,0 +1,53 @@ +**Level:** WARN|ERROR + +**Summary:** Detects replication slots that are inactive and retaining WAL beyond `max_slot_wal_keep_size`. + +**Ramification:** A replication slot with no active consumer keeps every WAL segment since its `restart_lsn` on disk indefinitely. Left unattended, this can fill the primary's disk and cause an outage. + +--- + +### Rationale + +Postgres will never recycle WAL a replication slot still needs, even if nothing is reading from that slot anymore. This is normal and required for the slot to still be useful to a consumer that reconnects — but if the consumer (a read replica, a logical replication client, a CDC tool) is gone for good, the slot just accumulates WAL forever. + +Postgres itself tracks how close a slot is to actually causing harm via `pg_replication_slots.wal_status`: + +- `reserved` — normal, claimed WAL files are within `max_wal_size`. +- `extended` — `max_wal_size` is exceeded but the files are still retained (by the slot or by `wal_keep_size`). This is benign and can happen on perfectly healthy, currently-active slots (e.g. during a burst of write traffic) — it does not by itself indicate a problem. +- `unreserved` — the slot no longer retains its required WAL and some of it is due to be removed at the next checkpoint. This is what actually happens once retained WAL exceeds `max_slot_wal_keep_size`, and it's still recoverable (can return to `reserved`/`extended` if the consumer catches up before the next checkpoint). +- `lost` — the slot has been invalidated; the WAL it needed is already gone, and the slot can no longer be used to resume replication. + +This lint fires on `unreserved` or `lost`, not merely on `active = false` — a slot that's briefly inactive (e.g. its replica restarting) but still `reserved` (or even `extended`) is not yet a problem. + +### How to Resolve + +**Option 1: Drop the slot if its consumer is gone for good** + +```sql +select pg_drop_replication_slot(''); +``` + +**Option 2: If a consumer is expected to reconnect, investigate the disconnect** + +Check why the replica/consumer isn't connecting (network issue, instance down, credentials) and monitor disk usage on the primary in the meantime. Once it reconnects and catches up, `wal_status` returns to `reserved` on its own. + +### Example + +Given a physical replication slot whose replica was deleted weeks ago: + +```sql +select slot_name, active, wal_status from pg_replication_slots; +-- slot_name | active | wal_status +-- ---------------------+--------+------------ +-- replica_abandoned | f | unreserved +``` + +Fix: + +```sql +select pg_drop_replication_slot('replica_abandoned'); +``` + +### False Positives + +A slot that is `active = false` but still `wal_status = 'reserved'` or `'extended'` will not fire — this covers a replica restarting, being briefly taken offline for maintenance, or a currently-healthy slot that's simply using more than `max_wal_size` right now, without generating noise. If this lint fires, the slot has already exceeded `max_slot_wal_keep_size` (or been invalidated entirely). diff --git a/lints/0030_unused_replication_slot.sql b/lints/0030_unused_replication_slot.sql new file mode 100644 index 00000000..35b2d21c --- /dev/null +++ b/lints/0030_unused_replication_slot.sql @@ -0,0 +1,36 @@ +create view lint."0030_unused_replication_slot" as + +select + 'unused_replication_slot' as name, + 'Unused Replication Slot' as title, + -- 'lost' means the WAL is already gone and unrecoverable (ERROR). The only other status the WHERE clause admits, 'unreserved', can still self-heal before invalidation, so it's just a WARN. + case prs.wal_status + when 'lost' then 'ERROR' + else 'WARN' + end as level, + 'EXTERNAL' as facing, + array['PERFORMANCE'] as categories, + 'Detects replication slots that are inactive and at risk of exceeding max_slot_wal_keep_size, or already invalidated, risking disk bloat on the primary.' as description, + format( + 'Replication slot `%s` is inactive and its WAL retention status is `%s`', + prs.slot_name, + prs.wal_status + ) as detail, + 'https://supabase.com/docs/guides/database/database-linter?lint=0030_unused_replication_slot' as remediation, + jsonb_build_object( + 'name', prs.slot_name, + 'entity', prs.slot_name, + 'type', 'replication_slot', + 'slot_type', prs.slot_type, + 'wal_status', prs.wal_status, + 'plugin', prs.plugin + ) as metadata, + format('unused_replication_slot_%s', prs.slot_name) as cache_key +from + pg_catalog.pg_replication_slots prs +where + prs.active = false + -- 'reserved'/'extended' are still within retention limits, or a replica is reconnecting. Only 'unreserved' (limit already exceeded) and 'lost' (already invalidated) are worth flagging. + and prs.wal_status in ('unreserved', 'lost') +order by + prs.slot_name; diff --git a/splinter.sql b/splinter.sql index 9cd66c95..9b9cce50 100644 --- a/splinter.sql +++ b/splinter.sql @@ -1838,4 +1838,40 @@ from order by schema_name, function_name, - function_args) \ No newline at end of file + function_args) +union all +( +select + 'unused_replication_slot' as name, + 'Unused Replication Slot' as title, + -- 'lost' means the WAL is already gone and unrecoverable (ERROR). The only other status the WHERE clause admits, 'unreserved', can still self-heal before invalidation, so it's just a WARN. + case prs.wal_status + when 'lost' then 'ERROR' + else 'WARN' + end as level, + 'EXTERNAL' as facing, + array['PERFORMANCE'] as categories, + 'Detects replication slots that are inactive and at risk of exceeding max_slot_wal_keep_size, or already invalidated, risking disk bloat on the primary.' as description, + format( + 'Replication slot `%s` is inactive and its WAL retention status is `%s`', + prs.slot_name, + prs.wal_status + ) as detail, + 'https://supabase.com/docs/guides/database/database-linter?lint=0030_unused_replication_slot' as remediation, + jsonb_build_object( + 'name', prs.slot_name, + 'entity', prs.slot_name, + 'type', 'replication_slot', + 'slot_type', prs.slot_type, + 'wal_status', prs.wal_status, + 'plugin', prs.plugin + ) as metadata, + format('unused_replication_slot_%s', prs.slot_name) as cache_key +from + pg_catalog.pg_replication_slots prs +where + prs.active = false + -- 'reserved'/'extended' are still within retention limits, or a replica is reconnecting. Only 'unreserved' (limit already exceeded) and 'lost' (already invalidated) are worth flagging. + and prs.wal_status in ('unreserved', 'lost') +order by + prs.slot_name) \ No newline at end of file diff --git a/test/expected/0030_unused_replication_slot.out b/test/expected/0030_unused_replication_slot.out new file mode 100644 index 00000000..fc295cfa --- /dev/null +++ b/test/expected/0030_unused_replication_slot.out @@ -0,0 +1,113 @@ +-- Unlike other lint tests, this file cannot use begin/savepoint/rollback: replication slots are non-transactional (they survive a rollback) and ALTER SYSTEM is rejected inside a transaction block, so every statement here autocommits individually and cleanup is explicit instead. +set search_path = ''; +-- BASELINE: 0 issues, no slots exist +select * from lint."0030_unused_replication_slot"; + name | title | level | facing | categories | description | detail | remediation | metadata | cache_key +------+-------+-------+--------+------------+-------------+--------+-------------+----------+----------- +(0 rows) + +-- NEGATIVE EXAMPLE: a freshly created slot (LSN reserved, as a real replica connection would) is wal_status='reserved' while inactive -- e.g. a replica that just restarted -- and must NOT fire. +do $$ begin perform pg_catalog.pg_create_physical_replication_slot('splinter_test_negative_slot', true); end $$; +select name, detail, cache_key from lint."0030_unused_replication_slot"; -- expect 0 rows + name | detail | cache_key +------+--------+----------- +(0 rows) + +select pg_catalog.pg_drop_replication_slot('splinter_test_negative_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +-- NEGATIVE EXAMPLE (logical slot): the same reserved/inactive guarantee applies to logical slots, not just physical +do $$ begin perform pg_catalog.pg_create_logical_replication_slot('splinter_test_negative_logical_slot', 'test_decoding'); end $$; +select name, detail, cache_key from lint."0030_unused_replication_slot"; -- expect 0 rows + name | detail | cache_key +------+--------+----------- +(0 rows) + +select pg_catalog.pg_drop_replication_slot('splinter_test_negative_logical_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +-- NEGATIVE EXAMPLE (extended): max_wal_size exceeded but max_slot_wal_keep_size left at its default (disabled), so the slot is 'extended', not 'unreserved' -- must NOT fire +do $$ begin perform pg_catalog.pg_create_physical_replication_slot('splinter_test_extended_slot', true); end $$; +alter system set max_wal_size = '2MB'; +select pg_catalog.pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +-- pg_switch_wal() is a no-op once already sitting at a segment boundary, so each loop iteration writes one trivial WAL record first to move off the boundary before switching again. +do $$ begin for i in 1..10 loop perform pg_catalog.pg_logical_emit_message(false, 'splinter_test', 'advance'); perform pg_catalog.pg_switch_wal(); end loop; end $$; +select slot_name, wal_status from pg_catalog.pg_replication_slots where slot_name = 'splinter_test_extended_slot'; -- confirm wal_status is actually 'extended' + slot_name | wal_status +-----------------------------+------------ + splinter_test_extended_slot | extended +(1 row) + +select name, detail, cache_key from lint."0030_unused_replication_slot"; -- expect 0 rows + name | detail | cache_key +------+--------+----------- +(0 rows) + +select pg_catalog.pg_drop_replication_slot('splinter_test_extended_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +alter system reset max_wal_size; +select pg_catalog.pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +-- POSITIVE EXAMPLE: shrink max_slot_wal_keep_size and generate enough WAL past it so the slot's retained WAL exceeds the limit. +do $$ begin perform pg_catalog.pg_create_physical_replication_slot('splinter_test_positive_slot', true); end $$; +alter system set max_slot_wal_keep_size = '1MB'; +select pg_catalog.pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +do $$ begin perform pg_catalog.pg_logical_emit_message(false, 'splinter_test', 'advance'); perform pg_catalog.pg_switch_wal(); end $$; +-- before any checkpoint runs, the slot has exceeded max_slot_wal_keep_size but hasn't been invalidated yet -- wal_status is 'unreserved', level WARN +select name, detail, cache_key from lint."0030_unused_replication_slot"; -- expect 1 row, wal_status unreserved + name | detail | cache_key +-------------------------+---------------------------------------------------------------------------------------------------------+----------------------------------------------------- + unused_replication_slot | Replication slot `splinter_test_positive_slot` is inactive and its WAL retention status is `unreserved` | unused_replication_slot_splinter_test_positive_slot +(1 row) + +-- a checkpoint is what actually invalidates an 'unreserved' slot once its required WAL is gone -- wal_status becomes 'lost', level ERROR +checkpoint; +select name, detail, cache_key from lint."0030_unused_replication_slot"; -- expect 1 row, wal_status lost + name | detail | cache_key +-------------------------+---------------------------------------------------------------------------------------------------+----------------------------------------------------- + unused_replication_slot | Replication slot `splinter_test_positive_slot` is inactive and its WAL retention status is `lost` | unused_replication_slot_splinter_test_positive_slot +(1 row) + +-- dropping is the only way to stop WAL accumulation once the consumer is confirmed gone for good +select pg_catalog.pg_drop_replication_slot('splinter_test_positive_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +select * from lint."0030_unused_replication_slot"; -- expect 0 rows + name | title | level | facing | categories | description | detail | remediation | metadata | cache_key +------+-------+-------+--------+------------+-------------+--------+-------------+----------+----------- +(0 rows) + +alter system reset max_slot_wal_keep_size; +select pg_catalog.pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + diff --git a/test/expected/queries_are_unionable.out b/test/expected/queries_are_unionable.out index 3b1005d4..563d8ace 100644 --- a/test/expected/queries_are_unionable.out +++ b/test/expected/queries_are_unionable.out @@ -54,7 +54,9 @@ begin; union all select * from lint."0028_anon_security_definer_function_executable" union all - select * from lint."0029_authenticated_security_definer_function_executable"; + select * from lint."0029_authenticated_security_definer_function_executable" + union all + select * from lint."0030_unused_replication_slot"; name | title | level | facing | categories | description | detail | remediation | metadata | cache_key ------+-------+-------+--------+------------+-------------+--------+-------------+----------+----------- (0 rows) diff --git a/test/sql/0030_unused_replication_slot.sql b/test/sql/0030_unused_replication_slot.sql new file mode 100644 index 00000000..72518265 --- /dev/null +++ b/test/sql/0030_unused_replication_slot.sql @@ -0,0 +1,47 @@ +-- Unlike other lint tests, this file cannot use begin/savepoint/rollback: replication slots are non-transactional (they survive a rollback) and ALTER SYSTEM is rejected inside a transaction block, so every statement here autocommits individually and cleanup is explicit instead. +set search_path = ''; + +-- BASELINE: 0 issues, no slots exist +select * from lint."0030_unused_replication_slot"; + +-- NEGATIVE EXAMPLE: a freshly created slot (LSN reserved, as a real replica connection would) is wal_status='reserved' while inactive -- e.g. a replica that just restarted -- and must NOT fire. +do $$ begin perform pg_catalog.pg_create_physical_replication_slot('splinter_test_negative_slot', true); end $$; +select name, detail, cache_key from lint."0030_unused_replication_slot"; -- expect 0 rows +select pg_catalog.pg_drop_replication_slot('splinter_test_negative_slot'); + +-- NEGATIVE EXAMPLE (logical slot): the same reserved/inactive guarantee applies to logical slots, not just physical +do $$ begin perform pg_catalog.pg_create_logical_replication_slot('splinter_test_negative_logical_slot', 'test_decoding'); end $$; +select name, detail, cache_key from lint."0030_unused_replication_slot"; -- expect 0 rows +select pg_catalog.pg_drop_replication_slot('splinter_test_negative_logical_slot'); + +-- NEGATIVE EXAMPLE (extended): max_wal_size exceeded but max_slot_wal_keep_size left at its default (disabled), so the slot is 'extended', not 'unreserved' -- must NOT fire +do $$ begin perform pg_catalog.pg_create_physical_replication_slot('splinter_test_extended_slot', true); end $$; +alter system set max_wal_size = '2MB'; +select pg_catalog.pg_reload_conf(); +-- pg_switch_wal() is a no-op once already sitting at a segment boundary, so each loop iteration writes one trivial WAL record first to move off the boundary before switching again. +do $$ begin for i in 1..10 loop perform pg_catalog.pg_logical_emit_message(false, 'splinter_test', 'advance'); perform pg_catalog.pg_switch_wal(); end loop; end $$; +select slot_name, wal_status from pg_catalog.pg_replication_slots where slot_name = 'splinter_test_extended_slot'; -- confirm wal_status is actually 'extended' +select name, detail, cache_key from lint."0030_unused_replication_slot"; -- expect 0 rows +select pg_catalog.pg_drop_replication_slot('splinter_test_extended_slot'); +alter system reset max_wal_size; +select pg_catalog.pg_reload_conf(); + +-- POSITIVE EXAMPLE: shrink max_slot_wal_keep_size and generate enough WAL past it so the slot's retained WAL exceeds the limit. +do $$ begin perform pg_catalog.pg_create_physical_replication_slot('splinter_test_positive_slot', true); end $$; +alter system set max_slot_wal_keep_size = '1MB'; +select pg_catalog.pg_reload_conf(); +do $$ begin perform pg_catalog.pg_logical_emit_message(false, 'splinter_test', 'advance'); perform pg_catalog.pg_switch_wal(); end $$; + +-- before any checkpoint runs, the slot has exceeded max_slot_wal_keep_size but hasn't been invalidated yet -- wal_status is 'unreserved', level WARN +select name, detail, cache_key from lint."0030_unused_replication_slot"; -- expect 1 row, wal_status unreserved + +-- a checkpoint is what actually invalidates an 'unreserved' slot once its required WAL is gone -- wal_status becomes 'lost', level ERROR +checkpoint; +select name, detail, cache_key from lint."0030_unused_replication_slot"; -- expect 1 row, wal_status lost + +-- dropping is the only way to stop WAL accumulation once the consumer is confirmed gone for good +select pg_catalog.pg_drop_replication_slot('splinter_test_positive_slot'); +select * from lint."0030_unused_replication_slot"; -- expect 0 rows + +alter system reset max_slot_wal_keep_size; +select pg_catalog.pg_reload_conf(); diff --git a/test/sql/queries_are_unionable.sql b/test/sql/queries_are_unionable.sql index fe43d3da..70393d42 100644 --- a/test/sql/queries_are_unionable.sql +++ b/test/sql/queries_are_unionable.sql @@ -56,6 +56,8 @@ begin; union all select * from lint."0028_anon_security_definer_function_executable" union all - select * from lint."0029_authenticated_security_definer_function_executable"; + select * from lint."0029_authenticated_security_definer_function_executable" + union all + select * from lint."0030_unused_replication_slot"; rollback;