diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 356d2d9..6a5fca7 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -22,6 +22,10 @@ jobs: spock_ref: main runs-on: ${{ matrix.os }} + # Without this a hung tester runs until the 6 hour job ceiling, on every + # matrix entry, and the run is cancelled rather than failed -- which skips + # the diagnostic steps below and leaves nothing to debug with. + timeout-minutes: 45 steps: - name: Checkout lolor @@ -115,7 +119,36 @@ jobs: id: test_step continue-on-error: true run: | + # The tester exits when run-tests.sh finishes. Cap the wait so that a + # test that blocks -- on a lock, or on spock.sub_wait_for_sync() -- + # fails the step while the cluster is still up to be inspected. + deadline=$(( SECONDS + 25 * 60 )) while [ "$(docker inspect -f '{{.State.Running}}' tester)" == "true" ]; do + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::Tests did not finish within 25 minutes; dumping cluster state" + PGV=${{ matrix.pgver }} + echo "===== tester output so far =====" + tail -n 200 tests/out.txt || true + for n in n1 n2 n3; do + echo "===== $n: sessions and what they are waiting on =====" + docker exec "$n" bash -c "source /home/pgedge/pgedge/pg${PGV}/pg${PGV}.env && psql -U admin -d demo -x -c \" + SELECT pid, state, wait_event_type, wait_event, now() - xact_start AS xact_age, + left(query, 400) AS query + FROM pg_stat_activity + WHERE datname = 'demo' AND pid <> pg_backend_pid() + ORDER BY xact_start\"" || true + echo "===== $n: ungranted locks =====" + docker exec "$n" bash -c "source /home/pgedge/pgedge/pg${PGV}/pg${PGV}.env && psql -U admin -d demo -c \" + SELECT l.pid, l.locktype, l.mode, l.granted, + coalesce(c.relname, l.classid::text) AS object + FROM pg_locks l LEFT JOIN pg_class c ON c.oid = l.relation + WHERE NOT l.granted OR l.mode LIKE 'Share%' OR l.mode LIKE '%Exclusive%' + ORDER BY l.granted, l.pid\"" || true + echo "===== $n: subscription status =====" + docker exec "$n" bash -c "source /home/pgedge/pgedge/pg${PGV}/pg${PGV}.env && psql -U admin -d demo -c 'SELECT * FROM spock.sub_show_status()'" || true + done + exit 1 + fi echo "Waiting for tests to complete..." sleep 1 done @@ -124,6 +157,17 @@ jobs: docker logs n3 grep -Eq "FAIL|ERROR" tests/out.txt && exit 1 || exit 0 + - name: Dump container logs when the tests failed + if: steps.test_step.outcome == 'failure' + run: | + cd docker + for c in n1 n2 n3 tester; do + echo "===== $c =====" + docker logs "$c" > /tmp/$c.log 2>&1 || true + echo "--- first 120 lines (setup) ---"; head -n 120 /tmp/$c.log || true + echo "--- last 120 lines ---"; tail -n 120 /tmp/$c.log || true + done + - name: Upload Log File as Artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: diff --git a/.gitignore b/.gitignore index 4c75984..228ad22 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ *.so *.bc *.o +*.dylib +results/ +regression.diffs +regression.out +tmp_check/ +log/ +delete_old_cluster.sh diff --git a/Makefile b/Makefile index caa2ca0..94412aa 100644 --- a/Makefile +++ b/Makefile @@ -5,10 +5,11 @@ MODULE_big = lolor EXTENSION = lolor DATA = lolor--1.0.sql \ lolor--1.0--1.2.1.sql lolor--1.2.1--1.2.2.sql \ - lolor--1.2.2--1.3.0.sql + lolor--1.2.2--1.3.0.sql lolor--1.3.0--1.4.0.sql PGFILEDESC = "lolor - drop in large objects replacement for logical replication" -OBJS = src/lolor.o src/lolor_fsstubs.o src/lolor_inv_api.o src/lolor_largeobject.o +OBJS = src/lolor.o src/lolor_fsstubs.o src/lolor_inv_api.o src/lolor_largeobject.o \ + src/lolor_migrate.o REGRESS = lolor TAP_TESTS = 1 diff --git a/README.md b/README.md index b08bec5..fec5de9 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,24 @@ SELECT lolor.migrate_to_native(); -- manual DROP EXTENSION lolor; -- automatic ``` -Both directions preserve original OIDs, owners, ACLs, and data. +Both directions preserve original OIDs, owners, ACLs, comments and data, and +copy pages verbatim so sparse large objects stay sparse. Moving an object to +native storage also reinstates its `pg_shdepend` entries, so `DROP ROLE`, +`REASSIGN OWNED` and `DROP OWNED` continue to see it. + +`migrate_to_native()` does not require lolor to be enabled; it operates on the +catalogs directly. + +Security labels on large objects cannot be represented in lolor storage and +cannot be reinstated without their label provider, so `migrate_from_native()` +refuses rather than discarding them. Remove them first if you hit this. + +Two helpers support verifying a migration: + +```sql +SELECT * FROM lolor.digest(); -- per-object checksum, compare across nodes +SELECT * FROM lolor.check_orphans(); -- lolor objects whose owner no longer exists +``` When the spock extension is installed, both migration functions run under `spock.repair_mode()`, so the row-shuffling migration DML is **not** @@ -101,6 +118,7 @@ Even with spock, migration is refused if a non-spock logical replication slot own output plugin and those consumers would still decode the migration DML: `migrate_to_native()` raises an `ERROR`, while `migrate_from_native()` warns and returns -1 without doing anything. Drop the offending slots before migrating. +The check identifies spock's slots by their `spock_output` plugin. Without spock, the migration DML cannot be excluded from logical decoding, so both functions refuse to migrate while logical replication slots exist in the @@ -111,8 +129,19 @@ doing anything (0 is reserved for "nothing to migrate"), while migrating; merely disabling a subscription is not sufficient, since its slot retains the changes and delivers them when replication resumes. +### Security + +`lo_import()` and `lo_export()` read and write files on the server host. As in +core PostgreSQL, `EXECUTE` on them is revoked from `PUBLIC`; grant it +deliberately if a non-superuser needs server-side file access. + +Versions 1.0 through 1.3.0 left these two functions executable by every +database user. Upgrading to 1.4.0 revokes the privilege; see the release notes. + ### Limitations - Native large object functionality cannot be used while you are using the lolor extension. -- lolor does not support the following statements: `ALTER LARGE OBJECT`, `GRANT ON LARGE OBJECT`, `COMMENT ON LARGE OBJECT`, and `REVOKE ON LARGE OBJECT`. -- Large object migration is node-local. Native large objects live in `pg_catalog.pg_largeobject`, which is never replicated, so each node holds an independent set and `migrate_from_native()` migrates only the local node's objects; with spock installed, the migration DML runs in repair mode and is not replicated. Run the migration on every node that holds native large objects — for example with `spock.replicate_ddl('SELECT lolor.migrate_from_native()')`, which queues the command so that each node executes it locally. Migrated objects keep their original native OIDs, which are not node-encoded: if different nodes hold different objects under the same OID, the nodes' lolor contents will diverge and later replicated changes to those objects can conflict. Newly created large objects are collision-free, since new OIDs are node-encoded via `lolor.node` and checked against existing rows. +- lolor does not support the following statements against objects held in lolor storage: `ALTER LARGE OBJECT`, `GRANT ON LARGE OBJECT`, `COMMENT ON LARGE OBJECT`, and `REVOKE ON LARGE OBJECT`. Owners, ACLs and comments set while an object was in native storage are preserved across migration in both directions. +- Objects in lolor storage are rows in ordinary tables and so cannot participate in `pg_shdepend`: `DROP ROLE` will not notice that a role still owns them, the way it does for native large objects. Use `lolor.check_orphans()` to find objects whose owner has been dropped. +- `lolor.enable()` and `lolor.disable()` change which function OID owns each `pg_catalog.lo_*` name. libpq resolves those OIDs once per connection and caches them, so existing client sessions must reconnect afterwards. +- Large object migration is node-local. Native large objects live in `pg_catalog.pg_largeobject`, which is never replicated, so each node holds an independent set and `migrate_from_native()` migrates only the local node's objects; with spock installed, the migration DML runs in repair mode and is not replicated. Run the migration on every node that holds native large objects — for example with `spock.replicate_ddl('SELECT lolor.migrate_from_native()')`, which queues the command so that each node executes it locally. Migrated objects keep their original native OIDs, which are not node-encoded: if different nodes hold different objects under the same OID, the nodes' lolor contents will diverge and later replicated changes to those objects can conflict. Newly created large objects are collision-free, since new OIDs are node-encoded via `lolor.node` and checked against existing rows. To make that hazard an error rather than silent divergence, collect the other nodes' OIDs with `lolor.native_lo_oids()` and pass them in: `SELECT lolor.migrate_from_native(peer_oids => ARRAY[...])` refuses when any of them collide. After migrating every node, compare `lolor.digest()` across nodes to confirm they converged. diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index ac12080..330c412 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -9,8 +9,20 @@ echo ". /home/pgedge/pgedge/pg${PG_VER}/pg${PG_VER}.env" >> /home/pgedge/.bashrc # deprecated pgedge CLI. initdb -D "$PGDATA" -U admin --encoding=UTF8 --locale=C +# PostgreSQL 16.15, 17.11, 18.6 and 19 only allow logical decoding through an +# output plugin named in output_plugin_libraries, which defaults to +# 'pgoutput, test_decoding'. spock's plugin is not in that list, so +# CREATE_REPLICATION_SLOT fails, every apply worker exits and restarts, and no +# subscription ever reaches sync. Older minors do not have the parameter and +# refuse to start when it appears in the configuration, so ask the binary. +OUTPUT_PLUGIN_LIBRARIES="" +if postgres --describe-config 2>/dev/null | grep -q '^output_plugin_libraries'; then + OUTPUT_PLUGIN_LIBRARIES="output_plugin_libraries = 'pgoutput, test_decoding, spock_output'" +fi + cat >> "$PGDATA/postgresql.conf" <<_EOF_ listen_addresses = '*' +$OUTPUT_PLUGIN_LIBRARIES wal_level = logical track_commit_timestamp = on max_worker_processes = 32 @@ -33,6 +45,7 @@ while ! pg_isready -h /tmp; do sleep 1 done + # The admin user is what the tests connect as; the pgedge user is used for # the spock node and subscription DSNs (and matches the OS user, so plain # psql on the nodes works). @@ -51,20 +64,30 @@ _EOF_ IFS=',' read -r -a peer_names <<< "$PEER_NAMES" +# Bounded: an unbounded wait here leaves the container alive with the +# temporary postgres still listening, so pg_isready and the health check keep +# succeeding while setup never finishes. A node stuck in that state looks +# healthy to the test harness, which then blocks with no indication of why. for PEER_HOSTNAME in "${peer_names[@]}"; do - while : + peer_ready=0 + for attempt in $(seq 1 300); do + mapfile -t node_array < <(psql -A -t demo -h $PEER_HOSTNAME -c "SELECT node_name FROM spock.node;") + for element in "${node_array[@]}"; do - mapfile -t node_array < <(psql -A -t demo -h $PEER_HOSTNAME -c "SELECT node_name FROM spock.node;") - for element in "${node_array[@]}"; - do - if [[ "$element" == "$PEER_HOSTNAME" ]]; then - break 2 - fi - done - sleep 1 - echo "Waiting for $PEER_HOSTNAME..." + if [[ "$element" == "$PEER_HOSTNAME" ]]; then + peer_ready=1 + break + fi done + [ "$peer_ready" = "1" ] && break + sleep 1 + echo "Waiting for $PEER_HOSTNAME..." + done + if [ "$peer_ready" != "1" ]; then + echo "ERROR: peer $PEER_HOSTNAME did not register a spock node within 300s" >&2 + exit 1 + fi done # spock.sub_create connects to the provider synchronously, and the peer diff --git a/docs/lolor_release_notes.md b/docs/lolor_release_notes.md index 1373901..369dc4e 100644 --- a/docs/lolor_release_notes.md +++ b/docs/lolor_release_notes.md @@ -1,5 +1,26 @@ # lolor Release Notes +## lolor 1.4.0 + +* Rewrote large object migration between native and lolor storage as a direct relation-to-relation copy in C (`lolor.migrate_storage()`), replacing the previous reverse-migration loop that rewrote every object through the large object API. Fidelity and correctness fixes that follow from it: + * **Ownership and ACLs are now recorded in `pg_shdepend`.** Through 1.3.0 the reverse migration set `lomowner` with a raw catalog `UPDATE`, which left no shared dependency: `DROP ROLE` would succeed on a role that still owned migrated large objects, `REASSIGN OWNED` skipped them, and `DROP OWNED BY` the migrating superuser could delete other users' objects. + * **Comments are preserved.** `COMMENT ON LARGE OBJECT` text was silently discarded on migration. It is now parked in `lolor.pg_largeobject_description` while the object is in lolor storage and reinstated on the way back. + * **Sparse large objects stay sparse.** Pages are copied verbatim instead of being rewritten through the write API, which previously zero-filled every gap — a sparse 10 MB object materialised as 10 MB of pages. + * Native objects are now removed through `performMultipleDeletions()`, the same path `DROP` uses, so shared dependencies, comments and security labels are cleaned up rather than left behind. + * Migration no longer routes through the renamed `_orig` functions, so `migrate_to_native()` (and `DROP EXTENSION`) works whether or not lolor is enabled. This resolves the previous "lolor must be enabled before migration to native" failure. + * Storage layouts are verified against the running server's catalogs at migration time, so a future PostgreSQL catalog change produces a clear error instead of silent corruption. + * `migrate_from_native()` refuses rather than discarding large object security labels, which lolor storage cannot represent. + * OID conflicts are reported with the conflicting OIDs instead of only their existence. +* **Security fix: `lo_import()` and `lo_export()` were executable by any database user.** These functions read and write files on the server as the operating system account PostgreSQL runs under, and core revokes `EXECUTE` on them from `PUBLIC`. lolor replaces them by renaming the originals to `*_orig`; an ACL belongs to a function rather than to a name, so the restriction stayed behind on the parked original while each replacement was created with the default of `EXECUTE TO PUBLIC`. Any user could therefore read an arbitrary server file with `lo_import()` or overwrite one with `lo_export()`. The replacements are now locked down at install time, and the upgrade to 1.4.0 revokes the privilege on existing installations in either the enabled or the disabled state. All versions from 1.0 through 1.3.0 are affected. +* Fixed the `lolor.node` upper bound. The GUC accepted 0..16 while a generated OID reserves only four bits for the node id, so node 16 did not fit and was silently encoded as node 0. The bound is now derived from the encoding (`LOLOR_MAX_NODE_ID`), giving a valid range of 0..15; a configuration using node 16 will now be rejected rather than mis-encoding its OIDs. +* Migration now holds `ShareRowExclusiveLock` on both stores until the transaction commits. Ordinary large object reads and writes take `RowExclusiveLock`, which does not conflict with itself, so a concurrent `lo_write()` could previously commit between the point where the migration copied a page and the point where it emptied the source store — losing the write with no error. Two migrations running in opposite directions also now take their locks in a fixed order and cannot deadlock against each other. +* Cleanup now runs for every spelling of the drop. `DROP SCHEMA lolor CASCADE` and `DROP OWNED BY` reach the extension by dependency cascade rather than as `DROP EXTENSION`; the event trigger did not fire for them, so the large objects were destroyed along with the lolor tables and `pg_catalog` was left without a working `lo_open()`. +* `lolor.enable()`, `lolor.disable()` and `lolor.is_enabled()` now probe exact function signatures in `pg_catalog`. They previously matched on `proname` across every schema, so any user with `CREATE` on any schema could create a function named `lolor_lo_open` and permanently wedge lolor into an "inconsistent state" — which also blocked `DROP EXTENSION`. Both functions now also take an advisory lock so concurrent calls serialise. +* The extension is no longer marked `trusted`. Installing lolor renames functions in `pg_catalog` for the whole database, which is not an operation a non-superuser should be able to perform. +* New helpers: `lolor.digest()` for comparing lolor storage across nodes after a node-local migration, `lolor.check_orphans()` for finding objects whose owner has been dropped, and `lolor.native_lo_oids()` plus a `peer_oids` argument to `lolor.migrate_from_native()` that turns the documented cross-node OID collision hazard into a pre-flight refusal. +* `lolor.enable()` and `lolor.disable()` now emit a notice that client sessions must reconnect, since libpq caches the large object function OIDs per connection. +* Regression tests added for sparse objects, comment round trips, `pg_shdepend` restoration, `DROP SCHEMA CASCADE` cleanup, dropping the extension while disabled, and name-squatting resistance. The committed expected output was stale and has been regenerated. + ## lolor 1.3.0 * Add bidirectional large object migration between native PostgreSQL and lolor storage: diff --git a/docs/pg_upgrade_with_lolor.md b/docs/pg_upgrade_with_lolor.md index b816190..9ea394f 100644 --- a/docs/pg_upgrade_with_lolor.md +++ b/docs/pg_upgrade_with_lolor.md @@ -25,3 +25,9 @@ Then, use psql to enable `lolor`: ``` db1_18=# SELECT lolor.enable(); ``` + +Reconnect any client sessions afterwards. `lolor.enable()` and +`lolor.disable()` change which function OID owns each `pg_catalog.lo_*` name, +and libpq resolves those OIDs once per connection and caches them for the life +of the connection. A session that used a large object before the switch would +otherwise keep calling the previous implementation. diff --git a/expected/lolor.out b/expected/lolor.out index 0d5441b..09bd20f 100644 --- a/expected/lolor.out +++ b/expected/lolor.out @@ -217,7 +217,8 @@ SELECT lo_close(:fd); END; DROP EXTENSION lolor; -NOTICE: migrated 1 large object(s) from lolor to native storage +NOTICE: migrated 4 large object(s), 4 data page(s), to native storage +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs -- Check extension upgrade CREATE EXTENSION lolor VERSION '1.0'; SELECT lo_creat(-1) AS loid \gset @@ -251,25 +252,25 @@ NOTICE: migrated 1 large object(s) from lolor to native storage (1 row) SELECT lolor.migrate_from_native(); -- two objects -NOTICE: migrated 2 large object(s) (2 data page(s)) from native to lolor storage +NOTICE: migrated 5 large object(s) (5 data page(s)) from native to lolor storage migrate_from_native --------------------- - 2 + 5 (1 row) -- Repeat conversion cycle - should see the same two objects SELECT lolor.migrate_to_native(); -NOTICE: migrated 2 large object(s) from lolor to native storage +NOTICE: migrated 5 large object(s) from lolor to native storage migrate_to_native ------------------- - 2 + 5 (1 row) SELECT lolor.migrate_from_native(); -NOTICE: migrated 2 large object(s) (2 data page(s)) from native to lolor storage +NOTICE: migrated 5 large object(s) (5 data page(s)) from native to lolor storage migrate_from_native --------------------- - 2 + 5 (1 row) -- @@ -336,43 +337,46 @@ SELECT lolor.enable(); -- Check that no tails existing after the extension drop in both enabled and -- disabled states. DROP EXTENSION lolor; -NOTICE: migrated 3 large object(s) from lolor to native storage +NOTICE: migrated 6 large object(s) from lolor to native storage SELECT oid, proname FROM pg_proc WHERE proname IN ('lo_open_orig', 'lolor_lo_open'); oid | proname -----+--------- (0 rows) --- Check: we can't just delete LOLOR without LO migration in disabled mode. --- XXX: should we introduce a 'forced' flag to allow this? +-- DROP EXTENSION while lolor is disabled. Through 1.3.0 this failed with +-- "lolor must be enabled before migration to native", because the reverse +-- migration ran through the renamed _orig functions. It now works against +-- the catalogs directly, so the disabled state is no longer a special case +-- and the objects are still rescued. CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'stored before disabling') AS disabled_drop_oid \gset SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t (1 row) DROP EXTENSION lolor; -ERROR: lolor must be enabled before migration to native -SELECT extname FROM pg_extension; -- lolor is here +NOTICE: migrated 1 large object(s), 1 data page(s), to native storage +SELECT extname FROM pg_extension; -- check lolor removal extname --------- plpgsql - lolor -(2 rows) +(1 row) -SELECT lolor.enable(); - enable --------- - t +-- The object was migrated to native storage, not dropped with lolor's tables +SELECT convert_from(lo_get(:disabled_drop_oid), 'UTF8') AS survived_disabled_drop; + survived_disabled_drop +------------------------- + stored before disabling (1 row) -DROP EXTENSION lolor; -NOTICE: no lolor large objects to migrate -SELECT extname FROM pg_extension; -- check lolor removal - extname ---------- - plpgsql +SELECT lo_unlink(:disabled_drop_oid); + lo_unlink +----------- + 1 (1 row) -- @@ -385,16 +389,16 @@ SELECT lo_from_bytea(0, 'Native object number two') AS native_oid2 \gset SELECT count(*) AS native_lo_count FROM pg_catalog.pg_largeobject_metadata; native_lo_count ----------------- - 6 + 9 (1 row) -- Install lolor and migrate native LOs into lolor storage CREATE EXTENSION lolor; SELECT lolor.migrate_from_native(); -NOTICE: migrated 6 large object(s) (6 data page(s)) from native to lolor storage +NOTICE: migrated 9 large object(s), 9 data page(s), to lolor storage migrate_from_native --------------------- - 6 + 9 (1 row) -- After forward migration: expect 0 native objects @@ -407,7 +411,7 @@ SELECT count(*) AS native_after_migrate FROM pg_catalog.pg_largeobject_metadata; SELECT count(*) AS lolor_after_migrate FROM lolor.pg_largeobject_metadata; lolor_after_migrate --------------------- - 6 + 9 (1 row) -- Data integrity: expect "Native object number one" @@ -446,11 +450,12 @@ END; SELECT lo_from_bytea(0, 'Created directly in lolor') AS lolor_direct_oid \gset -- Reverse migration via DROP EXTENSION DROP EXTENSION lolor; -NOTICE: migrated 7 large object(s) from lolor to native storage +NOTICE: migrated 10 large object(s), 10 data page(s), to native storage +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs SELECT count(*) AS native_after_drop FROM pg_catalog.pg_largeobject_metadata; native_after_drop ------------------- - 7 + 10 (1 row) -- After DROP: expect "Native object number one" @@ -495,21 +500,22 @@ SELECT lo_unlink(:'lolor_direct_oid'::oid); CREATE EXTENSION lolor; SELECT lolor.migrate_from_native(); -NOTICE: migrated 4 large object(s) (4 data page(s)) from native to lolor storage +NOTICE: migrated 7 large object(s), 7 data page(s), to lolor storage migrate_from_native --------------------- - 4 + 7 (1 row) DROP EXTENSION lolor; -NOTICE: migrated 4 large object(s) from lolor to native storage +NOTICE: migrated 7 large object(s), 7 data page(s), to native storage +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs -- -- Manual migrate_to_native (not via DROP EXTENSION) -- CREATE EXTENSION lolor; SELECT lo_from_bytea(0, 'Manual reverse test') AS manual_oid \gset SELECT lolor.migrate_to_native(); -NOTICE: migrated 1 large object(s) from lolor to native storage +NOTICE: migrated 1 large object(s), 1 data page(s), to native storage migrate_to_native ------------------- 1 @@ -518,7 +524,7 @@ NOTICE: migrated 1 large object(s) from lolor to native storage SELECT count(*) AS native_after_manual FROM pg_catalog.pg_largeobject_metadata; native_after_manual --------------------- - 5 + 8 (1 row) SELECT count(*) AS lolor_after_manual FROM lolor.pg_largeobject_metadata; @@ -531,6 +537,7 @@ SELECT count(*) AS lolor_after_manual FROM lolor.pg_largeobject_metadata; BEGIN; -- Disable lolor to read from native storage directly SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t @@ -551,6 +558,7 @@ SELECT lo_unlink(:'manual_oid'::oid); (1 row) SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t @@ -558,6 +566,7 @@ SELECT lolor.enable(); DROP EXTENSION lolor; NOTICE: no lolor large objects to migrate +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs -- -- OID conflict detection -- @@ -569,11 +578,12 @@ INSERT INTO lolor.pg_largeobject_metadata (oid, lomowner, lomacl) VALUES (:'conflict_oid', (SELECT oid FROM pg_roles WHERE rolname = current_user), NULL); -- This should fail with OID conflict SELECT lolor.migrate_from_native(); -ERROR: OID conflict: some native large objects already exist in lolor storage +ERROR: 1 large object already exists in the destination storage -- Cleanup: remove the conflicting row and drop cleanly DELETE FROM lolor.pg_largeobject_metadata WHERE oid = :'conflict_oid'; DROP EXTENSION lolor; NOTICE: no lolor large objects to migrate +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs SELECT lo_unlink(:'conflict_oid'::oid); lo_unlink ----------- @@ -585,6 +595,7 @@ CREATE EXTENSION lolor; SELECT lo_from_bytea(0, 'Lolor side object') AS conflict_oid2 \gset -- Disable lolor to create a native LO with the same OID SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t @@ -599,6 +610,7 @@ SELECT :'created_oid' = :'conflict_oid2' AS oid_matches; (1 row) SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t @@ -606,9 +618,10 @@ SELECT lolor.enable(); -- migrate_to_native should detect the collision SELECT lolor.migrate_to_native(); -ERROR: OID conflict: some lolor large objects already exist in native storage +ERROR: 1 large object already exists in the destination storage -- Cleanup: remove the native duplicate, then drop cleanly SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t @@ -621,30 +634,35 @@ SELECT lo_unlink(:'conflict_oid2'::oid); (1 row) SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t (1 row) DROP EXTENSION lolor; -NOTICE: migrated 1 large object(s) from lolor to native storage +NOTICE: migrated 1 large object(s), 1 data page(s), to native storage +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs -- DROP EXTENSION should be rejected when migrate_to_native has OID conflict CREATE EXTENSION lolor; SELECT lo_from_bytea(0, 'Drop conflict test') AS drop_conflict_oid \gset -- Create a native LO with the same OID to force conflict at DROP time SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t (1 row) -SELECT lo_create(:'drop_conflict_oid'); - lo_create ------------ - 268385 +-- Print a stable boolean rather than the generated OID, which varies per run +SELECT lo_create(:'drop_conflict_oid') = :'drop_conflict_oid'::oid AS native_oid_honored; + native_oid_honored +-------------------- + t (1 row) SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t @@ -652,7 +670,7 @@ SELECT lolor.enable(); -- DROP EXTENSION should ERROR to prevent data loss DROP EXTENSION lolor; -ERROR: OID conflict: some lolor large objects already exist in native storage +ERROR: 1 large object already exists in the destination storage -- Extension should still be installed SELECT extname FROM pg_extension WHERE extname = 'lolor'; extname @@ -669,6 +687,7 @@ SELECT count(*) FROM lolor.pg_largeobject; -- Resolve the conflict: remove the native duplicate, then retry SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t @@ -681,6 +700,7 @@ SELECT lo_unlink(:'drop_conflict_oid'::oid); (1 row) SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t @@ -688,4 +708,1049 @@ SELECT lolor.enable(); -- Now DROP should succeed DROP EXTENSION lolor; -NOTICE: migrated 1 large object(s) from lolor to native storage +NOTICE: migrated 1 large object(s), 1 data page(s), to native storage +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +-- +-- Fidelity of the storage relocation (lolor 1.4.0) +-- +CREATE EXTENSION lolor; +CREATE ROLE lolor_owner; +CREATE ROLE lolor_grantee; +-- A sparse object: two pages holding a 10 MB logical object. Migration must +-- not fill the hole; the old implementation rewrote it through the LO API and +-- materialised every intervening page. +SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + disable +--------- + t +(1 row) + +SELECT lo_create(0) AS sparse_oid \gset +BEGIN; +SELECT lo_open(:sparse_oid, x'60000'::int) AS fd \gset +SELECT lowrite(:fd, 'start'); + lowrite +--------- + 5 +(1 row) + +SELECT lo_lseek64(:fd, 10000000, 0); + lo_lseek64 +------------ + 10000000 +(1 row) + +SELECT lowrite(:fd, 'end'); + lowrite +--------- + 3 +(1 row) + +SELECT lo_close(:fd); + lo_close +---------- + 0 +(1 row) + +END; +-- An object carrying owner, ACL and a comment +SELECT lo_from_bytea(0, 'annotated object') AS annotated_oid \gset +ALTER LARGE OBJECT :annotated_oid OWNER TO lolor_owner; +GRANT SELECT ON LARGE OBJECT :annotated_oid TO lolor_grantee; +COMMENT ON LARGE OBJECT :annotated_oid IS 'kept across migration'; +-- An object with no data pages at all +SELECT lo_create(0) AS empty_oid \gset +SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + enable +-------- + t +(1 row) + +SELECT lolor.migrate_from_native(); +NOTICE: migrated 12 large object(s), 12 data page(s), to lolor storage + migrate_from_native +--------------------- + 12 +(1 row) + +-- Sparse object keeps its exact page count (2), not 4883 +SELECT count(*) AS sparse_pages FROM lolor.pg_largeobject WHERE loid = :sparse_oid; + sparse_pages +-------------- + 2 +(1 row) + +SELECT lo_get(:sparse_oid) IS NOT NULL AS sparse_readable; + sparse_readable +----------------- + t +(1 row) + +SELECT length(lo_get(:sparse_oid)) AS sparse_length; + sparse_length +--------------- + 10000003 +(1 row) + +SELECT length(lo_get(:empty_oid)) AS empty_length; + empty_length +-------------- + 0 +(1 row) + +-- Owner and ACL survive; the comment is parked for the round trip +SELECT pg_get_userbyid(lomowner) AS owner, lomacl IS NOT NULL AS has_acl +FROM lolor.pg_largeobject_metadata WHERE oid = :annotated_oid; + owner | has_acl +-------------+--------- + lolor_owner | t +(1 row) + +SELECT description FROM lolor.pg_largeobject_description WHERE loid = :annotated_oid; + description +----------------------- + kept across migration +(1 row) + +-- Native side is fully cleaned up, including shared deps and comments +SELECT count(*) AS native_objs FROM pg_catalog.pg_largeobject_metadata; + native_objs +------------- + 0 +(1 row) + +SELECT count(*) AS native_shdep FROM pg_shdepend + WHERE classid = 'pg_largeobject'::regclass + AND dbid = (SELECT oid FROM pg_database WHERE datname = current_database()); + native_shdep +-------------- + 0 +(1 row) + +SELECT count(*) AS native_comments FROM pg_description + WHERE classoid = 'pg_largeobject'::regclass; + native_comments +----------------- + 0 +(1 row) + +-- Back to native storage +SELECT lolor.migrate_to_native(); +NOTICE: migrated 12 large object(s), 12 data page(s), to native storage + migrate_to_native +------------------- + 12 +(1 row) + +-- Ownership is recorded in pg_shdepend, not merely in lomowner: a raw catalog +-- UPDATE (what 1.3.0 did) left DROP ROLE unable to see the object. +SELECT pg_get_userbyid(lomowner) AS owner +FROM pg_catalog.pg_largeobject_metadata WHERE oid = :annotated_oid; + owner +------------- + lolor_owner +(1 row) + +SELECT deptype, refobjid::regrole::text AS role FROM pg_shdepend + WHERE classid = 'pg_largeobject'::regclass AND objid = :annotated_oid + AND dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + ORDER BY deptype; + deptype | role +---------+--------------- + a | lolor_grantee + o | lolor_owner +(2 rows) + +SELECT description FROM pg_description + WHERE classoid = 'pg_largeobject'::regclass AND objoid = :annotated_oid; + description +----------------------- + kept across migration +(1 row) + +-- DROP ROLE must refuse for both the owner and the ACL grantee +DROP ROLE lolor_owner; +ERROR: role "lolor_owner" cannot be dropped because some objects depend on it +DROP ROLE lolor_grantee; +ERROR: role "lolor_grantee" cannot be dropped because some objects depend on it +-- Sparse object is still sparse after the round trip +SELECT count(*) AS sparse_pages_native FROM pg_catalog.pg_largeobject + WHERE loid = :sparse_oid; + sparse_pages_native +--------------------- + 2 +(1 row) + +-- Comment parking table is emptied once the comments are reinstated +SELECT count(*) AS parked_left FROM lolor.pg_largeobject_description; + parked_left +------------- + 0 +(1 row) + +-- Cleanup +SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + disable +--------- + t +(1 row) + +SELECT lo_unlink(:sparse_oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lo_unlink(:annotated_oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lo_unlink(:empty_oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + enable +-------- + t +(1 row) + +DROP EXTENSION lolor; +NOTICE: no lolor large objects to migrate +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +DROP ROLE lolor_owner; +DROP ROLE lolor_grantee; +-- +-- An unprivileged user must not be able to wedge lolor by squatting the +-- names the state probes look for. Before 1.4.0 this made is_enabled() +-- raise "inconsistent state", which also blocked DROP EXTENSION. +-- +CREATE EXTENSION lolor; +CREATE ROLE lolor_squatter; +GRANT CREATE ON SCHEMA public TO lolor_squatter; +SET ROLE lolor_squatter; +CREATE FUNCTION public.lolor_lo_open(oid, int4) RETURNS int4 + AS 'SELECT 1' LANGUAGE sql; +CREATE FUNCTION public.lo_close_orig(int4) RETURNS int4 + AS 'SELECT 1' LANGUAGE sql; +RESET ROLE; +SELECT lolor.is_enabled() AS unaffected_by_squatting; + unaffected_by_squatting +------------------------- + t +(1 row) + +DROP FUNCTION public.lolor_lo_open(oid, int4); +DROP FUNCTION public.lo_close_orig(int4); +REVOKE CREATE ON SCHEMA public FROM lolor_squatter; +DROP ROLE lolor_squatter; +DROP EXTENSION lolor; +NOTICE: no lolor large objects to migrate +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +-- +-- DROP SCHEMA lolor CASCADE reaches the extension by dependency cascade +-- rather than as DROP EXTENSION. The cleanup trigger must still run, or the +-- objects are destroyed and pg_catalog is left without a working lo_open(). +-- +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'rescued from drop schema') AS rescued_oid \gset +DROP SCHEMA lolor CASCADE; +NOTICE: migrated 1 large object(s), 1 data page(s), to native storage +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +NOTICE: drop cascades to extension lolor +SELECT count(*) AS ext_left FROM pg_extension WHERE extname = 'lolor'; + ext_left +---------- + 0 +(1 row) + +SELECT to_regprocedure('pg_catalog.lo_open(oid,int4)') IS NOT NULL AS lo_open_restored; + lo_open_restored +------------------ + t +(1 row) + +SELECT to_regprocedure('pg_catalog.lo_open_orig(oid,int4)') IS NULL AS no_orig_left; + no_orig_left +-------------- + t +(1 row) + +SELECT convert_from(lo_get(:rescued_oid), 'UTF8') AS rescued_content; + rescued_content +-------------------------- + rescued from drop schema +(1 row) + +SELECT lo_unlink(:rescued_oid); + lo_unlink +----------- + 1 +(1 row) + +-- +-- lo_import() and lo_export() read and write files on the server, so core +-- revokes EXECUTE on them from PUBLIC. lolor replaces them by renaming the +-- originals out of the way, and an ACL belongs to a function rather than to a +-- name: the restriction stays on the parked original, and the replacement is +-- created with the default (EXECUTE TO PUBLIC) unless locked down explicitly. +-- Before 1.4.0 that let any database user read or overwrite server files. +-- +CREATE EXTENSION lolor; +SELECT r.proname || '(' || pg_get_function_arguments(r.oid) || ')' AS func, + EXISTS (SELECT 1 FROM aclexplode(coalesce(r.proacl, acldefault('f', r.proowner))) a + WHERE a.grantee = 0) AS replacement_public_execute, + EXISTS (SELECT 1 FROM aclexplode(coalesce(o.proacl, acldefault('f', o.proowner))) a + WHERE a.grantee = 0) AS original_public_execute +FROM pg_proc r +JOIN pg_namespace n ON n.oid = r.pronamespace AND n.nspname = 'pg_catalog' +JOIN pg_proc o ON o.pronamespace = r.pronamespace + AND o.proname = r.proname || '_orig' + AND o.proargtypes = r.proargtypes +WHERE r.proname IN ('lo_import', 'lo_export') +ORDER BY 1; + func | replacement_public_execute | original_public_execute +----------------------+----------------------------+------------------------- + lo_export(oid, text) | f | f + lo_import(text) | f | f + lo_import(text, oid) | f | f +(3 rows) + +DROP EXTENSION lolor; +NOTICE: no lolor large objects to migrate +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +-- +-- lolor.node is bounded by the OID encoding, not by an independently written +-- constant: the low LOLOR_NODEID_BITS of a generated OID carry the node id, so +-- 16 does not fit and was previously accepted while encoding as node 0. +-- +SET lolor.node = 16; +ERROR: 16 is outside the valid range for parameter "lolor.node" (0 .. 15) +SET lolor.node = 15; +SET lolor.node = 1; +-- +-- Permission enforcement. +-- +-- Objects in lolor storage have no catalog entry, so lolor cannot use the +-- syscache-backed owner and ACL checks and reimplements them against its own +-- tables. Exercise that path rather than assuming it matches core. +-- +CREATE EXTENSION lolor; +CREATE ROLE lolor_alice; +CREATE ROLE lolor_bob; +-- Large object error messages quote the OID, which is generated and so +-- differs between runs. Report the message with digits masked instead. +CREATE FUNCTION lolor_expect_error(cmd text) RETURNS text AS $$ +BEGIN + EXECUTE cmd; + RETURN 'unexpectedly succeeded'; +EXCEPTION WHEN OTHERS THEN + RETURN regexp_replace(SQLERRM, '[0-9]+', 'NNN', 'g'); +END +$$ LANGUAGE plpgsql; +SET ROLE lolor_alice; +SELECT lo_from_bytea(0, 'alice private data') AS alice_oid \gset +SELECT convert_from(lo_get(:alice_oid), 'UTF8') AS owner_can_read; + owner_can_read +-------------------- + alice private data +(1 row) + +RESET ROLE; +-- A different role gets nothing without a grant. +SET ROLE lolor_bob; +SELECT lolor_expect_error(format('SELECT lo_get(%s)', :alice_oid)) AS read_denied; + read_denied +---------------------------------------- + permission denied for large object NNN +(1 row) + +SELECT lolor_expect_error(format('SELECT lo_open(%s, 262144)', :alice_oid)) AS open_denied; + open_denied +---------------------------------------- + permission denied for large object NNN +(1 row) + +SELECT lolor_expect_error(format('SELECT lo_put(%s, 0, ''x'')', :alice_oid)) AS write_denied; + write_denied +---------------------------------------- + permission denied for large object NNN +(1 row) + +SELECT lolor_expect_error(format('SELECT lo_unlink(%s)', :alice_oid)) AS unlink_denied; + unlink_denied +----------------------------------- + must be owner of large object NNN +(1 row) + +RESET ROLE; +-- The superuser bypasses the check, as in core. +SELECT convert_from(lo_get(:alice_oid), 'UTF8') AS superuser_can_read; + superuser_can_read +-------------------- + alice private data +(1 row) + +-- GRANT/ALTER on a large object act on pg_largeobject_metadata, where an +-- object in lolor storage has no row. This limitation is documented; assert +-- it so that a change in behaviour is noticed. +SELECT lolor_expect_error( + format('GRANT SELECT ON LARGE OBJECT %s TO lolor_bob', :alice_oid)) AS grant_unsupported; + grant_unsupported +--------------------------------- + large object NNN does not exist +(1 row) + +SELECT lolor_expect_error( + format('ALTER LARGE OBJECT %s OWNER TO lolor_bob', :alice_oid)) AS alter_unsupported; + alter_unsupported +--------------------------------- + large object NNN does not exist +(1 row) + +SELECT lo_unlink(:alice_oid); + lo_unlink +----------- + 1 +(1 row) + +-- +-- Objects in lolor storage are rows in ordinary tables and cannot participate +-- in pg_shdepend, so DROP ROLE does not notice that a role still owns one. +-- lolor.check_orphans() exists to make the consequence findable. +-- +SET ROLE lolor_alice; +SELECT lo_from_bytea(0, 'owned by a role about to vanish') AS orphan_oid \gset +RESET ROLE; +SELECT count(*) AS orphans_before FROM lolor.check_orphans(); + orphans_before +---------------- + 0 +(1 row) + +DROP ROLE lolor_alice; +SELECT count(*) AS orphans_after FROM lolor.check_orphans(); + orphans_after +--------------- + 1 +(1 row) + +SELECT lo_unlink(:orphan_oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT count(*) AS orphans_cleared FROM lolor.check_orphans(); + orphans_cleared +----------------- + 0 +(1 row) + +DROP ROLE lolor_bob; +DROP FUNCTION lolor_expect_error(text); +-- +-- 64-bit interface and page-boundary I/O. lo_put(), lo_tell64() and +-- lo_truncate64() had no coverage at all. +-- +SELECT current_setting('block_size')::int / 4 AS loblksize \gset +-- Write straddling a page boundary, then read the fragment back. +SELECT lo_create(0) AS span_oid \gset +SELECT lo_put(:span_oid, (:loblksize - 4)::bigint, '\x4142434445464748'::bytea); + lo_put +-------- + +(1 row) + +SELECT length(lo_get(:span_oid)) = :loblksize + 4 AS spans_two_pages; + spans_two_pages +----------------- + t +(1 row) + +SELECT count(*) = 2 AS two_data_pages FROM lolor.pg_largeobject WHERE loid = :span_oid; + two_data_pages +---------------- + t +(1 row) + +SELECT encode(lo_get(:span_oid, (:loblksize - 4)::bigint, 8), 'hex') AS across_boundary; + across_boundary +------------------ + 4142434445464748 +(1 row) + +-- lo_truncate64() extending past the end leaves a hole rather than pages. +BEGIN; +SELECT lo_open(:span_oid, x'60000'::int) AS fd \gset +SELECT lo_truncate64(:fd, (:loblksize * 4)::bigint); + lo_truncate64 +--------------- + 0 +(1 row) + +SELECT lo_lseek64(:fd, 0, 2) = (:loblksize * 4)::bigint AS seek_end_matches; + seek_end_matches +------------------ + t +(1 row) + +SELECT lo_tell64(:fd) = (:loblksize * 4)::bigint AS tell64_matches; + tell64_matches +---------------- + t +(1 row) + +SELECT lo_close(:fd); + lo_close +---------- + 0 +(1 row) + +END; +SELECT length(lo_get(:span_oid)) = :loblksize * 4 AS truncate64_extended; + truncate64_extended +--------------------- + t +(1 row) + +SELECT count(*) < 4 AS hole_not_materialised + FROM lolor.pg_largeobject WHERE loid = :span_oid; + hole_not_materialised +----------------------- + t +(1 row) + +-- Truncating back down releases the pages beyond the new length. +BEGIN; +SELECT lo_open(:span_oid, x'60000'::int) AS fd \gset +SELECT lo_truncate64(:fd, 10); + lo_truncate64 +--------------- + 0 +(1 row) + +SELECT lo_close(:fd); + lo_close +---------- + 0 +(1 row) + +END; +SELECT length(lo_get(:span_oid)) AS len_after_shrink; + len_after_shrink +------------------ + 10 +(1 row) + +SELECT lo_unlink(:span_oid); + lo_unlink +----------- + 1 +(1 row) + +-- +-- Subtransaction cleanup. A descriptor opened inside an aborted +-- subtransaction must be closed by the rollback, one opened in the enclosing +-- transaction must survive it, and data written in the aborted +-- subtransaction must not persist. +-- +BEGIN; +SELECT lo_from_bytea(0, 'outer') AS sub_oid \gset +SELECT lo_open(:sub_oid, x'60000'::int) AS outer_fd \gset +SAVEPOINT s1; +SELECT lo_open(:sub_oid, x'60000'::int) AS inner_fd \gset +SELECT lo_put(:sub_oid, 0, 'INNER'); + lo_put +-------- + +(1 row) + +ROLLBACK TO s1; +SAVEPOINT s2; +SELECT lo_tell(:inner_fd); +ERROR: invalid large-object descriptor: 1 +ROLLBACK TO s2; +SELECT lo_tell(:outer_fd) AS outer_descriptor_survives; + outer_descriptor_survives +--------------------------- + 0 +(1 row) + +SELECT lo_close(:outer_fd); + lo_close +---------- + 0 +(1 row) + +COMMIT; +SELECT convert_from(lo_get(:sub_oid), 'UTF8') AS subxact_write_rolled_back; + subxact_write_rolled_back +--------------------------- + outer +(1 row) + +SELECT lo_unlink(:sub_oid); + lo_unlink +----------- + 1 +(1 row) + +-- +-- A rolled back transaction must leave no trace in lolor storage. +-- +SELECT count(*) AS rows_before FROM lolor.pg_largeobject_metadata; + rows_before +------------- + 0 +(1 row) + +BEGIN; +SELECT lo_from_bytea(0, 'discarded') IS NOT NULL AS created_in_aborted_xact; + created_in_aborted_xact +------------------------- + t +(1 row) + +ROLLBACK; +SELECT count(*) AS rows_after FROM lolor.pg_largeobject_metadata; + rows_after +------------ + 0 +(1 row) + +-- +-- Seek variants and read/write edge cases. +-- +SELECT lo_from_bytea(0, '0123456789abcdef') AS seek_oid \gset +BEGIN; +SELECT lo_open(:seek_oid, x'60000'::int) AS fd \gset +SELECT lo_lseek(:fd, 4, 0) AS seek_set; + seek_set +---------- + 4 +(1 row) + +SELECT lo_lseek(:fd, 2, 1) AS seek_cur; + seek_cur +---------- + 6 +(1 row) + +SELECT lo_lseek(:fd, -3, 2) AS seek_end; + seek_end +---------- + 13 +(1 row) + +SELECT lo_tell(:fd) AS tell_after_seeks; + tell_after_seeks +------------------ + 13 +(1 row) + +SELECT convert_from(loread(:fd, 3), 'UTF8') AS read_tail; + read_tail +----------- + def +(1 row) + +-- A read at end of object returns nothing rather than failing. +SELECT length(loread(:fd, 100)) AS read_past_eof; + read_past_eof +--------------- + 0 +(1 row) + +-- Zero-length read and empty write are both no-ops. +SELECT lo_lseek(:fd, 0, 0); + lo_lseek +---------- + 0 +(1 row) + +SELECT length(loread(:fd, 0)) AS zero_length_read; + zero_length_read +------------------ + 0 +(1 row) + +SELECT lowrite(:fd, '') AS empty_write; + empty_write +------------- + 0 +(1 row) + +SELECT lo_close(:fd); + lo_close +---------- + 0 +(1 row) + +END; +SELECT length(lo_get(:seek_oid)) AS unchanged_length; + unchanged_length +------------------ + 16 +(1 row) + +-- lo_get with a fragment length beyond the end is clamped, not an error. +SELECT convert_from(lo_get(:seek_oid, 10, 1000), 'UTF8') AS clamped_fragment; + clamped_fragment +------------------ + abcdef +(1 row) + +SELECT lo_unlink(:seek_oid); + lo_unlink +----------- + 1 +(1 row) + +-- +-- Reading a multi-page object back in chunks that do not align with the +-- page size exercises the page-assembly path in lolor_inv_read(). +-- +SELECT current_setting('block_size')::int / 4 AS loblksize \gset +SELECT lo_from_bytea(0, repeat('abcdefgh', (:loblksize * 3 / 8))::bytea) AS multi_oid \gset +SELECT length(lo_get(:multi_oid)) = :loblksize * 3 AS three_pages_written; + three_pages_written +--------------------- + t +(1 row) + +SELECT count(*) AS page_rows FROM lolor.pg_largeobject WHERE loid = :multi_oid; + page_rows +----------- + 3 +(1 row) + +BEGIN; +SELECT lo_open(:multi_oid, 262144) AS fd \gset +SELECT length(loread(:fd, 1000)) AS chunk1; + chunk1 +-------- + 1000 +(1 row) + +SELECT length(loread(:fd, 5000)) AS chunk2; + chunk2 +-------- + 5000 +(1 row) + +SELECT length(loread(:fd, 100000)) AS chunk_rest; + chunk_rest +------------ + 144 +(1 row) + +SELECT lo_close(:fd); + lo_close +---------- + 0 +(1 row) + +END; +SELECT md5(lo_get(:multi_oid)) = md5(repeat('abcdefgh', (:loblksize * 3 / 8))::bytea) + AS content_round_trips; + content_round_trips +--------------------- + t +(1 row) + +SELECT lo_unlink(:multi_oid); + lo_unlink +----------- + 1 +(1 row) + +-- +-- Error paths. +-- +SELECT lo_get(0); +ERROR: large object 0 does not exist +SELECT lo_unlink(0); +ERROR: large object 0 does not exist +BEGIN; +SELECT lo_open(0, 262144); +ERROR: large object 0 does not exist +ROLLBACK; +-- +-- Verification helpers introduced in 1.4.0. +-- +-- Start from a clean slate so the counts below do not depend on what earlier +-- sections happened to leave behind. The notices carry those counts, so they +-- are suppressed for the duration. +SET client_min_messages = warning; +SELECT lolor.migrate_to_native() IS NOT NULL AS drained_to_native; + drained_to_native +------------------- + t +(1 row) + +SELECT lolor.disable(); + disable +--------- + t +(1 row) + +DO $$ +DECLARE r record; +BEGIN + FOR r IN SELECT oid FROM pg_catalog.pg_largeobject_metadata LOOP + PERFORM lo_unlink(r.oid); + END LOOP; +END +$$; +SELECT lolor.enable(); + enable +-------- + t +(1 row) + +RESET client_min_messages; +SELECT count(*) AS native_oids_when_empty FROM lolor.native_lo_oids(); + native_oids_when_empty +------------------------ + 0 +(1 row) + +-- Both directions report zero rather than failing when there is nothing. +SELECT lolor.migrate_to_native() AS nothing_to_move; +NOTICE: no lolor large objects to migrate + nothing_to_move +----------------- + 0 +(1 row) + +SELECT lolor.migrate_from_native() AS nothing_to_take; +NOTICE: no native large objects to migrate + nothing_to_take +----------------- + 0 +(1 row) + +-- digest() reports one row per object. The OID and owner vary between runs; +-- the page count, byte count and content digest do not. +SELECT lo_from_bytea(0, 'digest one') AS d1 \gset +SELECT lo_from_bytea(0, 'digest two, longer') AS d2 \gset +SELECT npages, nbytes, digest FROM lolor.digest() ORDER BY nbytes, digest; + npages | nbytes | digest +--------+--------+---------------------------------- + 1 | 10 | be0bbea1ab8a74be81d08e575166e9e4 + 1 | 18 | ce7a8cf8a81e8ec180a8a8a224f473c6 +(2 rows) + +SELECT lo_unlink(:d1); + lo_unlink +----------- + 1 +(1 row) + +SELECT lo_unlink(:d2); + lo_unlink +----------- + 1 +(1 row) + +-- migrate_storage() is the mechanism behind both migration functions. Grant +-- schema access so that the function's own privileges are what is exercised +-- rather than USAGE on the schema. +CREATE ROLE lolor_nosuper; +GRANT USAGE ON SCHEMA lolor TO lolor_nosuper; +SET ROLE lolor_nosuper; +SELECT lolor.migrate_storage(true); +ERROR: permission denied for function migrate_storage +SELECT lolor.migrate_from_native(); +ERROR: permission denied for function migrate_from_native +SELECT lolor.migrate_to_native(); +ERROR: permission denied for function migrate_to_native +RESET ROLE; +REVOKE USAGE ON SCHEMA lolor FROM lolor_nosuper; +DROP ROLE lolor_nosuper; +-- Native OIDs are not node-encoded, so two nodes can hold different objects +-- under the same OID. Passing the peer OIDs makes that a refusal instead of +-- silent divergence once the migration is hidden from replication. +SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + disable +--------- + t +(1 row) + +SELECT lo_create(0) AS peer_oid \gset +SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + enable +-------- + t +(1 row) + +SELECT lolor.migrate_from_native(peer_oids => ARRAY[:peer_oid]::oid[]); +ERROR: cannot migrate: 1 OID(s) are also held natively by another node +-- Security labels cannot be represented in lolor storage and cannot be +-- reinstated without their provider, so migration refuses rather than +-- discarding them. +INSERT INTO pg_catalog.pg_seclabel (objoid, classoid, objsubid, provider, label) +VALUES (:peer_oid, 'pg_catalog.pg_largeobject'::regclass, 0, 'lolor_test', 'secret'); +SELECT lolor.migrate_from_native(); +ERROR: cannot migrate: 1 large object security label(s) present +DELETE FROM pg_catalog.pg_seclabel + WHERE classoid = 'pg_catalog.pg_largeobject'::regclass AND provider = 'lolor_test'; +-- With the label gone the migration proceeds. +SELECT lolor.migrate_from_native() AS migrated; +NOTICE: migrated 1 large object(s), 0 data page(s), to lolor storage + migrated +---------- + 1 +(1 row) + +SELECT lo_unlink(:peer_oid); + lo_unlink +----------- + 1 +(1 row) + +DROP EXTENSION lolor; +NOTICE: no lolor large objects to migrate +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +-- +-- A parked comment must not outlive the object it describes. Object OIDs are +-- only checked against pg_largeobject_metadata when a new one is generated, so +-- a comment left behind by lo_unlink() would be handed to whatever object next +-- took that OID. +-- +CREATE EXTENSION lolor; +SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + disable +--------- + t +(1 row) + +SELECT lo_from_bytea(0, 'has a comment') AS commented_oid \gset +COMMENT ON LARGE OBJECT :commented_oid IS 'parked then orphaned'; +SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + enable +-------- + t +(1 row) + +SELECT lolor.migrate_from_native(); +NOTICE: migrated 1 large object(s), 1 data page(s), to lolor storage + migrate_from_native +--------------------- + 1 +(1 row) + +SELECT count(*) AS parked FROM lolor.pg_largeobject_description + WHERE loid = :commented_oid; + parked +-------- + 1 +(1 row) + +-- Unlinking must take the parked comment with it. +SELECT lo_unlink(:commented_oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT count(*) AS parked_after_unlink FROM lolor.pg_largeobject_description + WHERE loid = :commented_oid; + parked_after_unlink +--------------------- + 0 +(1 row) + +-- Re-create an object under the very same OID and send it back to native +-- storage. It must arrive with no comment. +SELECT lo_create(:commented_oid) = :commented_oid AS oid_reused; + oid_reused +------------ + t +(1 row) + +SELECT lolor.migrate_to_native(); +NOTICE: migrated 1 large object(s), 0 data page(s), to native storage + migrate_to_native +------------------- + 1 +(1 row) + +SELECT count(*) AS inherited_comment FROM pg_description + WHERE classoid = 'pg_largeobject'::regclass AND objoid = :commented_oid; + inherited_comment +------------------- + 0 +(1 row) + +SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + disable +--------- + t +(1 row) + +SELECT lo_unlink(:commented_oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + enable +-------- + t +(1 row) + +-- +-- DROP SCHEMA without CASCADE is RESTRICT and cannot remove a schema that +-- still holds the extension's tables. The cleanup must not run for a command +-- that is going to be rejected, or it would migrate every large object and +-- take the storage locks only to have the work rolled back. +-- +SELECT lo_from_bytea(0, 'still here afterwards') AS kept_oid \gset +DROP SCHEMA lolor; +ERROR: cannot drop schema lolor because other objects depend on it +SELECT count(*) AS extension_still_installed + FROM pg_extension WHERE extname = 'lolor'; + extension_still_installed +--------------------------- + 1 +(1 row) + +SELECT convert_from(lo_get(:kept_oid), 'UTF8') AS object_untouched; + object_untouched +----------------------- + still here afterwards +(1 row) + +SELECT count(*) AS still_in_lolor_storage + FROM lolor.pg_largeobject_metadata WHERE oid = :kept_oid; + still_in_lolor_storage +------------------------ + 1 +(1 row) + +SELECT lo_unlink(:kept_oid); + lo_unlink +----------- + 1 +(1 row) + +DROP EXTENSION lolor; +NOTICE: no lolor large objects to migrate +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs diff --git a/lolor--1.0.sql b/lolor--1.0.sql index e21be36..762d271 100644 --- a/lolor--1.0.sql +++ b/lolor--1.0.sql @@ -74,6 +74,11 @@ CREATE FUNCTION pg_catalog.lo_export(oid, text) RETURNS integer AS 'MODULE_PATHNAME', 'lolor_lo_export' LANGUAGE C STRICT VOLATILE; +-- lo_export writes a server-side file. The original's restrictive ACL stayed +-- behind on lo_export_orig when it was renamed, so the replacement must be +-- locked down explicitly or any user could write arbitrary files as the +-- server account. +REVOKE ALL ON FUNCTION pg_catalog.lo_export(oid, text) FROM PUBLIC; -- lo_from_bytea ALTER FUNCTION pg_catalog.lo_from_bytea(oid, bytea) @@ -106,6 +111,8 @@ CREATE FUNCTION pg_catalog.lo_import(text) RETURNS oid AS 'MODULE_PATHNAME', 'lolor_lo_import' LANGUAGE C STRICT VOLATILE; +-- Reads a server-side file; see the note on lo_export above. +REVOKE ALL ON FUNCTION pg_catalog.lo_import(text) FROM PUBLIC; -- lo_import ALTER FUNCTION pg_catalog.lo_import(text, oid) @@ -114,6 +121,8 @@ CREATE FUNCTION pg_catalog.lo_import(text, oid) RETURNS oid AS 'MODULE_PATHNAME', 'lolor_lo_import_with_oid' LANGUAGE C STRICT VOLATILE; +-- Reads a server-side file; see the note on lo_export above. +REVOKE ALL ON FUNCTION pg_catalog.lo_import(text, oid) FROM PUBLIC; -- lo_lseek ALTER FUNCTION pg_catalog.lo_lseek(integer, integer, integer) diff --git a/lolor--1.3.0--1.4.0.sql b/lolor--1.3.0--1.4.0.sql new file mode 100644 index 0000000..8ee7420 --- /dev/null +++ b/lolor--1.3.0--1.4.0.sql @@ -0,0 +1,604 @@ +/* lolor--1.3.0--1.4.0.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION lolor UPDATE" to load this file. \quit + +/* + * --------------------------------------------------------------------------- + * Comment parking + * --------------------------------------------------------------------------- + * + * COMMENT ON LARGE OBJECT stores its text in pg_description keyed by + * (classoid = 'pg_largeobject', objoid = loid). While an object lives in + * lolor storage there is no catalog object for that row to describe, so the + * comment is parked here and restored on the way back to native storage. + * Without this the comment is silently lost the first time an object is + * migrated. + */ +CREATE TABLE lolor.pg_largeobject_description( + loid oid NOT NULL, + description text NOT NULL, + CONSTRAINT pg_largeobject_description_pkey PRIMARY KEY (loid)); +SELECT pg_catalog.pg_extension_config_dump('lolor.pg_largeobject_description', ''); + +/* + * --------------------------------------------------------------------------- + * Storage relocation mechanism + * --------------------------------------------------------------------------- + * + * lolor.migrate_storage() moves every large object from one store to the + * other by copying tuples directly between the two relations, which have + * identical layouts by construction. It carries ownership, ACLs and comments + * across, verifies the layouts at run time, and removes native objects + * through the same deletion path DROP uses. + * + * It is the mechanism only. Policy -- privileges and the interaction with + * logical replication -- lives in the wrappers below, which are the supported + * entry points. + */ +CREATE FUNCTION lolor.migrate_storage(to_native boolean) + RETURNS bigint + AS 'MODULE_PATHNAME', 'lolor_migrate_storage' + LANGUAGE C STRICT VOLATILE; + +REVOKE ALL ON FUNCTION lolor.migrate_storage(boolean) FROM PUBLIC; + +/* + * --------------------------------------------------------------------------- + * Shared replication guard + * --------------------------------------------------------------------------- + * + * Both migration directions face the same question: the row movement is a + * node-local storage relocation, but logical decoding cannot tell that apart + * from ordinary DML. If a subscriber decodes it, the two nodes diverge. + * + * Returns true when repair mode was engaged and the caller must turn it off, + * false when no suppression was needed, and NULL when the migration must be + * refused. NULL is only ever returned when strict_mode is false; a strict + * caller gets an ERROR instead. That asymmetry is deliberate and is + * explained at each call site. + */ +CREATE FUNCTION lolor._migration_guard(strict_mode boolean, refuse_hint text) +RETURNS boolean AS $$ +DECLARE + lr_slots boolean; + foreign_slots boolean; + spock_ready boolean; +BEGIN + SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_replication_slots + WHERE slot_type = 'logical' AND database = current_database() + ) INTO lr_slots; + + -- Nothing decodes this database: nothing to suppress. + IF NOT lr_slots THEN + RETURN false; + END IF; + + -- Use spock only when it is fully operational: the extension is installed + -- (pg_extension is superuser-gated, so the schema name cannot be squatted + -- by an unprivileged user), the function exists (older spock versions lack + -- it) and the GUC exists (the library is actually preloaded). + spock_ready := + EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'spock') + AND to_regprocedure('spock.repair_mode(boolean)') IS NOT NULL + AND current_setting('spock.replication_repair_mode', true) IS NOT NULL; + + IF NOT spock_ready THEN + IF strict_mode THEN + RAISE EXCEPTION 'cannot migrate large objects: logical replication slot(s) exist' + USING DETAIL = 'Without spock the migration DML cannot be excluded from ' + 'logical decoding, so subscribers would receive this ' + 'node-local storage relocation as ordinary row changes', + HINT = refuse_hint; + END IF; + RAISE WARNING 'not migrating: logical replication slot(s) exist' + USING DETAIL = 'This call is a no-op: no large objects were migrated', + HINT = refuse_hint; + RETURN NULL; + END IF; + + -- spock.repair_mode() suppresses spock's own output plugin only. That + -- plugin is the 'spock_output' module (spock's Makefile builds it as + -- MODULES = spock_output); any other plugin decoding this database would + -- still see the migration DML. + SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_replication_slots + WHERE slot_type = 'logical' AND database = current_database() + AND plugin <> 'spock_output' + ) INTO foreign_slots; + + IF foreign_slots THEN + IF strict_mode THEN + RAISE EXCEPTION 'cannot migrate large objects: non-spock logical replication slot(s) exist' + USING DETAIL = 'spock repair mode silences only the spock_output plugin; a slot ' + 'using another plugin (pgoutput, wal2json, decoderbufs, ...) would ' + 'still decode the migration and diverge from this node', + HINT = refuse_hint; + END IF; + RAISE WARNING 'not migrating: non-spock logical replication slot(s) exist' + USING DETAIL = 'This call is a no-op: no large objects were migrated', + HINT = refuse_hint; + RETURN NULL; + END IF; + + IF current_setting('spock.replication_repair_mode', true) = 'off' THEN + PERFORM spock.repair_mode(true); + RETURN true; + END IF; + + -- Repair mode was already on; leave it to whoever turned it on. + RETURN false; +END; +$$ LANGUAGE plpgsql VOLATILE; + +REVOKE ALL ON FUNCTION lolor._migration_guard(boolean, text) FROM PUBLIC; + +/* + * --------------------------------------------------------------------------- + * Migration entry points + * --------------------------------------------------------------------------- + */ + +/* + * lolor.migrate_from_native(peer_oids) + * + * Move every native large object into lolor storage, preserving OIDs, owners, + * ACLs, comments and exact page layout. Returns the number of objects moved, + * or -1 if the migration was refused because logical decoding of the movement + * could not be suppressed. + * + * Refusal is a soft -1 rather than an ERROR because this is a manual, + * non-destructive operation: on refusal the native objects are untouched, so + * the caller can drop the offending slots and retry. Callers acting on the + * result MUST check for a negative return; 0 means "nothing to migrate". + * + * peer_oids optionally carries the native large object OIDs held by the other + * nodes of the cluster. Native OIDs are not node-encoded, so two nodes can + * independently hold different objects under the same OID; migrating both + * would converge them onto one row and diverge the cluster. Because the + * movement is hidden from replication, nothing would detect that afterwards. + * Supplying peer_oids turns it into a pre-flight refusal. Collect them with + * lolor.native_lo_oids() on each node first. + */ +DROP FUNCTION lolor.migrate_from_native(); +CREATE FUNCTION lolor.migrate_from_native(peer_oids oid[] DEFAULT NULL) +RETURNS bigint AS $$ +DECLARE + lo_count bigint; + repair_enabled boolean; + overlap oid[]; + labelled bigint; +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = current_user AND rolsuper) THEN + RAISE EXCEPTION 'lolor.migrate_from_native() requires superuser privileges'; + END IF; + + SELECT count(*) INTO lo_count FROM pg_catalog.pg_largeobject_metadata; + + IF lo_count = 0 THEN + RAISE NOTICE 'no native large objects to migrate'; + RETURN 0; + END IF; + + /* + * Security labels are attached to the pg_largeobject catalog entry and have + * nowhere to live in lolor storage. Unlike comments they cannot be parked + * and replayed, because reinstating one has to go through the label + * provider. Refuse rather than discard them silently. + */ + SELECT count(*) INTO labelled + FROM pg_catalog.pg_seclabel + WHERE classoid = 'pg_catalog.pg_largeobject'::regclass; + + IF labelled > 0 THEN + RAISE EXCEPTION 'cannot migrate: % large object security label(s) present', labelled + USING DETAIL = 'lolor storage cannot represent security labels, and they ' + 'cannot be reinstated without their label provider', + HINT = 'Remove the labels with SECURITY LABEL ... IS NULL before migrating'; + END IF; + + /* Cross-node OID collision pre-flight; see the comment on this function. */ + IF peer_oids IS NOT NULL THEN + SELECT array_agg(m.oid ORDER BY m.oid) INTO overlap + FROM pg_catalog.pg_largeobject_metadata m + WHERE m.oid = ANY (peer_oids); + + IF overlap IS NOT NULL THEN + RAISE EXCEPTION 'cannot migrate: % OID(s) are also held natively by another node', + array_length(overlap, 1) + USING DETAIL = format('Colliding OID(s): %s', overlap), + HINT = 'Native OIDs are not node-encoded. Re-create the colliding objects ' + 'under fresh OIDs on one of the nodes before migrating'; + END IF; + END IF; + + repair_enabled := lolor._migration_guard( + false, + 'Drop the offending logical replication slots in this database and retry'); + + IF repair_enabled IS NULL THEN + RETURN -1; + END IF; + + PERFORM lolor.migrate_storage(false); + + /* + * Re-enable replication for the remainder of the caller's transaction, so + * repair mode covers exactly the migration and nothing after it. Error + * paths need no cleanup: they abort the whole transaction. + */ + IF repair_enabled THEN + PERFORM spock.repair_mode(false); + END IF; + + RETURN lo_count; +END; +$$ LANGUAGE plpgsql VOLATILE; + +/* + * lolor.migrate_to_native() + * + * Move every large object from lolor storage back into native storage. + * + * Called automatically by the drop event trigger, and safe to invoke by hand. + * Unlike previous versions this does not require lolor to be enabled: the + * movement is performed directly against the catalogs and never calls the + * renamed _orig functions. + * + * Failure is a hard ERROR, not a soft return: this runs on the DROP path, + * where silently losing large objects is far worse than a failed DROP. + */ +CREATE OR REPLACE FUNCTION lolor.migrate_to_native() +RETURNS bigint AS $$ +DECLARE + lo_count bigint; + repair_enabled boolean; +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = current_user AND rolsuper) THEN + RAISE EXCEPTION 'lolor.migrate_to_native() requires superuser privileges'; + END IF; + + SELECT count(*) INTO lo_count FROM lolor.pg_largeobject_metadata; + + IF lo_count = 0 THEN + RAISE NOTICE 'no lolor large objects to migrate'; + RETURN 0; + END IF; + + repair_enabled := lolor._migration_guard( + true, + 'Drop the offending logical replication slots in this database and retry'); + + PERFORM lolor.migrate_storage(true); + + IF repair_enabled THEN + PERFORM spock.repair_mode(false); + END IF; + + RETURN lo_count; +END; +$$ LANGUAGE plpgsql VOLATILE; + +REVOKE ALL ON FUNCTION lolor.migrate_from_native(oid[]) FROM PUBLIC; +REVOKE ALL ON FUNCTION lolor.migrate_to_native() FROM PUBLIC; + +/* + * --------------------------------------------------------------------------- + * Verification helpers + * --------------------------------------------------------------------------- + */ + +/* + * Native large object OIDs held by this node, for the cross-node pre-flight + * of lolor.migrate_from_native(). + */ +CREATE FUNCTION lolor.native_lo_oids() +RETURNS SETOF oid AS $$ + SELECT oid FROM pg_catalog.pg_largeobject_metadata ORDER BY oid +$$ LANGUAGE sql STABLE; + +/* + * Per-object digest of lolor storage, so that convergence across nodes can be + * checked instead of assumed. Compare the output on every node after + * migrating: because the movement is hidden from replication, divergence is + * otherwise silent until a later conflict. + * + * This reads every page and is deliberately not cheap; run it as a check, not + * on a schedule. + */ +CREATE FUNCTION lolor.digest() +RETURNS TABLE (loid oid, lomowner name, npages bigint, nbytes bigint, digest text) +AS $$ + SELECT m.oid, + pg_catalog.pg_get_userbyid(m.lomowner), + count(d.pageno), + coalesce(sum(length(d.data)), 0), + md5(coalesce(string_agg(md5(d.data), ',' ORDER BY d.pageno), '')) + FROM lolor.pg_largeobject_metadata m + LEFT JOIN lolor.pg_largeobject d ON d.loid = m.oid + GROUP BY m.oid, m.lomowner + ORDER BY m.oid +$$ LANGUAGE sql STABLE; + +/* + * Large objects in lolor storage whose owner no longer exists. + * + * Objects in lolor storage are rows in ordinary tables, so they cannot + * participate in pg_shdepend: DROP ROLE will not notice them the way it + * notices native large objects. This is inherent to storing them outside the + * catalogs. This function makes the consequence findable. + */ +CREATE FUNCTION lolor.check_orphans() +RETURNS TABLE (loid oid, lomowner oid) AS $$ + SELECT m.oid, m.lomowner + FROM lolor.pg_largeobject_metadata m + LEFT JOIN pg_catalog.pg_authid a ON a.oid = m.lomowner + WHERE a.oid IS NULL + ORDER BY m.oid +$$ LANGUAGE sql STABLE; + +/* + * --------------------------------------------------------------------------- + * Hardened enable / disable / is_enabled + * --------------------------------------------------------------------------- + */ + +CREATE OR REPLACE FUNCTION lolor.is_enabled() +RETURNS boolean AS $$ +DECLARE + parked_present boolean; + orig_present boolean; +BEGIN + -- Exact signatures, qualified to pg_catalog: see the note in lolor.enable(). + parked_present := to_regprocedure('pg_catalog.lolor_lo_open(oid,int4)') IS NOT NULL; + orig_present := to_regprocedure('pg_catalog.lo_open_orig(oid,int4)') IS NOT NULL; + + IF parked_present = orig_present THEN + RAISE EXCEPTION 'lolor is in inconsistent state' + USING DETAIL = format('pg_catalog.lolor_lo_open present: %s; pg_catalog.lo_open_orig present: %s', + parked_present, orig_present); + END IF; + + -- Our functions parked under lolor_* means the native ones are in place. + RETURN NOT parked_present; +END; +$$ LANGUAGE plpgsql STRICT STABLE; + + +/* + * Disable lolor functionality. + * + * Parks the lolor implementations under pg_catalog.lolor_* and restores the + * native pg_catalog names from their *_orig parking spot. Creates and drops + * nothing. Returns true on success, false on a handled no-op. + */ +CREATE OR REPLACE FUNCTION lolor.disable() +RETURNS boolean AS $$ +BEGIN + -- Serialise against a concurrent enable()/disable(). The probes below + -- are a check-then-act and the renames are not atomic on their own. + PERFORM pg_catalog.pg_advisory_xact_lock(4919420001); + + -- Probe exact signatures in pg_catalog. Earlier versions matched on + -- proname alone across every schema, which let any user with CREATE on any + -- schema squat a name such as 'lolor_lo_open' and wedge lolor into a + -- permanent 'inconsistent state' -- including blocking DROP EXTENSION. + IF to_regprocedure('pg_catalog.lo_close_orig(int4)') IS NULL THEN + RAISE NOTICE 'lolor is already disabled'; + RETURN false; + END IF; + IF to_regprocedure('pg_catalog.lolor_lo_open(oid,int4)') IS NOT NULL THEN + RAISE NOTICE 'lolor.disable() has been called before'; + RETURN false; + END IF; + + ALTER FUNCTION pg_catalog.lo_open(oid, int4) RENAME TO lolor_lo_open; + ALTER FUNCTION pg_catalog.lo_open_orig(oid, int4) RENAME TO lo_open; + ALTER FUNCTION pg_catalog.lo_close(int4) RENAME TO lolor_lo_close; + ALTER FUNCTION pg_catalog.lo_close_orig(int4) RENAME TO lo_close; + ALTER FUNCTION pg_catalog.lo_creat(integer) RENAME TO lolor_lo_creat; + ALTER FUNCTION pg_catalog.lo_creat_orig(integer) RENAME TO lo_creat; + ALTER FUNCTION pg_catalog.lo_create(oid) RENAME TO lolor_lo_create; + ALTER FUNCTION pg_catalog.lo_create_orig(oid) RENAME TO lo_create; + ALTER FUNCTION pg_catalog.loread(integer, integer) RENAME TO lolor_loread; + ALTER FUNCTION pg_catalog.loread_orig(integer, integer) RENAME TO loread; + ALTER FUNCTION pg_catalog.lowrite(integer, bytea) RENAME TO lolor_lowrite; + ALTER FUNCTION pg_catalog.lowrite_orig(integer, bytea) RENAME TO lowrite; + ALTER FUNCTION pg_catalog.lo_export(oid, text) RENAME TO lolor_lo_export; + ALTER FUNCTION pg_catalog.lo_export_orig(oid, text) RENAME TO lo_export; + ALTER FUNCTION pg_catalog.lo_from_bytea(oid, bytea) RENAME TO lolor_lo_from_bytea; + ALTER FUNCTION pg_catalog.lo_from_bytea_orig(oid, bytea) RENAME TO lo_from_bytea; + ALTER FUNCTION pg_catalog.lo_get(oid) RENAME TO lolor_lo_get; + ALTER FUNCTION pg_catalog.lo_get_orig(oid) RENAME TO lo_get; + ALTER FUNCTION pg_catalog.lo_get(oid, bigint, integer) RENAME TO lolor_lo_get; + ALTER FUNCTION pg_catalog.lo_get_orig(oid, bigint, integer) RENAME TO lo_get; + ALTER FUNCTION pg_catalog.lo_import(text) RENAME TO lolor_lo_import; + ALTER FUNCTION pg_catalog.lo_import_orig(text) RENAME TO lo_import; + ALTER FUNCTION pg_catalog.lo_import(text, oid) RENAME TO lolor_lo_import; + ALTER FUNCTION pg_catalog.lo_import_orig(text, oid) RENAME TO lo_import; + ALTER FUNCTION pg_catalog.lo_lseek(integer, integer, integer) RENAME TO lolor_lo_lseek; + ALTER FUNCTION pg_catalog.lo_lseek_orig(integer, integer, integer) RENAME TO lo_lseek; + ALTER FUNCTION pg_catalog.lo_lseek64(integer, bigint, integer) RENAME TO lolor_lo_lseek64; + ALTER FUNCTION pg_catalog.lo_lseek64_orig(integer, bigint, integer) RENAME TO lo_lseek64; + ALTER FUNCTION pg_catalog.lo_put(oid, bigint, bytea) RENAME TO lolor_lo_put; + ALTER FUNCTION pg_catalog.lo_put_orig(oid, bigint, bytea) RENAME TO lo_put; + ALTER FUNCTION pg_catalog.lo_tell(integer) RENAME TO lolor_lo_tell; + ALTER FUNCTION pg_catalog.lo_tell_orig(integer) RENAME TO lo_tell; + ALTER FUNCTION pg_catalog.lo_tell64(integer) RENAME TO lolor_lo_tell64; + ALTER FUNCTION pg_catalog.lo_tell64_orig(integer) RENAME TO lo_tell64; + ALTER FUNCTION pg_catalog.lo_truncate(integer, integer) RENAME TO lolor_lo_truncate; + ALTER FUNCTION pg_catalog.lo_truncate_orig(integer, integer) RENAME TO lo_truncate; + ALTER FUNCTION pg_catalog.lo_truncate64(integer, bigint) RENAME TO lolor_lo_truncate64; + ALTER FUNCTION pg_catalog.lo_truncate64_orig(integer, bigint) RENAME TO lo_truncate64; + ALTER FUNCTION pg_catalog.lo_unlink(oid) RENAME TO lolor_lo_unlink; + ALTER FUNCTION pg_catalog.lo_unlink_orig(oid) RENAME TO lo_unlink; + + -- Renaming changes which OID owns the name lo_open. libpq resolves the + -- large object fastpath OIDs once per connection and caches them, so + -- sessions that touched a large object before this call keep calling the + -- previous implementation until they reconnect. + RAISE NOTICE 'lolor: reconnect existing client sessions; they cache large object function OIDs'; + + RETURN true; +END; +$$ LANGUAGE plpgsql STRICT VOLATILE; + + +/* + * Enable lolor functionality, undoing lolor.disable(). + */ +CREATE OR REPLACE FUNCTION lolor.enable() +RETURNS boolean AS $$ +BEGIN + -- Serialise against a concurrent enable()/disable(). The probes below + -- are a check-then-act and the renames are not atomic on their own. + PERFORM pg_catalog.pg_advisory_xact_lock(4919420001); + + -- Probe exact signatures in pg_catalog. Earlier versions matched on + -- proname alone across every schema, which let any user with CREATE on any + -- schema squat a name such as 'lolor_lo_open' and wedge lolor into a + -- permanent 'inconsistent state' -- including blocking DROP EXTENSION. + IF to_regprocedure('pg_catalog.lolor_lo_open(oid,int4)') IS NULL THEN + RAISE NOTICE 'lolor is already enabled'; + RETURN false; + END IF; + IF to_regprocedure('pg_catalog.lo_close_orig(int4)') IS NOT NULL THEN + RAISE NOTICE 'lolor.enable() has been called before'; + RETURN false; + END IF; + + ALTER FUNCTION pg_catalog.lo_open(oid, int4) RENAME TO lo_open_orig; + ALTER FUNCTION pg_catalog.lolor_lo_open(oid, int4) RENAME TO lo_open; + ALTER FUNCTION pg_catalog.lo_close(int4) RENAME TO lo_close_orig; + ALTER FUNCTION pg_catalog.lolor_lo_close(int4) RENAME TO lo_close; + ALTER FUNCTION pg_catalog.lo_creat(integer) RENAME TO lo_creat_orig; + ALTER FUNCTION pg_catalog.lolor_lo_creat(integer) RENAME TO lo_creat; + ALTER FUNCTION pg_catalog.lo_create(oid) RENAME TO lo_create_orig; + ALTER FUNCTION pg_catalog.lolor_lo_create(oid) RENAME TO lo_create; + ALTER FUNCTION pg_catalog.loread(integer, integer) RENAME TO loread_orig; + ALTER FUNCTION pg_catalog.lolor_loread(integer, integer) RENAME TO loread; + ALTER FUNCTION pg_catalog.lowrite(integer, bytea) RENAME TO lowrite_orig; + ALTER FUNCTION pg_catalog.lolor_lowrite(integer, bytea) RENAME TO lowrite; + ALTER FUNCTION pg_catalog.lo_export(oid, text) RENAME TO lo_export_orig; + ALTER FUNCTION pg_catalog.lolor_lo_export(oid, text) RENAME TO lo_export; + ALTER FUNCTION pg_catalog.lo_from_bytea(oid, bytea) RENAME TO lo_from_bytea_orig; + ALTER FUNCTION pg_catalog.lolor_lo_from_bytea(oid, bytea) RENAME TO lo_from_bytea; + ALTER FUNCTION pg_catalog.lo_get(oid) RENAME TO lo_get_orig; + ALTER FUNCTION pg_catalog.lolor_lo_get(oid) RENAME TO lo_get; + ALTER FUNCTION pg_catalog.lo_get(oid, bigint, integer) RENAME TO lo_get_orig; + ALTER FUNCTION pg_catalog.lolor_lo_get(oid, bigint, integer) RENAME TO lo_get; + ALTER FUNCTION pg_catalog.lo_import(text) RENAME TO lo_import_orig; + ALTER FUNCTION pg_catalog.lolor_lo_import(text) RENAME TO lo_import; + ALTER FUNCTION pg_catalog.lo_import(text, oid) RENAME TO lo_import_orig; + ALTER FUNCTION pg_catalog.lolor_lo_import(text, oid) RENAME TO lo_import; + ALTER FUNCTION pg_catalog.lo_lseek(integer, integer, integer) RENAME TO lo_lseek_orig; + ALTER FUNCTION pg_catalog.lolor_lo_lseek(integer, integer, integer) RENAME TO lo_lseek; + ALTER FUNCTION pg_catalog.lo_lseek64(integer, bigint, integer) RENAME TO lo_lseek64_orig; + ALTER FUNCTION pg_catalog.lolor_lo_lseek64(integer, bigint, integer) RENAME TO lo_lseek64; + ALTER FUNCTION pg_catalog.lo_put(oid, bigint, bytea) RENAME TO lo_put_orig; + ALTER FUNCTION pg_catalog.lolor_lo_put(oid, bigint, bytea) RENAME TO lo_put; + ALTER FUNCTION pg_catalog.lo_tell(integer) RENAME TO lo_tell_orig; + ALTER FUNCTION pg_catalog.lolor_lo_tell(integer) RENAME TO lo_tell; + ALTER FUNCTION pg_catalog.lo_tell64(integer) RENAME TO lo_tell64_orig; + ALTER FUNCTION pg_catalog.lolor_lo_tell64(integer) RENAME TO lo_tell64; + ALTER FUNCTION pg_catalog.lo_truncate(integer, integer) RENAME TO lo_truncate_orig; + ALTER FUNCTION pg_catalog.lolor_lo_truncate(integer, integer) RENAME TO lo_truncate; + ALTER FUNCTION pg_catalog.lo_truncate64(integer, bigint) RENAME TO lo_truncate64_orig; + ALTER FUNCTION pg_catalog.lolor_lo_truncate64(integer, bigint) RENAME TO lo_truncate64; + ALTER FUNCTION pg_catalog.lo_unlink(oid) RENAME TO lo_unlink_orig; + ALTER FUNCTION pg_catalog.lolor_lo_unlink(oid) RENAME TO lo_unlink; + + -- Renaming changes which OID owns the name lo_open. libpq resolves the + -- large object fastpath OIDs once per connection and caches them, so + -- sessions that touched a large object before this call keep calling the + -- previous implementation until they reconnect. + RAISE NOTICE 'lolor: reconnect existing client sessions; they cache large object function OIDs'; + + RETURN true; +END; +$$ LANGUAGE plpgsql STRICT VOLATILE; + + +/* + * --------------------------------------------------------------------------- + * Drop cleanup + * --------------------------------------------------------------------------- + * + * The extension does not only go away through DROP EXTENSION: DROP SCHEMA + * lolor CASCADE and DROP OWNED BY reach it by dependency + * cascade. Those fire under their own command tags, so the trigger has to be + * registered for them too -- otherwise the cleanup never runs, the large + * objects are destroyed with the lolor tables, and pg_catalog is left without + * a working lo_open(). Tags cannot be altered in place, so re-create it. + */ +DROP EVENT TRIGGER lo_on_drop_extension; +CREATE EVENT TRIGGER lo_on_drop_extension + ON ddl_command_start + WHEN tag IN ('DROP EXTENSION', 'DROP SCHEMA', 'DROP OWNED') + EXECUTE FUNCTION pg_catalog.lo_on_drop_extension(); +ALTER EVENT TRIGGER lo_on_drop_extension ENABLE ALWAYS; + +/* + * --------------------------------------------------------------------------- + * Restore the ACLs on the server-side file access functions + * --------------------------------------------------------------------------- + * + * pg_catalog.lo_import() and lo_export() read and write files on the server as + * the operating system account PostgreSQL runs under, so core revokes EXECUTE + * on them from PUBLIC. + * + * lolor replaces them by renaming the originals to *_orig and creating its own + * versions. An ACL belongs to a function, not to a name, so the restriction + * stayed behind on the parked originals while the replacements were created + * with the default: EXECUTE granted to PUBLIC. Any database user could + * therefore call lo_import('/etc/passwd') or overwrite a file with + * lo_export(). Installations created before 1.4.0 are affected whether or not + * lolor is currently enabled. + * + * Revoke on both spellings so the fix lands regardless of which state the + * installation is in. Revoking on a native function that is already + * restricted is a no-op. + */ +DO $$ +DECLARE + target text; +BEGIN + FOREACH target IN ARRAY ARRAY[ + 'lo_import(text)', + 'lo_import(text,oid)', + 'lo_export(oid,text)', + 'lolor_lo_import(text)', + 'lolor_lo_import(text,oid)', + 'lolor_lo_export(oid,text)' + ] + LOOP + IF to_regprocedure('pg_catalog.' || target) IS NOT NULL THEN + EXECUTE format('REVOKE ALL ON FUNCTION pg_catalog.%s FROM PUBLIC', target); + END IF; + END LOOP; +END; +$$; + +/* + * --------------------------------------------------------------------------- + * Remove bogus pg_shdepend rows left by earlier versions + * --------------------------------------------------------------------------- + * + * Through 1.3.0, creating a large object recorded a pg_shdepend row whose + * classId was the OID of lolor.pg_largeobject -- an ordinary table, not a + * catalog the dependency machinery can describe. DROP ROLE on any role that + * had created a large object failed with "unrecognized object class", and the + * rows were never removed because inv_drop() deletes with + * PERFORM_DELETION_SKIP_ORIGINAL. lolor no longer records them; delete the + * ones already there. + * + * pg_shdepend is shared across the cluster, so restrict the delete to this + * database: the same classId value in another database refers to some + * unrelated relation. + */ +DELETE FROM pg_catalog.pg_shdepend +WHERE dbid = (SELECT oid FROM pg_catalog.pg_database + WHERE datname = current_database()) + AND classid IN ('lolor.pg_largeobject'::regclass, + 'lolor.pg_largeobject_metadata'::regclass); diff --git a/lolor.control b/lolor.control index ceb5dcc..52fc3dd 100644 --- a/lolor.control +++ b/lolor.control @@ -1,7 +1,9 @@ # lolor extension comment = 'Large Objects support for logical replication' -default_version = '1.3.0' +default_version = '1.4.0' module_pathname = '$libdir/lolor' relocatable = false -trusted = true +# Not trusted: installing lolor renames functions in pg_catalog for the whole +# database, which is not something a non-superuser should be able to do. +trusted = false schema = lolor diff --git a/sql/lolor.sql b/sql/lolor.sql index d2c7509..7ad84de 100644 --- a/sql/lolor.sql +++ b/sql/lolor.sql @@ -142,15 +142,19 @@ DROP EXTENSION lolor; SELECT oid, proname FROM pg_proc WHERE proname IN ('lo_open_orig', 'lolor_lo_open'); --- Check: we can't just delete LOLOR without LO migration in disabled mode. --- XXX: should we introduce a 'forced' flag to allow this? +-- DROP EXTENSION while lolor is disabled. Through 1.3.0 this failed with +-- "lolor must be enabled before migration to native", because the reverse +-- migration ran through the renamed _orig functions. It now works against +-- the catalogs directly, so the disabled state is no longer a special case +-- and the objects are still rescued. CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'stored before disabling') AS disabled_drop_oid \gset SELECT lolor.disable(); DROP EXTENSION lolor; -SELECT extname FROM pg_extension; -- lolor is here -SELECT lolor.enable(); -DROP EXTENSION lolor; SELECT extname FROM pg_extension; -- check lolor removal +-- The object was migrated to native storage, not dropped with lolor's tables +SELECT convert_from(lo_get(:disabled_drop_oid), 'UTF8') AS survived_disabled_drop; +SELECT lo_unlink(:disabled_drop_oid); -- -- Migration tests: migrate_from_native / migrate_to_native / DROP EXTENSION @@ -269,7 +273,8 @@ CREATE EXTENSION lolor; SELECT lo_from_bytea(0, 'Drop conflict test') AS drop_conflict_oid \gset -- Create a native LO with the same OID to force conflict at DROP time SELECT lolor.disable(); -SELECT lo_create(:'drop_conflict_oid'); +-- Print a stable boolean rather than the generated OID, which varies per run +SELECT lo_create(:'drop_conflict_oid') = :'drop_conflict_oid'::oid AS native_oid_honored; SELECT lolor.enable(); -- DROP EXTENSION should ERROR to prevent data loss DROP EXTENSION lolor; @@ -283,3 +288,452 @@ SELECT lo_unlink(:'drop_conflict_oid'::oid); SELECT lolor.enable(); -- Now DROP should succeed DROP EXTENSION lolor; + +-- +-- Fidelity of the storage relocation (lolor 1.4.0) +-- +CREATE EXTENSION lolor; +CREATE ROLE lolor_owner; +CREATE ROLE lolor_grantee; + +-- A sparse object: two pages holding a 10 MB logical object. Migration must +-- not fill the hole; the old implementation rewrote it through the LO API and +-- materialised every intervening page. +SELECT lolor.disable(); +SELECT lo_create(0) AS sparse_oid \gset +BEGIN; +SELECT lo_open(:sparse_oid, x'60000'::int) AS fd \gset +SELECT lowrite(:fd, 'start'); +SELECT lo_lseek64(:fd, 10000000, 0); +SELECT lowrite(:fd, 'end'); +SELECT lo_close(:fd); +END; + +-- An object carrying owner, ACL and a comment +SELECT lo_from_bytea(0, 'annotated object') AS annotated_oid \gset +ALTER LARGE OBJECT :annotated_oid OWNER TO lolor_owner; +GRANT SELECT ON LARGE OBJECT :annotated_oid TO lolor_grantee; +COMMENT ON LARGE OBJECT :annotated_oid IS 'kept across migration'; + +-- An object with no data pages at all +SELECT lo_create(0) AS empty_oid \gset + +SELECT lolor.enable(); +SELECT lolor.migrate_from_native(); + +-- Sparse object keeps its exact page count (2), not 4883 +SELECT count(*) AS sparse_pages FROM lolor.pg_largeobject WHERE loid = :sparse_oid; +SELECT lo_get(:sparse_oid) IS NOT NULL AS sparse_readable; +SELECT length(lo_get(:sparse_oid)) AS sparse_length; +SELECT length(lo_get(:empty_oid)) AS empty_length; + +-- Owner and ACL survive; the comment is parked for the round trip +SELECT pg_get_userbyid(lomowner) AS owner, lomacl IS NOT NULL AS has_acl +FROM lolor.pg_largeobject_metadata WHERE oid = :annotated_oid; +SELECT description FROM lolor.pg_largeobject_description WHERE loid = :annotated_oid; + +-- Native side is fully cleaned up, including shared deps and comments +SELECT count(*) AS native_objs FROM pg_catalog.pg_largeobject_metadata; +SELECT count(*) AS native_shdep FROM pg_shdepend + WHERE classid = 'pg_largeobject'::regclass + AND dbid = (SELECT oid FROM pg_database WHERE datname = current_database()); +SELECT count(*) AS native_comments FROM pg_description + WHERE classoid = 'pg_largeobject'::regclass; + +-- Back to native storage +SELECT lolor.migrate_to_native(); + +-- Ownership is recorded in pg_shdepend, not merely in lomowner: a raw catalog +-- UPDATE (what 1.3.0 did) left DROP ROLE unable to see the object. +SELECT pg_get_userbyid(lomowner) AS owner +FROM pg_catalog.pg_largeobject_metadata WHERE oid = :annotated_oid; +SELECT deptype, refobjid::regrole::text AS role FROM pg_shdepend + WHERE classid = 'pg_largeobject'::regclass AND objid = :annotated_oid + AND dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + ORDER BY deptype; +SELECT description FROM pg_description + WHERE classoid = 'pg_largeobject'::regclass AND objoid = :annotated_oid; + +-- DROP ROLE must refuse for both the owner and the ACL grantee +DROP ROLE lolor_owner; +DROP ROLE lolor_grantee; + +-- Sparse object is still sparse after the round trip +SELECT count(*) AS sparse_pages_native FROM pg_catalog.pg_largeobject + WHERE loid = :sparse_oid; + +-- Comment parking table is emptied once the comments are reinstated +SELECT count(*) AS parked_left FROM lolor.pg_largeobject_description; + +-- Cleanup +SELECT lolor.disable(); +SELECT lo_unlink(:sparse_oid); +SELECT lo_unlink(:annotated_oid); +SELECT lo_unlink(:empty_oid); +SELECT lolor.enable(); +DROP EXTENSION lolor; +DROP ROLE lolor_owner; +DROP ROLE lolor_grantee; + +-- +-- An unprivileged user must not be able to wedge lolor by squatting the +-- names the state probes look for. Before 1.4.0 this made is_enabled() +-- raise "inconsistent state", which also blocked DROP EXTENSION. +-- +CREATE EXTENSION lolor; +CREATE ROLE lolor_squatter; +GRANT CREATE ON SCHEMA public TO lolor_squatter; +SET ROLE lolor_squatter; +CREATE FUNCTION public.lolor_lo_open(oid, int4) RETURNS int4 + AS 'SELECT 1' LANGUAGE sql; +CREATE FUNCTION public.lo_close_orig(int4) RETURNS int4 + AS 'SELECT 1' LANGUAGE sql; +RESET ROLE; +SELECT lolor.is_enabled() AS unaffected_by_squatting; +DROP FUNCTION public.lolor_lo_open(oid, int4); +DROP FUNCTION public.lo_close_orig(int4); +REVOKE CREATE ON SCHEMA public FROM lolor_squatter; +DROP ROLE lolor_squatter; +DROP EXTENSION lolor; + +-- +-- DROP SCHEMA lolor CASCADE reaches the extension by dependency cascade +-- rather than as DROP EXTENSION. The cleanup trigger must still run, or the +-- objects are destroyed and pg_catalog is left without a working lo_open(). +-- +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'rescued from drop schema') AS rescued_oid \gset +DROP SCHEMA lolor CASCADE; +SELECT count(*) AS ext_left FROM pg_extension WHERE extname = 'lolor'; +SELECT to_regprocedure('pg_catalog.lo_open(oid,int4)') IS NOT NULL AS lo_open_restored; +SELECT to_regprocedure('pg_catalog.lo_open_orig(oid,int4)') IS NULL AS no_orig_left; +SELECT convert_from(lo_get(:rescued_oid), 'UTF8') AS rescued_content; +SELECT lo_unlink(:rescued_oid); + +-- +-- lo_import() and lo_export() read and write files on the server, so core +-- revokes EXECUTE on them from PUBLIC. lolor replaces them by renaming the +-- originals out of the way, and an ACL belongs to a function rather than to a +-- name: the restriction stays on the parked original, and the replacement is +-- created with the default (EXECUTE TO PUBLIC) unless locked down explicitly. +-- Before 1.4.0 that let any database user read or overwrite server files. +-- +CREATE EXTENSION lolor; +SELECT r.proname || '(' || pg_get_function_arguments(r.oid) || ')' AS func, + EXISTS (SELECT 1 FROM aclexplode(coalesce(r.proacl, acldefault('f', r.proowner))) a + WHERE a.grantee = 0) AS replacement_public_execute, + EXISTS (SELECT 1 FROM aclexplode(coalesce(o.proacl, acldefault('f', o.proowner))) a + WHERE a.grantee = 0) AS original_public_execute +FROM pg_proc r +JOIN pg_namespace n ON n.oid = r.pronamespace AND n.nspname = 'pg_catalog' +JOIN pg_proc o ON o.pronamespace = r.pronamespace + AND o.proname = r.proname || '_orig' + AND o.proargtypes = r.proargtypes +WHERE r.proname IN ('lo_import', 'lo_export') +ORDER BY 1; +DROP EXTENSION lolor; + +-- +-- lolor.node is bounded by the OID encoding, not by an independently written +-- constant: the low LOLOR_NODEID_BITS of a generated OID carry the node id, so +-- 16 does not fit and was previously accepted while encoding as node 0. +-- +SET lolor.node = 16; +SET lolor.node = 15; +SET lolor.node = 1; + +-- +-- Permission enforcement. +-- +-- Objects in lolor storage have no catalog entry, so lolor cannot use the +-- syscache-backed owner and ACL checks and reimplements them against its own +-- tables. Exercise that path rather than assuming it matches core. +-- +CREATE EXTENSION lolor; +CREATE ROLE lolor_alice; +CREATE ROLE lolor_bob; + +-- Large object error messages quote the OID, which is generated and so +-- differs between runs. Report the message with digits masked instead. +CREATE FUNCTION lolor_expect_error(cmd text) RETURNS text AS $$ +BEGIN + EXECUTE cmd; + RETURN 'unexpectedly succeeded'; +EXCEPTION WHEN OTHERS THEN + RETURN regexp_replace(SQLERRM, '[0-9]+', 'NNN', 'g'); +END +$$ LANGUAGE plpgsql; + +SET ROLE lolor_alice; +SELECT lo_from_bytea(0, 'alice private data') AS alice_oid \gset +SELECT convert_from(lo_get(:alice_oid), 'UTF8') AS owner_can_read; +RESET ROLE; + +-- A different role gets nothing without a grant. +SET ROLE lolor_bob; +SELECT lolor_expect_error(format('SELECT lo_get(%s)', :alice_oid)) AS read_denied; +SELECT lolor_expect_error(format('SELECT lo_open(%s, 262144)', :alice_oid)) AS open_denied; +SELECT lolor_expect_error(format('SELECT lo_put(%s, 0, ''x'')', :alice_oid)) AS write_denied; +SELECT lolor_expect_error(format('SELECT lo_unlink(%s)', :alice_oid)) AS unlink_denied; +RESET ROLE; + +-- The superuser bypasses the check, as in core. +SELECT convert_from(lo_get(:alice_oid), 'UTF8') AS superuser_can_read; + +-- GRANT/ALTER on a large object act on pg_largeobject_metadata, where an +-- object in lolor storage has no row. This limitation is documented; assert +-- it so that a change in behaviour is noticed. +SELECT lolor_expect_error( + format('GRANT SELECT ON LARGE OBJECT %s TO lolor_bob', :alice_oid)) AS grant_unsupported; +SELECT lolor_expect_error( + format('ALTER LARGE OBJECT %s OWNER TO lolor_bob', :alice_oid)) AS alter_unsupported; + +SELECT lo_unlink(:alice_oid); + +-- +-- Objects in lolor storage are rows in ordinary tables and cannot participate +-- in pg_shdepend, so DROP ROLE does not notice that a role still owns one. +-- lolor.check_orphans() exists to make the consequence findable. +-- +SET ROLE lolor_alice; +SELECT lo_from_bytea(0, 'owned by a role about to vanish') AS orphan_oid \gset +RESET ROLE; +SELECT count(*) AS orphans_before FROM lolor.check_orphans(); +DROP ROLE lolor_alice; +SELECT count(*) AS orphans_after FROM lolor.check_orphans(); +SELECT lo_unlink(:orphan_oid); +SELECT count(*) AS orphans_cleared FROM lolor.check_orphans(); +DROP ROLE lolor_bob; +DROP FUNCTION lolor_expect_error(text); + +-- +-- 64-bit interface and page-boundary I/O. lo_put(), lo_tell64() and +-- lo_truncate64() had no coverage at all. +-- +SELECT current_setting('block_size')::int / 4 AS loblksize \gset + +-- Write straddling a page boundary, then read the fragment back. +SELECT lo_create(0) AS span_oid \gset +SELECT lo_put(:span_oid, (:loblksize - 4)::bigint, '\x4142434445464748'::bytea); +SELECT length(lo_get(:span_oid)) = :loblksize + 4 AS spans_two_pages; +SELECT count(*) = 2 AS two_data_pages FROM lolor.pg_largeobject WHERE loid = :span_oid; +SELECT encode(lo_get(:span_oid, (:loblksize - 4)::bigint, 8), 'hex') AS across_boundary; + +-- lo_truncate64() extending past the end leaves a hole rather than pages. +BEGIN; +SELECT lo_open(:span_oid, x'60000'::int) AS fd \gset +SELECT lo_truncate64(:fd, (:loblksize * 4)::bigint); +SELECT lo_lseek64(:fd, 0, 2) = (:loblksize * 4)::bigint AS seek_end_matches; +SELECT lo_tell64(:fd) = (:loblksize * 4)::bigint AS tell64_matches; +SELECT lo_close(:fd); +END; +SELECT length(lo_get(:span_oid)) = :loblksize * 4 AS truncate64_extended; +SELECT count(*) < 4 AS hole_not_materialised + FROM lolor.pg_largeobject WHERE loid = :span_oid; + +-- Truncating back down releases the pages beyond the new length. +BEGIN; +SELECT lo_open(:span_oid, x'60000'::int) AS fd \gset +SELECT lo_truncate64(:fd, 10); +SELECT lo_close(:fd); +END; +SELECT length(lo_get(:span_oid)) AS len_after_shrink; +SELECT lo_unlink(:span_oid); + +-- +-- Subtransaction cleanup. A descriptor opened inside an aborted +-- subtransaction must be closed by the rollback, one opened in the enclosing +-- transaction must survive it, and data written in the aborted +-- subtransaction must not persist. +-- +BEGIN; +SELECT lo_from_bytea(0, 'outer') AS sub_oid \gset +SELECT lo_open(:sub_oid, x'60000'::int) AS outer_fd \gset +SAVEPOINT s1; +SELECT lo_open(:sub_oid, x'60000'::int) AS inner_fd \gset +SELECT lo_put(:sub_oid, 0, 'INNER'); +ROLLBACK TO s1; +SAVEPOINT s2; +SELECT lo_tell(:inner_fd); +ROLLBACK TO s2; +SELECT lo_tell(:outer_fd) AS outer_descriptor_survives; +SELECT lo_close(:outer_fd); +COMMIT; +SELECT convert_from(lo_get(:sub_oid), 'UTF8') AS subxact_write_rolled_back; +SELECT lo_unlink(:sub_oid); + +-- +-- A rolled back transaction must leave no trace in lolor storage. +-- +SELECT count(*) AS rows_before FROM lolor.pg_largeobject_metadata; +BEGIN; +SELECT lo_from_bytea(0, 'discarded') IS NOT NULL AS created_in_aborted_xact; +ROLLBACK; +SELECT count(*) AS rows_after FROM lolor.pg_largeobject_metadata; + +-- +-- Seek variants and read/write edge cases. +-- +SELECT lo_from_bytea(0, '0123456789abcdef') AS seek_oid \gset +BEGIN; +SELECT lo_open(:seek_oid, x'60000'::int) AS fd \gset +SELECT lo_lseek(:fd, 4, 0) AS seek_set; +SELECT lo_lseek(:fd, 2, 1) AS seek_cur; +SELECT lo_lseek(:fd, -3, 2) AS seek_end; +SELECT lo_tell(:fd) AS tell_after_seeks; +SELECT convert_from(loread(:fd, 3), 'UTF8') AS read_tail; +-- A read at end of object returns nothing rather than failing. +SELECT length(loread(:fd, 100)) AS read_past_eof; +-- Zero-length read and empty write are both no-ops. +SELECT lo_lseek(:fd, 0, 0); +SELECT length(loread(:fd, 0)) AS zero_length_read; +SELECT lowrite(:fd, '') AS empty_write; +SELECT lo_close(:fd); +END; +SELECT length(lo_get(:seek_oid)) AS unchanged_length; +-- lo_get with a fragment length beyond the end is clamped, not an error. +SELECT convert_from(lo_get(:seek_oid, 10, 1000), 'UTF8') AS clamped_fragment; +SELECT lo_unlink(:seek_oid); + +-- +-- Reading a multi-page object back in chunks that do not align with the +-- page size exercises the page-assembly path in lolor_inv_read(). +-- +SELECT current_setting('block_size')::int / 4 AS loblksize \gset +SELECT lo_from_bytea(0, repeat('abcdefgh', (:loblksize * 3 / 8))::bytea) AS multi_oid \gset +SELECT length(lo_get(:multi_oid)) = :loblksize * 3 AS three_pages_written; +SELECT count(*) AS page_rows FROM lolor.pg_largeobject WHERE loid = :multi_oid; +BEGIN; +SELECT lo_open(:multi_oid, 262144) AS fd \gset +SELECT length(loread(:fd, 1000)) AS chunk1; +SELECT length(loread(:fd, 5000)) AS chunk2; +SELECT length(loread(:fd, 100000)) AS chunk_rest; +SELECT lo_close(:fd); +END; +SELECT md5(lo_get(:multi_oid)) = md5(repeat('abcdefgh', (:loblksize * 3 / 8))::bytea) + AS content_round_trips; +SELECT lo_unlink(:multi_oid); + +-- +-- Error paths. +-- +SELECT lo_get(0); +SELECT lo_unlink(0); +BEGIN; +SELECT lo_open(0, 262144); +ROLLBACK; + +-- +-- Verification helpers introduced in 1.4.0. +-- +-- Start from a clean slate so the counts below do not depend on what earlier +-- sections happened to leave behind. The notices carry those counts, so they +-- are suppressed for the duration. +SET client_min_messages = warning; +SELECT lolor.migrate_to_native() IS NOT NULL AS drained_to_native; +SELECT lolor.disable(); +DO $$ +DECLARE r record; +BEGIN + FOR r IN SELECT oid FROM pg_catalog.pg_largeobject_metadata LOOP + PERFORM lo_unlink(r.oid); + END LOOP; +END +$$; +SELECT lolor.enable(); +RESET client_min_messages; + +SELECT count(*) AS native_oids_when_empty FROM lolor.native_lo_oids(); + +-- Both directions report zero rather than failing when there is nothing. +SELECT lolor.migrate_to_native() AS nothing_to_move; +SELECT lolor.migrate_from_native() AS nothing_to_take; + +-- digest() reports one row per object. The OID and owner vary between runs; +-- the page count, byte count and content digest do not. +SELECT lo_from_bytea(0, 'digest one') AS d1 \gset +SELECT lo_from_bytea(0, 'digest two, longer') AS d2 \gset +SELECT npages, nbytes, digest FROM lolor.digest() ORDER BY nbytes, digest; +SELECT lo_unlink(:d1); +SELECT lo_unlink(:d2); + +-- migrate_storage() is the mechanism behind both migration functions. Grant +-- schema access so that the function's own privileges are what is exercised +-- rather than USAGE on the schema. +CREATE ROLE lolor_nosuper; +GRANT USAGE ON SCHEMA lolor TO lolor_nosuper; +SET ROLE lolor_nosuper; +SELECT lolor.migrate_storage(true); +SELECT lolor.migrate_from_native(); +SELECT lolor.migrate_to_native(); +RESET ROLE; +REVOKE USAGE ON SCHEMA lolor FROM lolor_nosuper; +DROP ROLE lolor_nosuper; + +-- Native OIDs are not node-encoded, so two nodes can hold different objects +-- under the same OID. Passing the peer OIDs makes that a refusal instead of +-- silent divergence once the migration is hidden from replication. +SELECT lolor.disable(); +SELECT lo_create(0) AS peer_oid \gset +SELECT lolor.enable(); +SELECT lolor.migrate_from_native(peer_oids => ARRAY[:peer_oid]::oid[]); + +-- Security labels cannot be represented in lolor storage and cannot be +-- reinstated without their provider, so migration refuses rather than +-- discarding them. +INSERT INTO pg_catalog.pg_seclabel (objoid, classoid, objsubid, provider, label) +VALUES (:peer_oid, 'pg_catalog.pg_largeobject'::regclass, 0, 'lolor_test', 'secret'); +SELECT lolor.migrate_from_native(); +DELETE FROM pg_catalog.pg_seclabel + WHERE classoid = 'pg_catalog.pg_largeobject'::regclass AND provider = 'lolor_test'; + +-- With the label gone the migration proceeds. +SELECT lolor.migrate_from_native() AS migrated; +SELECT lo_unlink(:peer_oid); +DROP EXTENSION lolor; + +-- +-- A parked comment must not outlive the object it describes. Object OIDs are +-- only checked against pg_largeobject_metadata when a new one is generated, so +-- a comment left behind by lo_unlink() would be handed to whatever object next +-- took that OID. +-- +CREATE EXTENSION lolor; +SELECT lolor.disable(); +SELECT lo_from_bytea(0, 'has a comment') AS commented_oid \gset +COMMENT ON LARGE OBJECT :commented_oid IS 'parked then orphaned'; +SELECT lolor.enable(); +SELECT lolor.migrate_from_native(); +SELECT count(*) AS parked FROM lolor.pg_largeobject_description + WHERE loid = :commented_oid; + +-- Unlinking must take the parked comment with it. +SELECT lo_unlink(:commented_oid); +SELECT count(*) AS parked_after_unlink FROM lolor.pg_largeobject_description + WHERE loid = :commented_oid; + +-- Re-create an object under the very same OID and send it back to native +-- storage. It must arrive with no comment. +SELECT lo_create(:commented_oid) = :commented_oid AS oid_reused; +SELECT lolor.migrate_to_native(); +SELECT count(*) AS inherited_comment FROM pg_description + WHERE classoid = 'pg_largeobject'::regclass AND objoid = :commented_oid; +SELECT lolor.disable(); +SELECT lo_unlink(:commented_oid); +SELECT lolor.enable(); + +-- +-- DROP SCHEMA without CASCADE is RESTRICT and cannot remove a schema that +-- still holds the extension's tables. The cleanup must not run for a command +-- that is going to be rejected, or it would migrate every large object and +-- take the storage locks only to have the work rolled back. +-- +SELECT lo_from_bytea(0, 'still here afterwards') AS kept_oid \gset +DROP SCHEMA lolor; +SELECT count(*) AS extension_still_installed + FROM pg_extension WHERE extname = 'lolor'; +SELECT convert_from(lo_get(:kept_oid), 'UTF8') AS object_untouched; +SELECT count(*) AS still_in_lolor_storage + FROM lolor.pg_largeobject_metadata WHERE oid = :kept_oid; +SELECT lo_unlink(:kept_oid); +DROP EXTENSION lolor; diff --git a/src/lolor.c b/src/lolor.c index cedf90b..6636e69 100644 --- a/src/lolor.c +++ b/src/lolor.c @@ -18,11 +18,18 @@ #include "fmgr.h" #include "access/xact.h" #include "catalog/namespace.h" +#include "catalog/pg_extension.h" #include "commands/event_trigger.h" +#include "commands/extension.h" #include "executor/spi.h" #include "nodes/parsenodes.h" #include "nodes/value.h" #include "nodes/print.h" +#include "access/genam.h" +#include "access/htup_details.h" +#include "access/table.h" +#include "utils/acl.h" +#include "utils/fmgroids.h" #include "utils/builtins.h" #include "utils/inval.h" #include "utils/guc.h" @@ -42,24 +49,46 @@ static Oid LOLOR_LargeObjectRelationId = InvalidOid; static Oid LOLOR_LargeObjectLOidPNIndexId = InvalidOid; static Oid LOLOR_LargeObjectMetadataRelationId = InvalidOid; static Oid LOLOR_LargeObjectMetadataOidIndexId = InvalidOid; +static Oid LOLOR_LargeObjectDescriptionRelationId = InvalidOid; +static Oid LOLOR_LargeObjectDescriptionIndexId = InvalidOid; PG_FUNCTION_INFO_V1(lolor_on_drop_extension); static Oid -get_lobj_table_oid(const char *table) +get_lobj_table_oid_extended(const char *table, bool missing_ok) { Oid reloid; Oid nspoid; nspoid = get_namespace_oid(EXTENSION_NAME, false); reloid = get_relname_relid(table, nspoid); - if (reloid == InvalidOid) + if (reloid == InvalidOid && !missing_ok) elog(ERROR, "cache lookup failed for relation %s.%s", EXTENSION_NAME, table); return reloid; } +static Oid +get_lobj_table_oid(const char *table) +{ + return get_lobj_table_oid_extended(table, false); +} + +/* + * Same, but returns InvalidOid when the relation is absent. + * + * The shared library is replaced before ALTER EXTENSION UPDATE runs, so a + * backend can be executing 1.4.0 code against a 1.3.0 schema in which + * lolor.pg_largeobject_description does not exist yet. Paths that run during + * ordinary large object activity have to tolerate that rather than fail. + */ +Oid +get_LOLOR_LargeObjectDescriptionRelationIdIfExists(void) +{ + return get_lobj_table_oid_extended(LOLOR_LARGEOBJECT_DESCRIPTION, true); +} + Oid get_LOLOR_LargeObjectRelationId() { @@ -96,6 +125,31 @@ get_LOLOR_LargeObjectMetadataOidIndexId() return LOLOR_LargeObjectMetadataOidIndexId; } +/* + * lolor.pg_largeobject_description parks COMMENT ON LARGE OBJECT text while an + * object lives in lolor storage, where there is no catalog entry for a comment + * to hang off. See lolor_migrate.c. + */ +Oid +get_LOLOR_LargeObjectDescriptionRelationId() +{ + if (!OidIsValid(LOLOR_LargeObjectDescriptionRelationId)) + LOLOR_LargeObjectDescriptionRelationId = + get_lobj_table_oid(LOLOR_LARGEOBJECT_DESCRIPTION); + + return LOLOR_LargeObjectDescriptionRelationId; +} + +Oid +get_LOLOR_LargeObjectDescriptionIndexId() +{ + if (!OidIsValid(LOLOR_LargeObjectDescriptionIndexId)) + LOLOR_LargeObjectDescriptionIndexId = + get_lobj_table_oid(LOLOR_LARGEOBJECT_DESCRIPTION_PKEY); + + return LOLOR_LargeObjectDescriptionIndexId; +} + static void lolor_xact_callback(XactEvent event, void *arg) { @@ -142,6 +196,8 @@ relcache_invalidate_callback(Datum arg, Oid reloid) LOLOR_LargeObjectLOidPNIndexId = InvalidOid; LOLOR_LargeObjectMetadataRelationId = InvalidOid; LOLOR_LargeObjectMetadataOidIndexId = InvalidOid; + LOLOR_LargeObjectDescriptionRelationId = InvalidOid; + LOLOR_LargeObjectDescriptionIndexId = InvalidOid; } /* @@ -156,7 +212,7 @@ _PG_init(void) &lolor_node_id, 0, 0, - 16, + LOLOR_MAX_NODE_ID, PGC_SUSET, 0, NULL, NULL, NULL); @@ -172,6 +228,113 @@ _PG_init(void) CacheRegisterRelcacheCallback(relcache_invalidate_callback, (Datum) 0); } +/* + * lolor_extension_owner + * + * Owner of the installed lolor extension, or InvalidOid when it is not + * installed. + */ +static Oid +lolor_extension_owner(void) +{ + Relation rel; + ScanKeyData skey[1]; + SysScanDesc scan; + HeapTuple tup; + Oid owner = InvalidOid; + + rel = table_open(ExtensionRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], + Anum_pg_extension_extname, + BTEqualStrategyNumber, F_NAMEEQ, + CStringGetDatum(EXTENSION_NAME)); + + scan = systable_beginscan(rel, ExtensionNameIndexId, true, NULL, 1, skey); + + tup = systable_getnext(scan); + if (HeapTupleIsValid(tup)) + owner = ((Form_pg_extension) GETSTRUCT(tup))->extowner; + + systable_endscan(scan); + table_close(rel, AccessShareLock); + + return owner; +} + +/* + * lolor_is_being_dropped + * + * Decide whether the DDL that is about to run will remove the lolor + * extension. + * + * DROP EXTENSION is the obvious spelling, but the extension also goes away + * through DROP SCHEMA lolor CASCADE and through DROP OWNED BY . Those reach the extension by dependency cascade rather than by a + * DropStmt that names it, so each has to be recognised here. Missing one is + * not cosmetic: the large objects would be destroyed along with the lolor + * tables and the renamed pg_catalog functions would never be restored, + * leaving a database with no working lo_open(). + */ +static bool +lolor_is_being_dropped(Node *parsetree) +{ + ListCell *lc; + + if (IsA(parsetree, DropStmt)) + { + DropStmt *stmt = (DropStmt *) parsetree; + + /* + * DROP EXTENSION lolor and DROP SCHEMA lolor both name the object with + * a bare String, so one loop covers both. The extension and its + * schema share a name, which is enforced by lolor.control. + * + * Only CASCADE reaches the extension through the schema: a plain DROP + * SCHEMA is RESTRICT, which cannot remove a schema that still holds + * the extension's tables. Acting on it would run the whole migration + * and take the storage locks before PostgreSQL rejected the command + * and rolled all of it back. + */ + if (stmt->removeType != OBJECT_EXTENSION && + (stmt->removeType != OBJECT_SCHEMA || + stmt->behavior != DROP_CASCADE)) + return false; + + foreach(lc, stmt->objects) + { + Node *objname = (Node *) lfirst(lc); + + if (IsA(objname, String) && + strcmp(strVal(objname), EXTENSION_NAME) == 0) + return true; + } + + return false; + } + + if (IsA(parsetree, DropOwnedStmt)) + { + DropOwnedStmt *stmt = (DropOwnedStmt *) parsetree; + Oid extowner = lolor_extension_owner(); + + if (!OidIsValid(extowner)) + return false; + + foreach(lc, stmt->roles) + { + RoleSpec *rolespec = lfirst_node(RoleSpec, lc); + + if (get_rolespec_oid(rolespec, true) == extowner) + return true; + } + + return false; + } + + return false; +} + /* * lolor_on_drop_extension * @@ -196,51 +359,28 @@ Datum lolor_on_drop_extension(PG_FUNCTION_ARGS) { EventTriggerData *trigdata; - DropStmt *dropstmt; - ListCell *lc; - bool has_lolor_objs = false; /* Make sure we are called as an event trigger */ if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) elog(ERROR, "not fired by event trigger manager"); - /* Make sure we have a parsetree and that this is for a DROP EXTENSION */ trigdata = (EventTriggerData *) fcinfo->context; if (trigdata->parsetree == NULL) { - elog(LOG, "lo_on_drop_extension(): parsetree = NULL"); + elog(LOG, "lolor_on_drop_extension(): parsetree = NULL"); PG_RETURN_NULL(); } /* - * Check that this is DROP EXTENSION lolor + * The trigger is registered for several command tags, so most invocations + * are for drops that have nothing to do with lolor. Say nothing and let + * them proceed. */ - if (!IsA(trigdata->parsetree, DropStmt)) - { - elog(WARNING, "lo_on_drop_extension(): not a DropStmt"); - PG_RETURN_NULL(); - } - dropstmt = (DropStmt *)trigdata->parsetree; - if (dropstmt->removeType != OBJECT_EXTENSION) - { - elog(WARNING, "lo_on_drop_extension(): not a DropStmt for extension"); - PG_RETURN_NULL(); - } - foreach(lc, dropstmt->objects) - { - Node *objname = (Node *) lfirst(lc); - - if (strcmp(strVal(objname), "lolor") == 0) - { - has_lolor_objs = true; - break; - } - } - if (!has_lolor_objs) + if (!lolor_is_being_dropped(trigdata->parsetree)) PG_RETURN_NULL(); /* - * OK, this is DROP EXTENSION lolor. + * The lolor extension is going away. * * First, migrate any large objects stored in lolor tables back to * native PostgreSQL storage. This must happen while lolor is still @@ -249,8 +389,8 @@ lolor_on_drop_extension(PG_FUNCTION_ARGS) * still exist and are readable at this point. * * Then rename our replacement functions out of the way and restore - * the original PostgreSQL function names. The DROP EXTENSION itself - * will then drop the lolor schema and its objects. + * the original PostgreSQL function names. The drop itself will then + * remove the lolor schema and its objects. * * Guard the migrate_to_native() call with a pg_proc check so that * upgrades from versions < 1.3.0 (where the function does not exist) @@ -267,9 +407,9 @@ lolor_on_drop_extension(PG_FUNCTION_ARGS) { /* * If migrate_to_native() fails (e.g. OID conflict), the ERROR - * propagates and aborts the DROP EXTENSION. This is intentional: - * losing large objects silently is worse than a failed DROP. The - * user must resolve the conflict and retry. + * propagates and aborts the drop. This is intentional: losing + * large objects silently is worse than a failed DROP. The user + * must resolve the conflict and retry. */ if (SPI_execute("SELECT lolor.migrate_to_native()", false, 0) != SPI_OK_SELECT) ereport(ERROR, diff --git a/src/lolor.h b/src/lolor.h index 3c4bdbf..32f7d00 100644 --- a/src/lolor.h +++ b/src/lolor.h @@ -20,6 +20,20 @@ #define LOLOR_LARGEOBJECT_PKEY "pg_largeobject_pkey" #define LOLOR_LARGEOBJECT_METADATA "pg_largeobject_metadata" #define LOLOR_LARGEOBJECT_METADATA_PKEY "pg_largeobject_metadata_pkey" +#define LOLOR_LARGEOBJECT_DESCRIPTION "pg_largeobject_description" +#define LOLOR_LARGEOBJECT_DESCRIPTION_PKEY "pg_largeobject_description_pkey" + +/* + * Layout of a lolor-assigned large object OID: the low LOLOR_NODEID_BITS hold + * lolor.node and the remaining bits hold the generated OID, so that concurrent + * creation on different nodes cannot collide. The GUC bound is derived from + * the encoding rather than written out separately: they must agree, and node + * id 16 does not fit in four bits. Changing these changes the on-disk OID + * encoding and is not backward compatible. + */ +#define LOLOR_NODEID_BITS 4 +#define LOLOR_OID_BITS 28 +#define LOLOR_MAX_NODE_ID ((1 << LOLOR_NODEID_BITS) - 1) /* lolor.c */ extern int32 lolor_node_id; @@ -27,6 +41,9 @@ extern Oid get_LOLOR_LargeObjectRelationId(void); extern Oid get_LOLOR_LargeObjectLOidPNIndexId(void); extern Oid get_LOLOR_LargeObjectMetadataRelationId(void); extern Oid get_LOLOR_LargeObjectMetadataOidIndexId(void); +extern Oid get_LOLOR_LargeObjectDescriptionRelationId(void); +extern Oid get_LOLOR_LargeObjectDescriptionIndexId(void); +extern Oid get_LOLOR_LargeObjectDescriptionRelationIdIfExists(void); /* lolor_largeobject.c */ extern Oid LOLOR_LargeObjectCreate(Oid loid); @@ -87,4 +104,7 @@ extern Datum lolor_lo_get(PG_FUNCTION_ARGS); extern Datum lolor_lo_get_fragment(PG_FUNCTION_ARGS); extern Datum lolor_lo_put(PG_FUNCTION_ARGS); +/* lolor_migrate.c */ +extern Datum lolor_migrate_storage(PG_FUNCTION_ARGS); + #endif /* LOLOR_LARGEOBJECT_H */ diff --git a/src/lolor_inv_api.c b/src/lolor_inv_api.c index 6307186..0862a50 100644 --- a/src/lolor_inv_api.c +++ b/src/lolor_inv_api.c @@ -217,18 +217,26 @@ lolor_inv_create(Oid lobjId) lobjId_new = LOLOR_LargeObjectCreate(lobjId); /* - * dependency on the owner of largeobject + * No shared dependency is recorded for the owner, and no object access + * hook is invoked. * - * Note that LO dependencies are recorded using classId - * LOLOR_LargeObjectRelationId for backwards-compatibility reasons. Using - * LOLOR_LargeObjectMetadataRelationId instead would simplify matters for the - * backend, but it'd complicate pg_dump and possibly break other clients. + * Core records pg_shdepend entries for large objects under classId + * LargeObjectRelationId, which is a genuine catalog that the dependency + * machinery knows how to describe and delete. An object in lolor storage + * is a row in an ordinary table, and there is no classId that describes + * it: passing the OID of lolor.pg_largeobject produced pg_shdepend rows + * that DROP ROLE could not interpret, failing with "unrecognized object + * class" and leaving the role permanently undroppable. The rows were + * never removed either, because inv_drop() deletes with + * PERFORM_DELETION_SKIP_ORIGINAL. pg_shdepend is a shared catalog, so + * the stored classId was a per-database relation OID with no meaning to + * any other database. + * + * Not tracking the ownership is a real limitation -- DROP ROLE cannot + * warn that a role still owns large objects held by lolor -- and it is + * inherent in storing them outside the catalogs. lolor.check_orphans() + * reports objects whose owner no longer exists. */ - recordDependencyOnOwner(get_LOLOR_LargeObjectRelationId(), - lobjId_new, GetUserId()); - - /* Post creation hook for new large object */ - InvokeObjectPostCreateHook(get_LOLOR_LargeObjectRelationId(), lobjId_new, 0); /* * Advance command counter to make new tuple visible to later operations. @@ -347,16 +355,14 @@ lolor_inv_close(LargeObjectDesc *obj_desc) int lolor_inv_drop(Oid lobjId) { - ObjectAddress object; - /* - * Delete any comments and dependencies on the large object + * There are no comments, security labels or dependencies to remove: an + * object in lolor storage is not a catalog object, so nothing can be + * attached to it. See the note in lolor_inv_create(). The previous + * performDeletion() call here searched pg_depend under a classId that is + * an ordinary relation OID and could never match anything. */ - object.classId = get_LOLOR_LargeObjectRelationId(); - object.objectId = lobjId; - object.objectSubId = 0; - performDeletion(&object, DROP_CASCADE, PERFORM_DELETION_SKIP_ORIGINAL); - LOLOR_LargeObjectDrop(object.objectId); + LOLOR_LargeObjectDrop(lobjId); /* * Advance command counter so that tuple removal will be seen by later diff --git a/src/lolor_largeobject.c b/src/lolor_largeobject.c index b06686a..cd75b22 100644 --- a/src/lolor_largeobject.c +++ b/src/lolor_largeobject.c @@ -39,10 +39,6 @@ #define GETNEWOID_LOG_THRESHOLD 1000000 #define GETNEWOID_LOG_MAX_INTERVAL 128000000 -/* Parameters to determine new unique Oid. */ -#define MAX_NODEID_BITS 4 -#define MAX_OID_BITS 28 - /* * Create a large object having the given LO identifier. * @@ -104,6 +100,7 @@ LOLOR_LargeObjectDrop(Oid loid) ScanKeyData skey[1]; SysScanDesc scan; HeapTuple tuple; + Oid descoid; pg_lo_meta = table_open(get_LOLOR_LargeObjectMetadataRelationId(), RowExclusiveLock); @@ -154,6 +151,38 @@ LOLOR_LargeObjectDrop(Oid loid) table_close(pg_largeobject, RowExclusiveLock); table_close(pg_lo_meta, RowExclusiveLock); + + /* + * Drop any comment parked for this object while it lived in lolor + * storage. Leaving it behind would outlive the object: OIDs are only + * checked against pg_largeobject_metadata when a new one is generated, so + * a later object reusing this OID would inherit the stale comment on its + * way back to native storage. + * + * The relation is absent when the loaded library is newer than the + * installed extension version, which is the normal state between + * installing the package and running ALTER EXTENSION UPDATE. + */ + descoid = get_LOLOR_LargeObjectDescriptionRelationIdIfExists(); + if (OidIsValid(descoid)) + { + Relation pg_lo_desc = table_open(descoid, RowExclusiveLock); + + ScanKeyInit(&skey[0], + 1, /* loid */ + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(loid)); + + scan = systable_beginscan(pg_lo_desc, + get_LOLOR_LargeObjectDescriptionIndexId(), + true, NULL, 1, skey); + + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + CatalogTupleDelete(pg_lo_desc, &tuple->t_self); + + systable_endscan(scan); + table_close(pg_lo_desc, RowExclusiveLock); + } } /* @@ -204,8 +233,9 @@ LOLOR_LargeObjectExists(Oid loid) * LOLOR_GetNewOidWithIndex * Generate a new OID that is unique within the given relation. * - * The lower 4 bits contains the lolor_node_id. The 2^28 bits consist of Oid - * returned from GetNewObjectId and adjusted to remain within the range. + * The low LOLOR_NODEID_BITS contain lolor_node_id; the remaining + * LOLOR_OID_BITS hold an Oid returned from GetNewObjectId, adjusted to remain + * within range. * * See comments for GetNewOidWithIndex() for more details. */ @@ -236,11 +266,11 @@ LOLOR_GetNewOidWithIndex(Relation relation, Oid indexId, AttrNumber oidcolumn) * Keep the range within 1..2^28. Restart from start on overflow and see * if any of the Oids are avaialbe. */ - newOid = newOid % (1 << MAX_OID_BITS); + newOid = newOid % (1 << LOLOR_OID_BITS); if (newOid == 0) newOid = 1; - newOid = (newOid << MAX_NODEID_BITS) | lolor_node_id; + newOid = (newOid << LOLOR_NODEID_BITS) | lolor_node_id; if (IsBootstrapProcessingMode()) return newOid; diff --git a/src/lolor_migrate.c b/src/lolor_migrate.c new file mode 100644 index 0000000..d7f544f --- /dev/null +++ b/src/lolor_migrate.c @@ -0,0 +1,718 @@ +/*------------------------------------------------------------------------- + * + * lolor_migrate.c + * Relocate large objects between PostgreSQL's native catalog storage + * (pg_catalog.pg_largeobject{,_metadata}) and lolor's replicated user + * tables (lolor.pg_largeobject{,_metadata}). + * + * lolor's tables are created with exactly the same column layout as the + * catalogs they stand in for (see lolor--1.0.sql). That equivalence is what + * lets the rest of this extension cast their tuples to Form_pg_largeobject + * and friends, and it is what makes migration a plain tuple copy between two + * relations rather than a decode/re-encode through the large object API. + * + * Everything beyond the tuple copy is the bookkeeping that only the real + * catalogs participate in: + * + * - pg_shdepend rows for the owner and for ACL grantees, so that DROP ROLE, + * REASSIGN OWNED and DROP OWNED keep seeing the objects. + * - pg_description comments, which have nowhere to live while an object sits + * in lolor storage and are parked in lolor.pg_largeobject_description for + * the duration. + * + * Removal of the native objects goes through performMultipleDeletions() -- + * the same path DROP does -- so shared dependencies, comments and security + * labels are cleaned up by core rather than by hand. + * + * The layout equivalence is verified at run time instead of assumed, so a + * future PostgreSQL release that changes either catalog yields a clear error + * rather than silent corruption. + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/lolor/src/lolor_migrate.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/detoast.h" +#include "access/genam.h" +#include "access/htup_details.h" +#include "access/table.h" +#include "access/xact.h" +#include "catalog/dependency.h" +#include "catalog/indexing.h" +#include "catalog/objectaccess.h" +#include "catalog/objectaddress.h" +#include "catalog/pg_largeobject.h" +#include "catalog/pg_largeobject_metadata.h" +#include "commands/comment.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/memutils.h" +#include "utils/rel.h" +#include "utils/snapmgr.h" + +#include "lolor.h" + +/* + * How many objects to hand to performMultipleDeletions() at a time. Bounds + * the memory used by the ObjectAddresses array on databases holding a very + * large number of large objects. + */ +#define MIGRATE_DELETE_CHUNK 1000 + +/* + * How many conflicting OIDs to name in the pre-flight error before + * summarising the remainder. + */ +#define MIGRATE_MAX_REPORTED_CONFLICTS 10 + +PG_FUNCTION_INFO_V1(lolor_migrate_storage); + +/* + * The four relations involved in one migration, plus the indexes needed to + * probe the destination. "src" is where the objects live now, "dst" is where + * they are going. + */ +typedef struct MigrateRels +{ + bool to_native; /* lolor -> pg_catalog when true */ + + Relation src_meta; + Relation dst_meta; + Oid src_meta_idx; + Oid dst_meta_idx; + + Relation src_data; + Relation dst_data; + Oid src_data_idx; + Oid dst_data_idx; + + Relation desc_rel; /* lolor.pg_largeobject_description */ + Oid desc_idx; +} MigrateRels; + +/* + * Confirm that two relations have interchangeable on-disk tuple layouts. + * + * lolor's tables are expected to mirror the catalogs exactly. Rather than + * trusting that across PostgreSQL major versions, check it every time: a + * mismatch means the extension was built against a different catalog + * definition than the server is running, and copying tuples would corrupt + * data. + */ +static void +validate_layout(Relation a, Relation b, int expected_natts) +{ + TupleDesc da = RelationGetDescr(a); + TupleDesc db = RelationGetDescr(b); + int i; + + if (da->natts != db->natts || da->natts != expected_natts) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("incompatible large object storage layout"), + errdetail("Relation \"%s\" has %d columns and \"%s\" has %d; %d expected.", + RelationGetRelationName(a), da->natts, + RelationGetRelationName(b), db->natts, + expected_natts))); + + for (i = 0; i < da->natts; i++) + { + Form_pg_attribute aa = TupleDescAttr(da, i); + Form_pg_attribute ab = TupleDescAttr(db, i); + + if (aa->attisdropped || ab->attisdropped) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("incompatible large object storage layout"), + errdetail("Column %d of \"%s\" or \"%s\" is dropped.", + i + 1, RelationGetRelationName(a), + RelationGetRelationName(b)))); + + if (aa->atttypid != ab->atttypid || + aa->attlen != ab->attlen || + aa->attbyval != ab->attbyval || + aa->attalign != ab->attalign || + aa->attndims != ab->attndims) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("incompatible large object storage layout"), + errdetail("Column %d differs between \"%s\" and \"%s\".", + i + 1, RelationGetRelationName(a), + RelationGetRelationName(b)), + errhint("The lolor extension was built against a different " + "PostgreSQL catalog definition than this server uses."))); + } +} + +/* + * Flatten any toasted or short-header varlena in a deformed tuple. + * + * heap_form_tuple() copies whatever pointer it is handed, so an external + * datum belonging to the source relation's TOAST table would otherwise be + * carried into the destination as a dangling reference once the source row is + * removed. lolor.pg_largeobject is an ordinary table with a TOAST table of + * its own, and a full LOBLKSIZE page of incompressible data can be pushed out + * of line there, so this is reachable in practice rather than theoretical. + * + * detoast_attr() returns its argument untouched when the datum is already a + * plain 4-byte-header varlena, so this is cheap in the common case. + */ +static void +flatten_varlenas(TupleDesc desc, Datum *values, const bool *nulls) +{ + int i; + + for (i = 0; i < desc->natts; i++) + { + Form_pg_attribute att = TupleDescAttr(desc, i); + + if (nulls[i] || att->attbyval || att->attlen != -1) + continue; + + values[i] = PointerGetDatum( + detoast_attr((struct varlena *) DatumGetPointer(values[i]))); + } +} + +/* + * Does "loid" already have a metadata row in rel? + */ +static bool +metadata_exists(Relation rel, Oid indexId, Oid loid) +{ + ScanKeyData skey[1]; + SysScanDesc scan; + bool found; + + ScanKeyInit(&skey[0], + Anum_pg_largeobject_metadata_oid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(loid)); + + scan = systable_beginscan(rel, indexId, true, NULL, 1, skey); + found = HeapTupleIsValid(systable_getnext(scan)); + systable_endscan(scan); + + return found; +} + +/* + * Pre-flight OID conflict check. + * + * Runs before anything is written so that a conflict leaves both stores + * untouched. Reports the offending OIDs rather than merely their existence, + * because resolving a conflict means acting on specific objects. + */ +static void +check_oid_conflicts(MigrateRels *rels) +{ + SysScanDesc scan; + HeapTuple tup; + StringInfoData buf; + int64 nconflicts = 0; + + initStringInfo(&buf); + + scan = systable_beginscan(rels->src_meta, InvalidOid, false, NULL, 0, NULL); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Oid loid; + bool isnull; + + CHECK_FOR_INTERRUPTS(); + + loid = DatumGetObjectId(heap_getattr(tup, + Anum_pg_largeobject_metadata_oid, + RelationGetDescr(rels->src_meta), + &isnull)); + Assert(!isnull); + + if (!metadata_exists(rels->dst_meta, rels->dst_meta_idx, loid)) + continue; + + if (nconflicts < MIGRATE_MAX_REPORTED_CONFLICTS) + appendStringInfo(&buf, "%s%u", nconflicts > 0 ? ", " : "", loid); + nconflicts++; + } + systable_endscan(scan); + + if (nconflicts == 0) + { + pfree(buf.data); + return; + } + + if (nconflicts > MIGRATE_MAX_REPORTED_CONFLICTS) + appendStringInfo(&buf, " and " INT64_FORMAT " more", + nconflicts - MIGRATE_MAX_REPORTED_CONFLICTS); + + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg_plural("%lld large object already exists in the destination storage", + "%lld large objects already exist in the destination storage", + (unsigned long) nconflicts, + (long long) nconflicts), + errdetail("Conflicting OID(s): %s.", buf.data), + errhint("Remove or rename the conflicting large objects and retry; " + "nothing has been migrated."))); +} + +/* + * Fetch the parked comment for loid from lolor.pg_largeobject_description, + * or NULL when the object has none. Caller owns the returned string. + */ +static char * +fetch_parked_comment(MigrateRels *rels, Oid loid) +{ + Relation rel = rels->desc_rel; + TupleDesc desc = RelationGetDescr(rel); + ScanKeyData skey[1]; + SysScanDesc scan; + HeapTuple tup; + char *result = NULL; + + ScanKeyInit(&skey[0], 1, BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(loid)); + + scan = systable_beginscan(rel, rels->desc_idx, true, NULL, 1, skey); + + tup = systable_getnext(scan); + if (HeapTupleIsValid(tup)) + { + bool isnull; + Datum d = heap_getattr(tup, 2, desc, &isnull); + + if (!isnull) + result = TextDatumGetCString(d); + } + + systable_endscan(scan); + + return result; +} + +/* + * Park a comment for loid in lolor.pg_largeobject_description. + */ +static void +park_comment(MigrateRels *rels, Oid loid, const char *comment) +{ + Relation rel = rels->desc_rel; + HeapTuple tup; + Datum values[2]; + bool nulls[2]; + + values[0] = ObjectIdGetDatum(loid); + values[1] = CStringGetTextDatum(comment); + nulls[0] = false; + nulls[1] = false; + + tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); + CatalogTupleInsert(rel, tup); + heap_freetuple(tup); +} + +/* + * Remove every row from lolor.pg_largeobject_description. + */ +static void +clear_parked_comments(MigrateRels *rels) +{ + Relation rel = rels->desc_rel; + SysScanDesc scan; + HeapTuple tup; + + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + CHECK_FOR_INTERRUPTS(); + CatalogTupleDelete(rel, &tup->t_self); + } + systable_endscan(scan); +} + +/* + * Copy every metadata row from src_meta to dst_meta, carrying ownership, + * ACLs and comments across the storage boundary. + * + * Returns the number of objects copied. + */ +static int64 +copy_metadata(MigrateRels *rels) +{ + TupleDesc srcdesc = RelationGetDescr(rels->src_meta); + TupleDesc dstdesc = RelationGetDescr(rels->dst_meta); + CatalogIndexState indstate; + SysScanDesc scan; + HeapTuple tup; + int64 count = 0; + MemoryContext tmpcxt; + MemoryContext oldcxt; + + /* + * Each iteration allocates: the detoasted page or ACL from + * flatten_varlenas(), the formed tuple, index scratch from the catalog + * insert, and any comment text. CurrentMemoryContext is not reset for + * the duration of a function call, so on a database with millions of + * large objects those allocations would accumulate until the backend ran + * out of memory -- aborting an all-or-nothing migration. Give each row + * its own context and reset it. + */ + tmpcxt = AllocSetContextCreate(CurrentMemoryContext, + "lolor migrate metadata", + ALLOCSET_DEFAULT_SIZES); + + indstate = CatalogOpenIndexes(rels->dst_meta); + + scan = systable_beginscan(rels->src_meta, InvalidOid, false, NULL, 0, NULL); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Datum values[Natts_pg_largeobject_metadata]; + bool nulls[Natts_pg_largeobject_metadata]; + HeapTuple newtup; + Oid loid; + Oid owner; + + CHECK_FOR_INTERRUPTS(); + + oldcxt = MemoryContextSwitchTo(tmpcxt); + + heap_deform_tuple(tup, srcdesc, values, nulls); + flatten_varlenas(srcdesc, values, nulls); + + loid = DatumGetObjectId(values[Anum_pg_largeobject_metadata_oid - 1]); + owner = DatumGetObjectId(values[Anum_pg_largeobject_metadata_lomowner - 1]); + + newtup = heap_form_tuple(dstdesc, values, nulls); + CatalogTupleInsertWithInfo(rels->dst_meta, newtup, indstate); + heap_freetuple(newtup); + + if (rels->to_native) + { + /* + * Record the shared dependencies the catalog is expected to have. + * Note the owner is recorded directly rather than creating the + * object as the current user and transferring it afterwards, so + * pg_shdepend never passes through a wrong intermediate state. + */ + recordDependencyOnOwner(LargeObjectRelationId, loid, owner); + + if (!nulls[Anum_pg_largeobject_metadata_lomacl - 1]) + recordDependencyOnNewAcl(LargeObjectRelationId, loid, 0, owner, + DatumGetAclP(values[Anum_pg_largeobject_metadata_lomacl - 1])); + + /* Restore any comment parked on the way into lolor storage. */ + { + char *comment = fetch_parked_comment(rels, loid); + + if (comment != NULL) + { + CreateComments(loid, LargeObjectRelationId, 0, comment); + pfree(comment); + } + } + + InvokeObjectPostCreateHook(LargeObjectRelationId, loid, 0); + } + else + { + /* + * lolor's tables are ordinary tables, so a comment has nowhere to + * attach while the object lives there. Park it so the round trip + * back to native storage is lossless. The pg_description row + * itself is removed by performMultipleDeletions() later. + */ + char *comment = GetComment(loid, LargeObjectRelationId, 0); + + if (comment != NULL) + park_comment(rels, loid, comment); + } + + MemoryContextSwitchTo(oldcxt); + MemoryContextReset(tmpcxt); + + count++; + } + systable_endscan(scan); + + CatalogCloseIndexes(indstate); + MemoryContextDelete(tmpcxt); + CommandCounterIncrement(); + + return count; +} + +/* + * Copy every data page from src_data to dst_data. + * + * Page numbers are carried across verbatim, so a sparse large object stays + * sparse: there is no re-chunking, no offset arithmetic and therefore no + * dependence on LOBLKSIZE or on the 2 GB boundary. + * + * Returns the number of pages copied. + */ +static int64 +copy_data_pages(MigrateRels *rels) +{ + TupleDesc srcdesc = RelationGetDescr(rels->src_data); + TupleDesc dstdesc = RelationGetDescr(rels->dst_data); + CatalogIndexState indstate; + SysScanDesc scan; + HeapTuple tup; + int64 count = 0; + MemoryContext tmpcxt; + MemoryContext oldcxt; + + /* One row per page, so this is the loop that runs millions of times. */ + tmpcxt = AllocSetContextCreate(CurrentMemoryContext, + "lolor migrate pages", + ALLOCSET_DEFAULT_SIZES); + + indstate = CatalogOpenIndexes(rels->dst_data); + + scan = systable_beginscan(rels->src_data, InvalidOid, false, NULL, 0, NULL); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Datum values[Natts_pg_largeobject]; + bool nulls[Natts_pg_largeobject]; + HeapTuple newtup; + + CHECK_FOR_INTERRUPTS(); + + oldcxt = MemoryContextSwitchTo(tmpcxt); + + heap_deform_tuple(tup, srcdesc, values, nulls); + flatten_varlenas(srcdesc, values, nulls); + + newtup = heap_form_tuple(dstdesc, values, nulls); + CatalogTupleInsertWithInfo(rels->dst_data, newtup, indstate); + + MemoryContextSwitchTo(oldcxt); + MemoryContextReset(tmpcxt); + + count++; + } + systable_endscan(scan); + + CatalogCloseIndexes(indstate); + MemoryContextDelete(tmpcxt); + CommandCounterIncrement(); + + return count; +} + +/* + * Remove the native large objects that have just been copied into lolor. + * + * performMultipleDeletions() is the same path DROP takes, so pg_shdepend, + * pg_description and pg_seclabel rows go away with the objects instead of + * being left behind for someone to discover later. + */ +static void +drop_native_objects(MigrateRels *rels) +{ + SysScanDesc scan; + HeapTuple tup; + ObjectAddresses *addrs; + int pending = 0; + + addrs = new_object_addresses(); + + scan = systable_beginscan(rels->src_meta, InvalidOid, false, NULL, 0, NULL); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + ObjectAddress obj; + bool isnull; + + CHECK_FOR_INTERRUPTS(); + + obj.classId = LargeObjectRelationId; + obj.objectId = DatumGetObjectId(heap_getattr(tup, + Anum_pg_largeobject_metadata_oid, + RelationGetDescr(rels->src_meta), + &isnull)); + obj.objectSubId = 0; + Assert(!isnull); + + add_exact_object_address(&obj, addrs); + + if (++pending >= MIGRATE_DELETE_CHUNK) + { + performMultipleDeletions(addrs, DROP_CASCADE, + PERFORM_DELETION_INTERNAL | + PERFORM_DELETION_QUIETLY); + free_object_addresses(addrs); + addrs = new_object_addresses(); + pending = 0; + } + } + systable_endscan(scan); + + if (pending > 0) + performMultipleDeletions(addrs, DROP_CASCADE, + PERFORM_DELETION_INTERNAL | + PERFORM_DELETION_QUIETLY); + + free_object_addresses(addrs); + CommandCounterIncrement(); +} + +/* + * Delete every row of an ordinary lolor storage table. + */ +static void +truncate_lolor_table(Relation rel) +{ + SysScanDesc scan; + HeapTuple tup; + + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + CHECK_FOR_INTERRUPTS(); + CatalogTupleDelete(rel, &tup->t_self); + } + systable_endscan(scan); +} + +/* + * lolor.migrate_storage(to_native boolean) returns bigint + * + * Moves every large object from one storage to the other and returns the + * number of objects moved. The whole move is one transaction: on any error + * nothing has changed in either store. + * + * This is the mechanism only. Policy -- who may run it, and how it interacts + * with logical replication -- lives in the SQL wrappers, which are the + * supported entry points. + */ +Datum +lolor_migrate_storage(PG_FUNCTION_ARGS) +{ + bool to_native = PG_GETARG_BOOL(0); + MigrateRels rels; + Relation native_meta; + Relation native_data; + Relation lolor_meta; + Relation lolor_data; + Oid lolor_meta_id; + Oid lolor_data_id; + int64 nobjects; + int64 npages; + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to migrate large object storage"))); + + memset(&rels, 0, sizeof(rels)); + rels.to_native = to_native; + + lolor_meta_id = get_LOLOR_LargeObjectMetadataRelationId(); + lolor_data_id = get_LOLOR_LargeObjectRelationId(); + + /* + * Lock both stores against concurrent large object activity for the whole + * migration. + * + * ShareRowExclusiveLock rather than RowExclusiveLock: ordinary large + * object reads and writes take RowExclusiveLock (see open_lo_relation() + * in lolor_inv_api.c), and RowExclusiveLock does not conflict with + * itself. A concurrent lo_write() could therefore commit between the + * point where this migration copies a page and the point where it removes + * the source rows, and the write would be lost without trace. + * ShareRowExclusiveLock conflicts with RowExclusiveLock and with itself, + * so large object writers wait and two migrations serialise. + * + * The locks are always taken in the same order regardless of direction, + * so that two migrations running opposite ways cannot deadlock against + * each other. + */ + native_meta = table_open(LargeObjectMetadataRelationId, ShareRowExclusiveLock); + native_data = table_open(LargeObjectRelationId, ShareRowExclusiveLock); + lolor_meta = table_open(lolor_meta_id, ShareRowExclusiveLock); + lolor_data = table_open(lolor_data_id, ShareRowExclusiveLock); + + rels.desc_rel = table_open(get_LOLOR_LargeObjectDescriptionRelationId(), + ShareRowExclusiveLock); + rels.desc_idx = get_LOLOR_LargeObjectDescriptionIndexId(); + + if (to_native) + { + rels.src_meta = lolor_meta; + rels.src_data = lolor_data; + rels.dst_meta = native_meta; + rels.dst_data = native_data; + + rels.src_meta_idx = get_LOLOR_LargeObjectMetadataOidIndexId(); + rels.src_data_idx = get_LOLOR_LargeObjectLOidPNIndexId(); + rels.dst_meta_idx = LargeObjectMetadataOidIndexId; + rels.dst_data_idx = LargeObjectLOidPNIndexId; + } + else + { + rels.src_meta = native_meta; + rels.src_data = native_data; + rels.dst_meta = lolor_meta; + rels.dst_data = lolor_data; + + rels.src_meta_idx = LargeObjectMetadataOidIndexId; + rels.src_data_idx = LargeObjectLOidPNIndexId; + rels.dst_meta_idx = get_LOLOR_LargeObjectMetadataOidIndexId(); + rels.dst_data_idx = get_LOLOR_LargeObjectLOidPNIndexId(); + } + + /* Never copy tuples between relations whose layouts might differ. */ + validate_layout(rels.src_meta, rels.dst_meta, + Natts_pg_largeobject_metadata); + validate_layout(rels.src_data, rels.dst_data, + Natts_pg_largeobject); + + /* Refuse before writing anything if the destination already has the OIDs. */ + check_oid_conflicts(&rels); + + nobjects = copy_metadata(&rels); + npages = copy_data_pages(&rels); + + if (to_native) + { + truncate_lolor_table(rels.src_data); + truncate_lolor_table(rels.src_meta); + clear_parked_comments(&rels); + CommandCounterIncrement(); + } + else + { + drop_native_objects(&rels); + } + + /* + * Close the relcache references but keep the locks until the caller's + * transaction ends. Passing the lock mode here would release them at + * once, re-opening the window this function is meant to close: between + * that release and the commit, another session could take + * RowExclusiveLock and write to a large object that has already been + * relocated, and that write would disappear when this transaction's + * emptying of the source store became visible. + */ + table_close(rels.desc_rel, NoLock); + table_close(lolor_data, NoLock); + table_close(lolor_meta, NoLock); + table_close(native_data, NoLock); + table_close(native_meta, NoLock); + + ereport(NOTICE, + (errmsg("migrated " INT64_FORMAT " large object(s), " INT64_FORMAT " data page(s), to %s storage", + nobjects, npages, to_native ? "native" : "lolor"))); + + PG_RETURN_INT64(nobjects); +} diff --git a/t/007_migration.pl b/t/007_migration.pl new file mode 100644 index 0000000..d64d2f6 --- /dev/null +++ b/t/007_migration.pl @@ -0,0 +1,205 @@ +# Check large object migration between native and lolor storage +# +# Copyright (c) 2022-2026, pgEdge, Inc. +# + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('migration'); +my ($result, $stdout, $stderr); + +$node->init(allows_streaming => 'logical'); +$node->append_conf('postgresql.conf', qq{lolor.node = 1}); +$node->start; +$node->safe_psql('postgres', "CREATE EXTENSION lolor"); + +# ############################################################################## +# +# Build native large objects covering the shapes that migration has to +# preserve, then move them into lolor storage and back. +# +# ############################################################################## + +$node->safe_psql('postgres', "SELECT lolor.disable()"); +$node->safe_psql('postgres', "CREATE ROLE lo_owner"); +$node->safe_psql('postgres', "CREATE ROLE lo_grantee"); + +my $plain = $node->safe_psql('postgres', + "SELECT lo_from_bytea(0, 'annotated native object')"); +$node->safe_psql('postgres', qq( + ALTER LARGE OBJECT $plain OWNER TO lo_owner; + GRANT SELECT ON LARGE OBJECT $plain TO lo_grantee; + COMMENT ON LARGE OBJECT $plain IS 'survives the round trip'; +)); + +# A sparse object: a few bytes at offset 0 and a few 10MB in. +my $sparse = $node->safe_psql('postgres', "SELECT lo_create(0)"); +$node->safe_psql('postgres', qq( + BEGIN; + SELECT lo_open($sparse, x'60000'::int) AS fd \\gset + SELECT lowrite(:fd, 'start'); + SELECT lo_lseek64(:fd, 10000000, 0); + SELECT lowrite(:fd, 'end'); + SELECT lo_close(:fd); + COMMIT; +)); + +# An object with no data pages at all. +my $empty = $node->safe_psql('postgres', "SELECT lo_create(0)"); + +my $native_pages = $node->safe_psql('postgres', + "SELECT count(*) FROM pg_catalog.pg_largeobject"); + +$node->safe_psql('postgres', "SELECT lolor.enable()"); +$node->safe_psql('postgres', "SELECT lolor.migrate_from_native()"); + +is($node->safe_psql('postgres', + "SELECT count(*) FROM pg_catalog.pg_largeobject_metadata"), + '0', "native storage emptied by migrate_from_native()"); + +is($node->safe_psql('postgres', + "SELECT count(*) FROM lolor.pg_largeobject"), + $native_pages, + "page count preserved exactly, so sparse objects stay sparse"); + +is($node->safe_psql('postgres', + "SELECT count(*) FROM pg_shdepend WHERE classid = 'pg_largeobject'::regclass " + . "AND dbid = (SELECT oid FROM pg_database WHERE datname = current_database())"), + '0', "native shared dependencies removed with the objects"); + +is($node->safe_psql('postgres', + "SELECT description FROM lolor.pg_largeobject_description WHERE loid = $plain"), + 'survives the round trip', "comment parked while in lolor storage"); + +is($node->safe_psql('postgres', + "SELECT pg_get_userbyid(lomowner) FROM lolor.pg_largeobject_metadata WHERE oid = $plain"), + 'lo_owner', "owner preserved into lolor storage"); + +is($node->safe_psql('postgres', "SELECT convert_from(lo_get($plain), 'UTF8')"), + 'annotated native object', "content readable from lolor storage"); + +is($node->safe_psql('postgres', "SELECT length(lo_get($sparse))"), + '10000003', "sparse object keeps its logical length"); + +is($node->safe_psql('postgres', "SELECT length(lo_get($empty))"), + '0', "object with no data pages survives"); + +# ############################################################################## +# +# Everything must come back intact, including the catalog bookkeeping that +# only native storage participates in. +# +# ############################################################################## + +$node->safe_psql('postgres', "SELECT lolor.migrate_to_native()"); + +is($node->safe_psql('postgres', + "SELECT count(*) FROM lolor.pg_largeobject_metadata"), + '0', "lolor storage emptied by migrate_to_native()"); + +is($node->safe_psql('postgres', + "SELECT pg_get_userbyid(lomowner) FROM pg_catalog.pg_largeobject_metadata WHERE oid = $plain"), + 'lo_owner', "owner restored"); + +is($node->safe_psql('postgres', + "SELECT string_agg(deptype::text || ':' || refobjid::regrole::text, ',' ORDER BY deptype) " + . "FROM pg_shdepend WHERE classid = 'pg_largeobject'::regclass AND objid = $plain " + . "AND dbid = (SELECT oid FROM pg_database WHERE datname = current_database())"), + 'a:lo_grantee,o:lo_owner', + "owner and ACL grantee recorded in pg_shdepend"); + +is($node->safe_psql('postgres', + "SELECT description FROM pg_description " + . "WHERE classoid = 'pg_largeobject'::regclass AND objoid = $plain"), + 'survives the round trip', "comment reinstated on the catalog object"); + +is($node->safe_psql('postgres', + "SELECT count(*) FROM lolor.pg_largeobject_description"), + '0', "comment parking table drained"); + +is($node->safe_psql('postgres', "SELECT count(*) FROM pg_catalog.pg_largeobject"), + $native_pages, "page count still exact after the return trip"); + +# DROP ROLE must now see the objects, which a raw catalog update would not +# have allowed. +($result, $stdout, $stderr) = $node->psql('postgres', "DROP ROLE lo_owner"); +like($stderr, qr/cannot be dropped because some objects depend on it/, + "DROP ROLE refuses while the role owns migrated large objects"); + +($result, $stdout, $stderr) = $node->psql('postgres', "DROP ROLE lo_grantee"); +like($stderr, qr/cannot be dropped because some objects depend on it/, + "DROP ROLE refuses for an ACL grantee of a migrated large object"); + +# ############################################################################## +# +# Migrated objects must survive a restart, and an unclean one. +# +# ############################################################################## + +$node->safe_psql('postgres', "SELECT lolor.migrate_from_native()"); +$node->restart; + +is($node->safe_psql('postgres', "SELECT convert_from(lo_get($plain), 'UTF8')"), + 'annotated native object', "content survives a clean restart"); + +$node->safe_psql('postgres', + "SELECT lo_from_bytea(0, 'written before a crash') AS o"); +my $crash_oid = $node->safe_psql('postgres', + "SELECT oid FROM lolor.pg_largeobject_metadata ORDER BY oid DESC LIMIT 1"); + +$node->stop('immediate'); +$node->start; + +is($node->safe_psql('postgres', "SELECT convert_from(lo_get($plain), 'UTF8')"), + 'annotated native object', "content survives an immediate stop and recovery"); + +is($node->safe_psql('postgres', "SELECT convert_from(lo_get($crash_oid), 'UTF8')"), + 'written before a crash', + "a committed object written just before the crash is recovered"); + +# ############################################################################## +# +# Without spock there is no way to keep the migration out of logical decoding, +# so the presence of a logical slot must stop it rather than let subscribers +# diverge. +# +# ############################################################################## + +$node->safe_psql('postgres', "SELECT lolor.migrate_to_native()"); +$node->safe_psql('postgres', + "SELECT pg_create_logical_replication_slot('lolor_test_slot', 'pgoutput')"); + +($result, $stdout, $stderr) = + $node->psql('postgres', "SELECT lolor.migrate_from_native()"); +is($stdout, '-1', + "migrate_from_native() refuses with a logical slot present and reports -1"); +like($stderr, qr/logical replication slot\(s\) exist/, + "refusal explains why"); + +is($node->safe_psql('postgres', + "SELECT count(*) > 0 FROM pg_catalog.pg_largeobject_metadata"), + 't', "refusal left the native objects untouched"); + +# The zero-object case returns early, so put something in lolor storage for +# the guard to actually be reached. +$node->safe_psql('postgres', "SELECT lo_from_bytea(0, 'held in lolor storage')"); + +($result, $stdout, $stderr) = + $node->psql('postgres', "SELECT lolor.migrate_to_native()"); +isnt($result, 0, "migrate_to_native() raises an error rather than reporting -1"); +like($stderr, qr/cannot migrate large objects/, + "migrate_to_native() refuses on the drop path rather than losing objects"); + +$node->safe_psql('postgres', + "SELECT pg_drop_replication_slot('lolor_test_slot')"); + +is($node->safe_psql('postgres', "SELECT lolor.migrate_from_native() > 0"), + 't', "migration proceeds once the slot is gone"); + +$node->stop; +done_testing(); diff --git a/t/008_concurrency_and_privileges.pl b/t/008_concurrency_and_privileges.pl new file mode 100644 index 0000000..5a988a3 --- /dev/null +++ b/t/008_concurrency_and_privileges.pl @@ -0,0 +1,201 @@ +# Check migration locking, server-side file privileges and cross-node OID +# collision detection +# +# Copyright (c) 2022-2026, pgEdge, Inc. +# + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Cwd qw(abs_path); + +my $node = PostgreSQL::Test::Cluster->new('concurrency'); +my ($result, $stdout, $stderr); + +$node->init; +$node->append_conf('postgresql.conf', qq{lolor.node = 1}); +$node->start; +$node->safe_psql('postgres', "CREATE EXTENSION lolor"); + +# ############################################################################## +# +# A migration must exclude concurrent large object activity for its whole +# transaction. Ordinary large object access takes RowExclusiveLock, which does +# not conflict with itself, so a migration holding only that lock would let a +# concurrent write commit between the copy and the emptying of the source +# store, losing it silently. +# +# ############################################################################## + +$node->safe_psql('postgres', "SELECT lo_from_bytea(0, 'to be migrated')"); + +my $bg = $node->background_psql('postgres'); +# query_safe() treats any stderr output as a failure, and the migration +# reports its progress as a NOTICE. +$bg->query_safe("SET client_min_messages = warning"); +$bg->query_safe("BEGIN"); +$bg->query_safe("SELECT lolor.migrate_to_native()"); + +is($node->safe_psql('postgres', qq( + SELECT count(*) FROM pg_locks l JOIN pg_class c ON c.oid = l.relation + WHERE c.relname IN ('pg_largeobject', 'pg_largeobject_metadata') + AND l.mode = 'ShareRowExclusiveLock')), + '4', + "migration holds ShareRowExclusiveLock on both stores"); + +($result, $stdout, $stderr) = $node->psql('postgres', + "SET lock_timeout = '2s'; SELECT lo_from_bytea(0, 'concurrent write')"); +like($stderr, qr/canceling statement due to lock timeout/, + "a concurrent large object write blocks while a migration is running"); + +$bg->query_safe("COMMIT"); +$bg->quit; + +is($node->safe_psql('postgres', + "SELECT lo_from_bytea(0, 'after commit') IS NOT NULL"), + 't', "the same write succeeds once the migration has committed"); + +# The migration itself must not have lost anything. +is($node->safe_psql('postgres', + "SELECT count(*) FROM pg_catalog.pg_largeobject_metadata"), + '1', "the migrated object is in native storage"); + +# ############################################################################## +# +# lo_import() and lo_export() read and write files as the server's operating +# system account. Core revokes EXECUTE on them from PUBLIC; lolor's +# replacements must be restricted the same way, or any user could read or +# overwrite arbitrary files. +# +# ############################################################################## + +$node->safe_psql('postgres', "CREATE ROLE lo_plain LOGIN"); + +# The server resolves relative paths against its data directory, so these +# have to be absolute. +my $filedir = abs_path(PostgreSQL::Test::Utils::tempdir()); +my $srcfile = "$filedir/lolor_import_source.txt"; +open(my $fh, '>', $srcfile) or die "could not write $srcfile: $!"; +print $fh "imported through a large object\n"; +close($fh); + +($result, $stdout, $stderr) = $node->psql('postgres', + "SELECT lo_import('$srcfile')", extra_params => [ '-U', 'lo_plain' ]); +like($stderr, qr/permission denied for function lo_import/, + "lo_import() is not executable by an ordinary user"); + +my $dstfile = "$filedir/lolor_export_target.txt"; +($result, $stdout, $stderr) = $node->psql('postgres', + "SELECT lo_export(1, '$dstfile')", extra_params => [ '-U', 'lo_plain' ]); +like($stderr, qr/permission denied for function lo_export/, + "lo_export() is not executable by an ordinary user"); + +ok(!-e $dstfile, "the refused lo_export() wrote no file"); + +# The superuser can still round-trip a file through lolor storage. +my $imported = $node->safe_psql('postgres', "SELECT lo_import('$srcfile')"); +is($node->safe_psql('postgres', "SELECT convert_from(lo_get($imported), 'UTF8')"), + "imported through a large object\n", + "lo_import() stores the file contents in lolor storage"); + +$node->safe_psql('postgres', "SELECT lo_export($imported, '$dstfile')"); +ok(-e $dstfile, "lo_export() wrote the file"); +is(slurp_file($dstfile), "imported through a large object\n", + "exported contents match what was imported"); + +# ############################################################################## +# +# DROP SCHEMA reaches the extension by dependency cascade rather than as DROP +# EXTENSION. The cleanup has to run anyway, or the large objects are +# destroyed along with the lolor tables and pg_catalog is left without a +# working lo_open(). +# +# ############################################################################## + +$node->safe_psql('postgres', "SELECT lolor.migrate_from_native()"); +my $rescued = $node->safe_psql('postgres', + "SELECT lo_from_bytea(0, 'rescued from drop schema')"); + +$node->safe_psql('postgres', "DROP SCHEMA lolor CASCADE"); + +is($node->safe_psql('postgres', + "SELECT count(*) FROM pg_extension WHERE extname = 'lolor'"), + '0', "DROP SCHEMA CASCADE removed the extension"); + +is($node->safe_psql('postgres', + "SELECT to_regprocedure('pg_catalog.lo_open(oid,int4)') IS NOT NULL"), + 't', "the native lo_open() was put back"); + +is($node->safe_psql('postgres', + "SELECT to_regprocedure('pg_catalog.lo_open_orig(oid,int4)') IS NULL"), + 't', "no *_orig functions were left behind"); + +is($node->safe_psql('postgres', "SELECT convert_from(lo_get($rescued), 'UTF8')"), + 'rescued from drop schema', + "the large object was migrated out rather than dropped with the schema"); + +$node->stop; + +# ############################################################################## +# +# Native large object OIDs are not node encoded, so two nodes can hold +# different objects under the same OID. Migrating both would converge them +# onto one row, and because the migration is hidden from replication nothing +# would report it. Passing the peer OIDs turns that into a refusal. +# +# ############################################################################## + +my $n1 = PostgreSQL::Test::Cluster->new('node1'); +my $n2 = PostgreSQL::Test::Cluster->new('node2'); +foreach my $n ($n1, $n2) +{ + $n->init; + $n->start; +} +$n1->append_conf('postgresql.conf', qq{lolor.node = 1}); +$n2->append_conf('postgresql.conf', qq{lolor.node = 2}); +$n1->restart; +$n2->restart; + +foreach my $n ($n1, $n2) +{ + $n->safe_psql('postgres', "CREATE EXTENSION lolor"); + $n->safe_psql('postgres', "SELECT lolor.disable()"); +} + +# Give both nodes a native object under the same OID but with different +# contents, which is exactly the situation that silently diverges. +$n1->safe_psql('postgres', "SELECT lo_from_bytea(500001, 'node one content')"); +$n2->safe_psql('postgres', "SELECT lo_from_bytea(500001, 'node two content')"); + +foreach my $n ($n1, $n2) +{ + $n->safe_psql('postgres', "SELECT lolor.enable()"); +} + +my $peer_oids = $n2->safe_psql('postgres', + "SELECT coalesce(string_agg(o::text, ','), '') FROM lolor.native_lo_oids() AS o"); +is($peer_oids, '500001', "native_lo_oids() reports the peer's OIDs"); + +($result, $stdout, $stderr) = $n1->psql('postgres', + "SELECT lolor.migrate_from_native(peer_oids => ARRAY[$peer_oids]::oid[])"); +isnt($result, 0, "migrate_from_native() refuses a colliding peer OID"); +like($stderr, qr/also held natively by another node/, + "the refusal names the reason"); + +is($n1->safe_psql('postgres', + "SELECT count(*) FROM pg_catalog.pg_largeobject_metadata"), + '1', "the refusal left node one's native objects in place"); + +# Without the peer list the collision is invisible, which is why the argument +# exists; the migration itself still succeeds locally. +is($n1->safe_psql('postgres', "SELECT lolor.migrate_from_native()"), + '1', "migration proceeds when no peer OIDs are supplied"); + +$n1->stop; +$n2->stop; + +done_testing();