diff --git a/.github/workflows/fork-ocr-model-check.yml b/.github/workflows/fork-ocr-model-check.yml new file mode 100644 index 0000000000000..b85a990c66fd7 --- /dev/null +++ b/.github/workflows/fork-ocr-model-check.yml @@ -0,0 +1,18 @@ +# Weekly check for a newer Claude Opus on Bedrock; opens/updates one issue here +# when OCR_BEDROCK_MODEL is behind. Notify-only (GITHUB_TOKEN cannot write +# Actions variables). Logic lives in gburd/ci-workflows. + +name: OCR model self-check + +on: + schedule: + - cron: '0 12 * * 1' # Mondays 12:00 UTC + workflow_dispatch: + +jobs: + check: + uses: gburd/ci-workflows/.github/workflows/ocr-model-check.yml@v1 + with: + aws_role_arn: ${{ vars.AWS_ROLE_ARN }} + aws_region: ${{ vars.AWS_REGION }} + bedrock_model: ${{ vars.OCR_BEDROCK_MODEL }} diff --git a/.github/workflows/fork-ocr-review.yml b/.github/workflows/fork-ocr-review.yml new file mode 100644 index 0000000000000..ae938754339b4 --- /dev/null +++ b/.github/workflows/fork-ocr-review.yml @@ -0,0 +1,43 @@ +# AI PR review (Claude Opus on Bedrock). Logic and review rules live in +# gburd/ci-workflows so this fork's master carries a stub instead of ~1000 +# lines that have to be rebased onto upstream forever. +# +# Needs repo variables AWS_ROLE_ARN, AWS_REGION, OCR_BEDROCK_MODEL. Auth is +# GitHub OIDC; no static AWS keys. See the ci-workflows README. + +name: OCR AI Review + +on: + pull_request: + # Note: no draft filter — drafts are reviewed too. + types: [opened, synchronize, reopened, ready_for_review] + issue_comment: + types: [created] + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to review' + required: true + type: number + +# One review per PR; cancel superseded runs to save Bedrock spend. +concurrency: + group: ocr-review-${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} + cancel-in-progress: true + +jobs: + review: + # Comment events only when the comment is on a PR and starts with the + # trigger keyword; PR events and manual dispatch always. + if: | + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/open-code-review') || + startsWith(github.event.comment.body, '@open-code-review') || + startsWith(github.event.comment.body, '/pg-history'))) + uses: gburd/ci-workflows/.github/workflows/ocr-review.yml@v1 + with: + aws_role_arn: ${{ vars.AWS_ROLE_ARN }} + aws_region: ${{ vars.AWS_REGION }} + bedrock_model: ${{ vars.OCR_BEDROCK_MODEL }} diff --git a/.github/workflows/fork-sync-upstream.yml b/.github/workflows/fork-sync-upstream.yml new file mode 100644 index 0000000000000..022b3101b5a9d --- /dev/null +++ b/.github/workflows/fork-sync-upstream.yml @@ -0,0 +1,48 @@ +# Keep this fork's master rebased on postgres/postgres master, carrying only +# the fork-owned CI stubs (.github/workflows/fork-*.yml). The workflows they +# call, and the OCR review rules, live in gburd/ci-workflows. +# +# Requires the SYNC_TOKEN secret: a PAT with repo+workflow scope. The default +# GITHUB_TOKEN is not allowed to push commits that touch .github/workflows/, +# and every commit we carry does. + +name: Sync from upstream + +on: + schedule: + - cron: '17 */3 * * *' + workflow_dispatch: + +concurrency: + group: fork-sync-upstream + +permissions: + contents: read + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + token: ${{ secrets.SYNC_TOKEN }} + + - name: Rebase master onto upstream + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git remote add upstream https://github.com/postgres/postgres.git + git fetch --no-tags upstream master + + # Master carries fork CI only. Anything else means work landed here + # that belongs on a branch; force-rebasing it on a timer would be wrong. + if git diff --name-only upstream/master...HEAD | + grep -v '^\.github/workflows/fork-'; then + echo "::error::master carries non-CI changes (listed above); move them to a branch" + exit 1 + fi + + git rebase upstream/master + git push --force-with-lease diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 0797dcf96da3a..7800bf9944b7a 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -49,6 +49,7 @@ + diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index 2101442c90fcb..0940a557ffa2e 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -164,6 +164,7 @@ break is not needed in a wider output rendering. &high-availability; &monitoring; &wal; + &undo; &logical-replication; &jit; ®ress; diff --git a/doc/src/sgml/undo.sgml b/doc/src/sgml/undo.sgml new file mode 100644 index 0000000000000..e9a7debfe4d6d --- /dev/null +++ b/doc/src/sgml/undo.sgml @@ -0,0 +1,137 @@ + + + + UNDO Logging + + + UNDO logging + + + + PostgreSQL provides UNDO logging infrastructure + that records the inverse of a change, so that the change can be reversed if + the transaction that made it rolls back or is still in progress when the + server crashes. UNDO records are written into the WAL stream, so they are + durable, replicated and archived like every other WAL record. + + + + UNDO is not a cluster-wide switch. It is always-on infrastructure that is + used only by the subsystems that write UNDO records. The default + heap table access method does not, and ordinary heap and + index changes carry no UNDO overhead. The one user of UNDO in + PostgreSQL today is transactional filesystem + operations (FILEOPS), which make the directory and file changes done by + CREATE DATABASE, ALTER DATABASE SET + TABLESPACE, CREATE TABLESPACE and + DROP TABLESPACE roll back with the transaction. + + + + Transactional Filesystem Operations + + + When a transaction creates, renames or writes a file, creates a directory, + changes a file's mode or owner, truncates a file, or creates a link, + FILEOPS makes the change and then writes a WAL record describing it and an + UNDO record describing how to reverse it. A change that fails is an + ordinary ERROR and leaves nothing in the WAL. If the + transaction commits, the UNDO record is never used. If it rolls back, the + abort record carries the inverses, and the backend applies them, newest + first, before ROLLBACK returns: a created file is + removed, a changed mode is restored, a new database directory is removed + with everything in it, and so on. Removing a file or directory is not + done immediately; it is deferred to commit and simply dropped on rollback. + + + + Between the change and its WAL record there is a short window in which a + crash leaves the change in place with nothing to reverse it, as it would + without FILEOPS. Doing it the other way round, logging first, would turn + every ordinary filesystem error into a PANIC. + + + + FILEOPS writes WAL and UNDO only when is + replica or higher. At minimal it + writes neither, and a crash in the middle of a transaction can leave a + partly done filesystem change behind, as without FILEOPS. + + + + + Crash Recovery and Replication + + + At the end of crash recovery, after redo, the startup process finds every + transaction that wrote UNDO records but neither committed nor aborted + before the crash, applies its UNDO records, and writes a WAL record of the + rollback, before the server accepts connections. Each checkpoint lists + the UNDO chains of transactions in progress, so this includes a + transaction whose UNDO records all precede the checkpoint redo starts + from. This step takes time proportional to + the number of UNDO records those transactions wrote. A prepared + transaction is left prepared; ROLLBACK PREPARED + applies its UNDO records later, and COMMIT PREPARED + discards them. + + + + An UNDO record that cannot be applied, for example because the server no + longer has permission to change the file it describes, is reported with a + WARNING and is not retried, and the change it describes + stays in place. The end-of-recovery summary then also reports how many + records could not be applied. + + + + A physical standby replays the WAL record of each filesystem change, and + the inverses carried by each abort record, so it ends up with the same + files as the primary. Crash recovery replays them the same way. A + transaction lost in a crash of the primary writes no abort record; the + primary's crash recovery writes the rollback record instead, and a + standby replays it. + + + + + UNDO Space Management + + + UNDO records live in the WAL stream as XLOG_UNDO_BATCH + records; there are no separate UNDO files. A checkpoint does not remove + WAL that still holds the UNDO records of a transaction that is in progress + or prepared, so a long-running or long-prepared transaction that wrote UNDO + holds back WAL removal until it ends, much as a replication slot does. + + + + + Monitoring + + + The server log shows UNDO being applied during crash recovery: + + + +LOG: UNDO recovery: rolling back transaction 742 +LOG: UNDO recovery complete: 1 transactions rolled back, 3 records applied + + + + and, if some records could not be applied: + + + +WARNING: UNDO recovery complete: 1 transactions processed, 0 records applied, 1 records could not be applied +DETAIL: The changes those records describe were not rolled back. + + + + pg_waldump shows UNDO records with + and the filesystem changes themselves with + . + + + + diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile index e88d72ea0397d..5778d0d7f8225 100644 --- a/src/backend/access/Makefile +++ b/src/backend/access/Makefile @@ -22,6 +22,7 @@ SUBDIRS = \ sequence \ table \ tablesample \ - transam + transam \ + undo \ include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/meson.build b/src/backend/access/meson.build index 5fd18de74f92b..d569ac4e6e32a 100644 --- a/src/backend/access/meson.build +++ b/src/backend/access/meson.build @@ -14,3 +14,4 @@ subdir('spgist') subdir('table') subdir('tablesample') subdir('transam') +subdir('undo') diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile index cd95eec37f148..5b0d8922ea272 100644 --- a/src/backend/access/rmgrdesc/Makefile +++ b/src/backend/access/rmgrdesc/Makefile @@ -13,6 +13,7 @@ OBJS = \ clogdesc.o \ committsdesc.o \ dbasedesc.o \ + fileopsdesc.o \ genericdesc.o \ gindesc.o \ gistdesc.o \ @@ -29,6 +30,7 @@ OBJS = \ spgdesc.o \ standbydesc.o \ tblspcdesc.o \ + undodesc.o \ xactdesc.o \ xlogdesc.o diff --git a/src/backend/access/rmgrdesc/fileopsdesc.c b/src/backend/access/rmgrdesc/fileopsdesc.c new file mode 100644 index 0000000000000..ba4cb111c372e --- /dev/null +++ b/src/backend/access/rmgrdesc/fileopsdesc.c @@ -0,0 +1,220 @@ +/*------------------------------------------------------------------------- + * + * fileopsdesc.c + * rmgr descriptor routines for storage/file/fileops.c + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/rmgrdesc/fileopsdesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "storage/fileops.h" + +void +fileops_desc(StringInfo buf, XLogReaderState *record) +{ + char *data = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_FILEOPS_CREATE: + { + xl_fileops_create *xlrec = (xl_fileops_create *) data; + const char *path = data + SizeOfFileOpsCreate; + + appendStringInfo(buf, "create \"%s\" flags 0x%x mode 0%o", + path, xlrec->flags, xlrec->mode); + } + break; + + case XLOG_FILEOPS_WRITE: + { + xl_fileops_write *xlrec = (xl_fileops_write *) data; + const char *path = data + SizeOfFileOpsWrite; + + appendStringInfo(buf, "write \"%s\" offset %lld len %u", + path, (long long) xlrec->offset, xlrec->len); + } + break; + + case XLOG_FILEOPS_RENAME: + { + xl_fileops_rename *xlrec = (xl_fileops_rename *) data; + const char *oldpath = data + SizeOfFileOpsRename; + const char *newpath = oldpath + xlrec->oldpath_len; + + appendStringInfo(buf, "rename \"%s\" to \"%s\"", + oldpath, newpath); + } + break; + + case XLOG_FILEOPS_DELETE: + { + xl_fileops_delete *xlrec = (xl_fileops_delete *) data; + const char *path = data + SizeOfFileOpsDelete; + + appendStringInfo(buf, "delete \"%s\" at_%s", + path, + xlrec->at_commit ? "commit" : "abort"); + } + break; + + case XLOG_FILEOPS_SYMLINK: + { + xl_fileops_symlink *xlrec = (xl_fileops_symlink *) data; + const char *target = data + SizeOfFileOpsSymlink; + const char *linkpath = target + xlrec->target_len; + + appendStringInfo(buf, "symlink \"%s\" -> \"%s\"", + linkpath, target); + } + break; + + case XLOG_FILEOPS_LINK: + { + xl_fileops_link *xlrec = (xl_fileops_link *) data; + const char *oldpath = data + SizeOfFileOpsLink; + const char *newpath = oldpath + xlrec->oldpath_len; + + appendStringInfo(buf, "link \"%s\" -> \"%s\"", + newpath, oldpath); + } + break; + + case XLOG_FILEOPS_MKDIR: + { + xl_fileops_mkdir *xlrec = (xl_fileops_mkdir *) data; + const char *path = data + SizeOfFileOpsMkdir; + + appendStringInfo(buf, "mkdir \"%s\" mode 0%o%s", + path, (unsigned int) xlrec->mode, + (xlrec->flags & FILEOPS_MKDIR_TREE) ? " tree" : ""); + } + break; + + case XLOG_FILEOPS_RMDIR: + { + xl_fileops_rmdir *xlrec = (xl_fileops_rmdir *) data; + const char *path = data + SizeOfFileOpsRmdir; + + appendStringInfo(buf, "rmdir \"%s\" at_%s", + path, + xlrec->at_commit ? "commit" : "abort"); + } + break; + + case XLOG_FILEOPS_CHMOD: + { + xl_fileops_chmod *xlrec = (xl_fileops_chmod *) data; + const char *path = data + SizeOfFileOpsChmod; + + appendStringInfo(buf, "chmod \"%s\" mode 0%o", + path, (unsigned int) xlrec->mode); + } + break; + + case XLOG_FILEOPS_CHOWN: + { + xl_fileops_chown *xlrec = (xl_fileops_chown *) data; + const char *path = data + SizeOfFileOpsChown; + + appendStringInfo(buf, "chown \"%s\" uid %d gid %d", + path, (int) xlrec->uid, (int) xlrec->gid); + } + break; + + case XLOG_FILEOPS_TRUNCATE: + { + xl_fileops_truncate *xlrec = (xl_fileops_truncate *) data; + const char *path = data + SizeOfFileOpsTruncate; + + appendStringInfo(buf, "truncate \"%s\" to %lld bytes", + path, (long long) xlrec->length); + } + break; + + case XLOG_FILEOPS_SETXATTR: + { + xl_fileops_setxattr *xlrec = (xl_fileops_setxattr *) data; + const char *path = data + SizeOfFileOpsSetxattr; + const char *name = path + xlrec->path_len; + + appendStringInfo(buf, "setxattr \"%s\" name \"%s\" len %u", + path, name, xlrec->value_len); + } + break; + + case XLOG_FILEOPS_REMOVEXATTR: + { + xl_fileops_removexattr *xlrec = + (xl_fileops_removexattr *) data; + const char *path = data + SizeOfFileOpsRemovexattr; + const char *name = path + xlrec->path_len; + + appendStringInfo(buf, "removexattr \"%s\" name \"%s\"", + path, name); + } + break; + + default: + appendStringInfo(buf, "unknown fileops op code %u", info); + break; + } +} + +const char * +fileops_identify(uint8 info) +{ + const char *id = NULL; + + switch (info & ~XLR_INFO_MASK) + { + case XLOG_FILEOPS_CREATE: + id = "CREATE"; + break; + case XLOG_FILEOPS_DELETE: + id = "DELETE"; + break; + case XLOG_FILEOPS_RENAME: + id = "RENAME"; + break; + case XLOG_FILEOPS_WRITE: + id = "WRITE"; + break; + case XLOG_FILEOPS_TRUNCATE: + id = "TRUNCATE"; + break; + case XLOG_FILEOPS_CHMOD: + id = "CHMOD"; + break; + case XLOG_FILEOPS_CHOWN: + id = "CHOWN"; + break; + case XLOG_FILEOPS_MKDIR: + id = "MKDIR"; + break; + case XLOG_FILEOPS_RMDIR: + id = "RMDIR"; + break; + case XLOG_FILEOPS_SYMLINK: + id = "SYMLINK"; + break; + case XLOG_FILEOPS_LINK: + id = "LINK"; + break; + case XLOG_FILEOPS_SETXATTR: + id = "SETXATTR"; + break; + case XLOG_FILEOPS_REMOVEXATTR: + id = "REMOVEXATTR"; + break; + } + + return id; +} diff --git a/src/backend/access/rmgrdesc/meson.build b/src/backend/access/rmgrdesc/meson.build index d9000ccd9fd10..91c6dd7da26e9 100644 --- a/src/backend/access/rmgrdesc/meson.build +++ b/src/backend/access/rmgrdesc/meson.build @@ -6,6 +6,7 @@ rmgr_desc_sources = files( 'clogdesc.c', 'committsdesc.c', 'dbasedesc.c', + 'fileopsdesc.c', 'genericdesc.c', 'gindesc.c', 'gistdesc.c', @@ -22,6 +23,7 @@ rmgr_desc_sources = files( 'spgdesc.c', 'standbydesc.c', 'tblspcdesc.c', + 'undodesc.c', 'xactdesc.c', 'xlogdesc.c', ) diff --git a/src/backend/access/rmgrdesc/undodesc.c b/src/backend/access/rmgrdesc/undodesc.c new file mode 100644 index 0000000000000..ba7ed874b5f1e --- /dev/null +++ b/src/backend/access/rmgrdesc/undodesc.c @@ -0,0 +1,107 @@ +/*------------------------------------------------------------------------- + * + * undodesc.c + * rmgr descriptor routines for access/undo + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/rmgrdesc/undodesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undo_xlog.h" +#include "access/xlogreader.h" + +/* + * undo_desc - Describe an UNDO WAL record for pg_waldump + * + * This function generates human-readable output for UNDO WAL records, + * used by pg_waldump and other debugging tools. + */ +void +undo_desc(StringInfo buf, XLogReaderState *record) +{ + char *rec = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_UNDO_ROLLBACK: + { + xl_undo_rollback *xlrec = (xl_undo_rollback *) rec; + + appendStringInfo(buf, "xid %u, nrecords %u", + xlrec->xid, xlrec->nrecords); + } + break; + + case XLOG_UNDO_INFLIGHT: + { + xl_undo_inflight *xlrec = (xl_undo_inflight *) rec; + + appendStringInfo(buf, "nxacts %u", xlrec->nxacts); + for (uint32 i = 0; i < xlrec->nxacts; i++) + { + xl_undo_inflight_xact *x = &xlrec->xacts[i]; + + appendStringInfo(buf, "; xid %u", x->xid); + for (int j = 0; j < NUndoPersistenceLevels; j++) + if (XLogRecPtrIsValid(x->last_batch_lsn[j])) + appendStringInfo(buf, " p%d %X/%08X", j, + LSN_FORMAT_ARGS(x->last_batch_lsn[j])); + } + } + break; + + case XLOG_UNDO_BATCH: + { + xl_undo_batch *xlrec = (xl_undo_batch *) rec; + + appendStringInfo(buf, + "undo batch: xid %u, nrecords %u, " + "total_len %u, chain_prev %X/%08X, " + "primary_reloid %u, persistence %d", + xlrec->xid, + xlrec->nrecords, + xlrec->total_len, + LSN_FORMAT_ARGS(xlrec->chain_prev), + xlrec->primary_reloid, + xlrec->persistence); + } + break; + + default: + appendStringInfo(buf, "unknown undo op code %u", info); + break; + } +} + +/* + * undo_identify - Identify an UNDO WAL record type + * + * Returns a string identifying the operation type for debugging output. + */ +const char * +undo_identify(uint8 info) +{ + const char *id = NULL; + + switch (info & ~XLR_INFO_MASK) + { + case XLOG_UNDO_ROLLBACK: + id = "ROLLBACK"; + break; + case XLOG_UNDO_BATCH: + id = "BATCH"; + break; + case XLOG_UNDO_INFLIGHT: + id = "INFLIGHT"; + break; + } + + return id; +} diff --git a/src/backend/access/rmgrdesc/xactdesc.c b/src/backend/access/rmgrdesc/xactdesc.c index 96aa6e17bd50c..f416e2d09f1e2 100644 --- a/src/backend/access/rmgrdesc/xactdesc.c +++ b/src/backend/access/rmgrdesc/xactdesc.c @@ -230,6 +230,12 @@ ParseAbortRecord(uint8 info, xl_xact_abort *xlrec, xl_xact_parsed_abort *parsed) data += sizeof(xl_xact_origin); } + + if (parsed->xinfo & XACT_XINFO_HAS_FILEOPS) + { + memcpy(&parsed->fileops_len, data, sizeof(uint32)); + parsed->fileops = data + sizeof(uint32); + } } /* @@ -391,6 +397,8 @@ xact_desc_abort(StringInfo buf, uint8 info, xl_xact_abort *xlrec, ReplOriginId o } xact_desc_stats(buf, "", parsed.nstats, parsed.stats); + if (parsed.fileops_len) + appendStringInfo(buf, "; fileops undo: %u bytes", parsed.fileops_len); } static void diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c index 4fda03a3cfcc6..22ef4170d25e7 100644 --- a/src/backend/access/transam/rmgr.c +++ b/src/backend/access/transam/rmgr.c @@ -30,6 +30,7 @@ #include "access/multixact.h" #include "access/nbtxlog.h" #include "access/spgxlog.h" +#include "access/undo_xlog.h" #include "access/xact.h" #include "catalog/storage_xlog.h" #include "commands/dbcommands_xlog.h" @@ -38,6 +39,7 @@ #include "replication/decode.h" #include "replication/message.h" #include "replication/origin.h" +#include "storage/fileops.h" #include "storage/standby.h" #include "utils/relmapper.h" /* IWYU pragma: end_keep */ diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index d89afb093209c..088b6a1584053 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -77,6 +77,8 @@ #include #include "access/commit_ts.h" +#include "access/undorecord.h" +#include "access/xactundo.h" #include "access/htup_details.h" #include "access/subtrans.h" #include "access/transam.h" @@ -98,6 +100,7 @@ #include "replication/origin.h" #include "replication/syncrep.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/ipc.h" #include "storage/md.h" #include "storage/predicate.h" @@ -148,6 +151,15 @@ int max_prepared_xacts = 0; * twophase.h */ +/* + * FILEOPS 2PC prepare hook (fileops.c). Declared locally by prototype to + * avoid pulling storage/fileops.h into this file. Serializes this + * transaction's pending file operations into the 2PC state file so + * COMMIT/ROLLBACK PREPARED -- possibly from another backend after a crash -- + * can perform the deferred commit-time ops or the abort-time filesystem undo. + */ +extern void FileOpsAtPrepare(void); + typedef struct GlobalTransactionData { GlobalTransaction next; /* list link for free list */ @@ -163,6 +175,17 @@ typedef struct GlobalTransactionData */ XLogRecPtr prepare_start_lsn; /* XLOG offset of prepare record start */ XLogRecPtr prepare_end_lsn; /* XLOG offset of prepare record end */ + + /* + * Permanent-level UNDO chain-head LSN for this prepared xact, or + * InvalidXLogRecPtr if it generated no cluster-wide UNDO. Set at PREPARE + * (MarkAsPreparingGuts), at redo (PrepareRedoAdd), and when a stale 2PC + * file is recovered from disk. UndoGetOldestBatchLSN() scans this so the + * UNDO-batch WAL a still-prepared xact needs for ROLLBACK PREPARED is not + * recycled -- the per-backend retention slot is cleared when the + * preparing backend exits, so it does not cover this case. + */ + XLogRecPtr undo_batch_lsn; FullTransactionId fxid; /* The GXACT full xid */ Oid owner; /* ID of user that executed the xact */ @@ -228,7 +251,9 @@ static void RecordTransactionAbortPrepared(TransactionId xid, RelFileLocator *rels, int nstats, xl_xact_stats_item *stats, - const char *gid); + const char *gid, + const char *fileops, + uint32 fileops_len); static void ProcessRecords(char *bufptr, FullTransactionId fxid, const TwoPhaseCallback callbacks[]); static void RemoveGXact(GlobalTransaction gxact); @@ -493,6 +518,7 @@ MarkAsPreparingGuts(GlobalTransaction gxact, FullTransactionId fxid, gxact->locking_backend = MyProcNumber; gxact->valid = false; gxact->inredo = false; + gxact->undo_batch_lsn = GetCurrentXactLastBatchLSN(UNDOPERSISTENCE_PERMANENT); strlcpy(gxact->gid, gid, GIDSIZE); /* @@ -979,8 +1005,20 @@ TwoPhaseFilePath(char *path, FullTransactionId fxid) /* * Header for a 2PC state file + * + * TWOPHASE_MAGIC must be bumped whenever xl_xact_prepare changes layout. + * The struct gained last_batch_lsn[NUndoPersistenceLevels] (24 bytes) for + * UNDO chain tracking across 2PC boundaries, requiring a bump from + * 0x57F94534 to 0x57F94535 to prevent old servers from silently misreading + * the variable-length arrays that follow the fixed header at the wrong offsets. + * + * Bumped again to 0x57F94536: FILEOPS now registers TWOPHASE_RM_FILEOPS_ID + * records so deferred commit-time file operations (RENAME/DELETE/RMDIR/RMTREE) + * and abort-time filesystem undo survive PREPARE. This adds a new 2PC record + * type (not a header field), but a bump still keeps a mixed-version cluster + * from replaying a state file whose record stream it does not understand. */ -#define TWOPHASE_MAGIC 0x57F94534 /* format identifier */ +#define TWOPHASE_MAGIC 0x57F94536 /* format identifier */ typedef xl_xact_prepare TwoPhaseFileHeader; @@ -1102,6 +1140,10 @@ StartPrepare(GlobalTransaction gxact) hdr.origin_lsn = InvalidXLogRecPtr; hdr.origin_timestamp = 0; + /* Save UNDO chain head LSNs so recovery can find UNDO records */ + for (int j = 0; j < NUndoPersistenceLevels; j++) + hdr.last_batch_lsn[j] = GetCurrentXactLastBatchLSN(j); + save_state_data(&hdr, sizeof(TwoPhaseFileHeader)); save_state_data(gxact->gid, hdr.gidlen); @@ -1155,6 +1197,14 @@ EndPrepare(GlobalTransaction gxact) StateFileChunk *record; bool replorigin; + /* + * FILEOPS: persist this transaction's pending file operations into the + * 2PC state file, so COMMIT PREPARED can run them after a crash. Must + * run before the END sentinel while the record chain is still open. A + * no-op if this transaction registered no file operations. + */ + FileOpsAtPrepare(); + /* Add the end sentinel to the list of 2PC records */ RegisterTwoPhaseRecord(TWOPHASE_RM_END_ID, 0, NULL, 0); @@ -1499,6 +1549,47 @@ StandbyTransactionIdIsPrepared(TransactionId xid) return result; } +/* + * RecoveryTransactionIdIsPrepared + * Check if a transaction ID is in the in-memory prepared transaction list. + * + * This is used during crash recovery UNDO phase, before prepared transaction + * files exist on disk. It checks the in-memory TwoPhaseState that was + * reconstructed from WAL replay. + */ +bool +RecoveryTransactionIdIsPrepared(TransactionId xid) +{ + int i; + FullTransactionId fxid; + + Assert(TransactionIdIsValid(xid)); + + if (max_prepared_xacts <= 0) + return false; /* 2PC not enabled */ + + if (TwoPhaseState == NULL) + return false; /* 2PC not initialized yet */ + + fxid = AdjustToFullTransactionId(xid); + + LWLockAcquire(TwoPhaseStateLock, LW_SHARED); + + for (i = 0; i < TwoPhaseState->numPrepXacts; i++) + { + GlobalTransaction gxact = TwoPhaseState->prepXacts[i]; + + if (FullTransactionIdEquals(gxact->fxid, fxid)) + { + LWLockRelease(TwoPhaseStateLock); + return true; + } + } + + LWLockRelease(TwoPhaseStateLock); + return false; +} + /* * FinishPreparedTransaction: execute COMMIT PREPARED or ROLLBACK PREPARED */ @@ -1586,12 +1677,59 @@ FinishPreparedTransaction(const char *gid, bool isCommit) hdr->ninvalmsgs, invalmsgs, hdr->initfileinval, gid); else + { + char *fileops = NULL; + uint32 fileops_len = 0; + char *scan = bufptr; + + /* + * The abort decision and the FILEOPS inverses must be in the same WAL + * record, so a standby and crash recovery apply the inverses whenever + * they replay the abort. PREPARE saved them as a 2PC record. + */ + for (;;) + { + TwoPhaseRecordOnDisk *record = (TwoPhaseRecordOnDisk *) scan; + + if (record->rmid == TWOPHASE_RM_END_ID) + break; + scan += MAXALIGN(sizeof(TwoPhaseRecordOnDisk)); + if (record->rmid == TWOPHASE_RM_FILEOPS_ID && + record->info == FILEOPS_TWOPHASE_INVERSES) + { + fileops = scan; + fileops_len = record->len; + break; + } + scan += MAXALIGN(record->len); + } + RecordTransactionAbortPrepared(xid, hdr->nsubxacts, children, hdr->nabortrels, abortrels, hdr->nabortstats, abortstats, - gid); + gid, fileops, fileops_len); + + /* + * ROLLBACK PREPARED: apply any cluster-wide UNDO chain synchronously, + * here, before the prepared transaction's resources are torn down. + * The permanent-level chain-head LSN was durably saved in the 2PC + * header at PREPARE; ApplyUndoChainFromWAL() walks the UNDO chain + * backwards from that LSN, restoring before-images. + * + * Only the permanent level is applied: TEMP undo is gone (the + * originating backend exited) and UNLOGGED forks are reset on crash; + * this matches the crash-recovery UNDO phase in undo_xlog.c. + */ + { + XLogRecPtr perm_lsn = + hdr->last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; + + if (XLogRecPtrIsValid(perm_lsn)) + (void) ApplyUndoChainFromWAL(perm_lsn); + } + } ProcArrayRemove(proc, latestXid); @@ -2173,6 +2311,14 @@ RecoverPreparedTransactions(void) hdr->prepared_at, hdr->owner, hdr->database); + /* + * MarkAsPreparingGuts read the live backend's last_batch_lsn (wrong + * xact during recovery); restore this xact's real value from the 2PC + * header so WAL retention pins its UNDO batch until ROLLBACK + * PREPARED. + */ + gxact->undo_batch_lsn = hdr->last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; + /* recovered, so reset the flag for entries generated by redo */ gxact->inredo = false; @@ -2480,7 +2626,9 @@ RecordTransactionAbortPrepared(TransactionId xid, RelFileLocator *rels, int nstats, xl_xact_stats_item *stats, - const char *gid) + const char *gid, + const char *fileops, + uint32 fileops_len) { XLogRecPtr recptr; bool replorigin; @@ -2525,7 +2673,7 @@ RecordTransactionAbortPrepared(TransactionId xid, nrels, rels, nstats, stats, MyXactFlags | XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK, - xid, gid); + xid, gid, fileops, fileops_len); if (replorigin) /* Move LSNs forward for this replication origin */ @@ -2538,11 +2686,30 @@ RecordTransactionAbortPrepared(TransactionId xid, /* Mark the transaction aborted in clog */ TransactionIdAbortTree(xid, nchildren, children); - /* Checkpoint can proceed now */ - MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; + /* + * Checkpoint can proceed now, unless FILEOPS inverses still have to be + * applied: a checkpoint must not move the redo pointer past this abort + * record until they are durable, so keep it delayed until then. + */ + if (!fileops_len) + MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; END_CRIT_SECTION(); + if (fileops_len) + { + PG_TRY(); + { + INJECTION_POINT("fileops-abort-after-flush", NULL); + FileOpsApplyAbort(fileops, fileops_len); + } + PG_FINALLY(); + { + MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; + } + PG_END_TRY(); + } + /* * Wait for synchronous replication, if required. * @@ -2646,6 +2813,7 @@ PrepareRedoAdd(FullTransactionId fxid, char *buf, gxact->valid = false; gxact->ondisk = !XLogRecPtrIsValid(start_lsn); gxact->inredo = true; /* yes, added in redo */ + gxact->undo_batch_lsn = hdr->last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; strlcpy(gxact->gid, gid, GIDSIZE); /* And insert it into the active array */ @@ -2927,3 +3095,36 @@ TwoPhaseGetOldestXidInCommit(void) return oldestRunningXid; } + +/* + * TwoPhaseGetOldestUndoBatchLSN + * Return the oldest permanent-level UNDO chain-head LSN across all + * prepared transactions, or InvalidXLogRecPtr if none generated UNDO. + * + * UndoGetOldestBatchLSN() folds this into the WAL-retention horizon so the + * UNDO-batch WAL a still-prepared xact needs for ROLLBACK PREPARED survives + * checkpoints and backend exit (see undo_batch_lsn in GlobalTransactionData). + */ +XLogRecPtr +TwoPhaseGetOldestUndoBatchLSN(void) +{ + XLogRecPtr oldest = InvalidXLogRecPtr; + + if (max_prepared_xacts <= 0) + return InvalidXLogRecPtr; + + LWLockAcquire(TwoPhaseStateLock, LW_SHARED); + + for (int i = 0; i < TwoPhaseState->numPrepXacts; i++) + { + XLogRecPtr lsn = TwoPhaseState->prepXacts[i]->undo_batch_lsn; + + if (XLogRecPtrIsValid(lsn) && + (!XLogRecPtrIsValid(oldest) || lsn < oldest)) + oldest = lsn; + } + + LWLockRelease(TwoPhaseStateLock); + + return oldest; +} diff --git a/src/backend/access/transam/twophase_rmgr.c b/src/backend/access/transam/twophase_rmgr.c index fae254c6e2364..687e77f7e6743 100644 --- a/src/backend/access/transam/twophase_rmgr.c +++ b/src/backend/access/transam/twophase_rmgr.c @@ -19,6 +19,14 @@ #include "pgstat.h" #include "storage/lock.h" #include "storage/predicate.h" +#include "storage/fileops.h" + +/* + * FILEOPS 2PC callbacks (fileops.c). Declared locally by prototype to avoid + * pulling storage/fileops.h in here; the signature is the generic + * TwoPhaseCallback. Postcommit performs the deferred commit-time file ops; + * postabort performs the abort-time filesystem undo. + */ const TwoPhaseCallback twophase_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -27,7 +35,8 @@ const TwoPhaseCallback twophase_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_recover, /* Lock */ NULL, /* pgstat */ multixact_twophase_recover, /* MultiXact */ - predicatelock_twophase_recover /* PredicateLock */ + predicatelock_twophase_recover, /* PredicateLock */ + NULL /* FILEOPS (post-commit/abort only) */ }; const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -36,7 +45,8 @@ const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_postcommit, /* Lock */ pgstat_twophase_postcommit, /* pgstat */ multixact_twophase_postcommit, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + fileops_twophase_postcommit /* FILEOPS */ }; const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -45,7 +55,8 @@ const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_postabort, /* Lock */ pgstat_twophase_postabort, /* pgstat */ multixact_twophase_postabort, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + fileops_twophase_postabort /* FILEOPS */ }; const TwoPhaseCallback twophase_standby_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -54,5 +65,6 @@ const TwoPhaseCallback twophase_standby_recover_callbacks[TWOPHASE_RM_MAX_ID + 1 lock_twophase_standby_recover, /* Lock */ NULL, /* pgstat */ NULL, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + NULL /* FILEOPS */ }; diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index d8d91bed42764..26f8589a50de8 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -26,6 +26,10 @@ #include "access/subtrans.h" #include "access/transam.h" #include "access/twophase.h" +#include "access/undo_xlog.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/xactundo.h" #include "access/xact.h" #include "access/xlog.h" #include "access/xloginsert.h" @@ -55,6 +59,7 @@ #include "storage/aio_subsys.h" #include "storage/condition_variable.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/lmgr.h" #include "storage/md.h" #include "storage/predicate.h" @@ -317,6 +322,16 @@ typedef struct XactCallbackItem static XactCallbackItem *Xact_callbacks = NULL; +/* + * Pending-structural-file-operation hooks, mirroring smgr's + * own pending-deletes housekeeping ordering (see access/xact.h). NULL when + * no such subsystem is compiled in. + */ +void (*PendingPhysOpsDo_hook) (bool isCommit) = NULL; +void (*PendingPhysOpsPostPrepare_hook) (void) = NULL; +void (*PendingPhysOpsAtSubCommit_hook) (void) = NULL; +void (*PendingPhysOpsAtSubAbort_hook) (void) = NULL; + /* * List of add-on start- and end-of-subxact callbacks */ @@ -419,6 +434,37 @@ IsAbortedTransactionBlockState(void) } +/* + * EnterInlineUndoApplyState / LeaveInlineUndoApplyState + * + * AtAbort_XactUndo() may apply UNDO records synchronously in this backend + * while AbortTransaction() has already set s->state = TRANS_ABORT but has + * not yet torn down the relcache, locks, or resource owner. UNDO appliers + * (e.g. a table AM's undo-apply callback) need to open relations, which trips + * Assert(IsTransactionState()) because TRANS_ABORT is not TRANS_INPROGRESS. + * + * These helpers temporarily present TRANS_INPROGRESS for the duration of the + * inline apply. This is safe because all backing resources are still live at + * this point in the abort sequence; only the state enum has advanced. The + * caller MUST pair Enter/Leave (use PG_TRY/PG_FINALLY) so the real abort state + * is always restored before the rest of AbortTransaction() proceeds. + */ +int +EnterInlineUndoApplyState(void) +{ + TransactionState s = CurrentTransactionState; + int saved = (int) s->state; + + s->state = TRANS_INPROGRESS; + return saved; +} + +void +LeaveInlineUndoApplyState(int saved) +{ + CurrentTransactionState->state = (TransState) saved; +} + /* * GetTopTransactionId * @@ -1814,6 +1860,8 @@ RecordTransactionAbort(bool isSubXact) TransactionId *children; TimestampTz xact_time; bool replorigin; + char *fileops; + uint32 fileops_len; /* * If we haven't been assigned an XID, nobody will care whether we aborted @@ -1832,9 +1880,9 @@ RecordTransactionAbort(bool isSubXact) /* * We have a valid XID, so we should write an ABORT record for it. * - * We do not flush XLOG to disk here, since the default assumption after a - * crash would be that we aborted, anyway. For the same reason, we don't - * need to worry about interlocking against checkpoint start. + * An ordinary abort need not flush WAL: a crash treats it as a loser. + * FILEOPS inverses, however, must follow a durable abort decision, with + * checkpoint start delayed until their filesystem effects are synced. */ /* @@ -1856,6 +1904,10 @@ RecordTransactionAbort(bool isSubXact) nchildren = xactGetCommittedChildren(&children); ndroppedstats = pgstat_get_transactional_drops(false, &droppedstats); + FileOpsGetAbortData(&fileops, &fileops_len); + if (fileops_len) + FileOpsBeginAbort(); + /* XXX do we really need a critical section here? */ START_CRIT_SECTION(); @@ -1872,7 +1924,7 @@ RecordTransactionAbort(bool isSubXact) nrels, rels, ndroppedstats, droppedstats, MyXactFlags, InvalidTransactionId, - NULL); + NULL, fileops, fileops_len); if (replorigin) /* Move LSNs forward for this replication origin */ @@ -1901,8 +1953,26 @@ RecordTransactionAbort(bool isSubXact) */ TransactionIdAbortTree(xid, nchildren, children); + if (fileops_len) + XLogFlush(XactLastRecEnd); END_CRIT_SECTION(); + if (fileops_len) + { + PG_TRY(); + { + INJECTION_POINT("fileops-abort-after-flush", NULL); + FileOpsApplyAbort(fileops, fileops_len); + } + PG_FINALLY(); + { + FileOpsEndAbort(); + } + PG_END_TRY(); + } + if (fileops) + pfree(fileops); + /* Compute latestXid while we have the child XIDs handy */ latestXid = TransactionIdLatest(xid, nchildren, children); @@ -2464,6 +2534,9 @@ CommitTransaction(void) CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_COMMIT : XACT_EVENT_COMMIT); + /* Clean up transaction undo state (free per-persistence record sets) */ + AtCommit_XactUndo(); + CurrentResourceOwner = NULL; ResourceOwnerRelease(TopTransactionResourceOwner, RESOURCE_RELEASE_BEFORE_LOCKS, @@ -2508,6 +2581,8 @@ CommitTransaction(void) * attempt to access affected files. */ smgrDoPendingDeletes(true); + if (PendingPhysOpsDo_hook) + PendingPhysOpsDo_hook(true); /* * Send out notification signals to other backends (and do other @@ -2796,6 +2871,8 @@ PrepareTransaction(void) PostPrepare_Inval(); PostPrepare_smgr(); + if (PendingPhysOpsPostPrepare_hook) + PendingPhysOpsPostPrepare_hook(); PostPrepare_MultiXact(fxid); @@ -2834,6 +2911,16 @@ PrepareTransaction(void) AtEOXact_LogicalCtl(); pgstat_report_xact_timestamp(0); + /* + * Drop this backend's references to the transaction's cluster-wide UNDO + * record sets. Ownership passed to the 2PC state (undo_batch_lsn) in + * EndPrepare; the record sets live in TopTransactionContext, which + * AtCommit_Memory() destroys just below, so the backend-private pointers + * must be dropped first (otherwise AtProcExit_XactUndo frees freed + * memory). + */ + AtPrepare_XactUndo(); + CurrentResourceOwner = NULL; ResourceOwnerDelete(TopTransactionResourceOwner); s->curTransactionOwner = NULL; @@ -2978,6 +3065,22 @@ AbortTransaction(void) s->parallelModeLevel = 0; s->parallelChildXact = false; /* should be false already */ + /* + * Apply the transaction's UNDO, except FILEOPS: its inverses ride in the + * abort record (RecordTransactionAbort), so that a standby and crash + * recovery apply them too. The generic chain is for crash losers. + */ + FileOpsSkipUndo = true; + PG_TRY(); + { + AtAbort_XactUndo(); + } + PG_FINALLY(); + { + FileOpsSkipUndo = false; + } + PG_END_TRY(); + /* * do abort processing */ @@ -3045,7 +3148,10 @@ AbortTransaction(void) ResourceOwnerRelease(TopTransactionResourceOwner, RESOURCE_RELEASE_AFTER_LOCKS, false, true); + smgrDoPendingDeletes(false); + if (PendingPhysOpsDo_hook) + PendingPhysOpsDo_hook(false); AtEOXact_GUC(false, 1); AtEOXact_SPI(false); @@ -5262,6 +5368,8 @@ CommitSubTransaction(void) AtEOSubXact_TypeCache(); AtEOSubXact_Inval(true); AtSubCommit_smgr(); + if (PendingPhysOpsAtSubCommit_hook) + PendingPhysOpsAtSubCommit_hook(); /* * The only lock we actually release here is the subtransaction XID lock. @@ -5453,6 +5561,8 @@ AbortSubTransaction(void) RESOURCE_RELEASE_AFTER_LOCKS, false, false); AtSubAbort_smgr(); + if (PendingPhysOpsAtSubAbort_hook) + PendingPhysOpsAtSubAbort_hook(); AtEOXact_GUC(false, s->gucNestLevel); AtEOSubXact_SPI(false, s->subTransactionId); @@ -6092,7 +6202,8 @@ XactLogAbortRecord(TimestampTz abort_time, int nrels, RelFileLocator *rels, int ndroppedstats, xl_xact_stats_item *droppedstats, int xactflags, TransactionId twophase_xid, - const char *twophase_gid) + const char *twophase_gid, + const char *fileops, uint32 fileops_len) { xl_xact_abort xlrec; xl_xact_xinfo xl_xinfo; @@ -6107,7 +6218,7 @@ XactLogAbortRecord(TimestampTz abort_time, Assert(CritSectionCount > 0); - xl_xinfo.xinfo = 0; + xl_xinfo.xinfo = fileops_len ? XACT_XINFO_HAS_FILEOPS : 0; /* decide between a plain and 2pc abort */ if (!TransactionIdIsValid(twophase_xid)) @@ -6220,6 +6331,12 @@ XactLogAbortRecord(TimestampTz abort_time, if (xl_xinfo.xinfo & XACT_XINFO_HAS_ORIGIN) XLogRegisterData(&xl_origin, sizeof(xl_xact_origin)); + if (fileops_len) + { + XLogRegisterData(&fileops_len, sizeof(fileops_len)); + XLogRegisterData(fileops, fileops_len); + } + /* Include the replication origin */ XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN); @@ -6441,6 +6558,13 @@ xact_redo_abort(xl_xact_parsed_abort *parsed, TransactionId xid, false /* backward */ , false /* WAL */ ); } + if (parsed->fileops_len) + { + XLogFlush(lsn); + INJECTION_POINT("fileops-abort-after-flush", NULL); + FileOpsApplyAbort(parsed->fileops, parsed->fileops_len); + } + /* Make sure files supposed to be dropped are dropped */ if (parsed->nrels > 0) { @@ -6478,6 +6602,11 @@ xact_redo(XLogReaderState *record) ParseCommitRecord(XLogRecGetInfo(record), xlrec, &parsed); xact_redo_commit(&parsed, XLogRecGetXid(record), record->EndRecPtr, XLogRecGetOrigin(record)); + + /* + * Remove from UNDO recovery tracking; committed, no rollback needed + */ + UndoRecoveryRemoveXid(XLogRecGetXid(record)); } else if (info == XLOG_XACT_COMMIT_PREPARED) { @@ -6492,6 +6621,9 @@ xact_redo(XLogReaderState *record) LWLockAcquire(TwoPhaseStateLock, LW_EXCLUSIVE); PrepareRedoRemove(parsed.twophase_xid, false); LWLockRelease(TwoPhaseStateLock); + + /* Remove from UNDO recovery tracking */ + UndoRecoveryRemoveXid(parsed.twophase_xid); } else if (info == XLOG_XACT_ABORT) { @@ -6501,6 +6633,13 @@ xact_redo(XLogReaderState *record) ParseAbortRecord(XLogRecGetInfo(record), xlrec, &parsed); xact_redo_abort(&parsed, XLogRecGetXid(record), record->EndRecPtr, XLogRecGetOrigin(record)); + + /* + * Remove from UNDO recovery tracking; abort record present means the + * UNDO rollback was already completed (or will be handled by the + * abort record's own redo logic). + */ + UndoRecoveryRemoveXid(XLogRecGetXid(record)); } else if (info == XLOG_XACT_ABORT_PREPARED) { @@ -6515,12 +6654,21 @@ xact_redo(XLogReaderState *record) LWLockAcquire(TwoPhaseStateLock, LW_EXCLUSIVE); PrepareRedoRemove(parsed.twophase_xid, false); LWLockRelease(TwoPhaseStateLock); + + /* Remove from UNDO recovery tracking */ + UndoRecoveryRemoveXid(parsed.twophase_xid); } else if (info == XLOG_XACT_PREPARE) { + xl_xact_prepare *xlrec = (xl_xact_prepare *) XLogRecGetData(record); + /* * Store xid and start/end pointers of the WAL record in TwoPhaseState * gxact entry. + * + * NB: xl_xact_prepare includes last_batch_lsn[NUndoPersistenceLevels] + * for UNDO chain tracking across 2PC boundaries, which extended the + * on-disk struct and required an XLOG_PAGE_MAGIC bump. */ LWLockAcquire(TwoPhaseStateLock, LW_EXCLUSIVE); PrepareRedoAdd(InvalidFullTransactionId, @@ -6529,6 +6677,21 @@ xact_redo(XLogReaderState *record) record->EndRecPtr, XLogRecGetOrigin(record)); LWLockRelease(TwoPhaseStateLock); + + /* + * Restore UNDO recovery tracking for the prepared transaction. The + * UNDO chain LSNs were saved in the prepare record so that if the + * server crashes after PREPARE but before COMMIT/ROLLBACK PREPARED, + * recovery can still find and roll back UNDO records. + */ + for (int j = 0; j < NUndoPersistenceLevels; j++) + { + if (!XLogRecPtrIsInvalid(xlrec->last_batch_lsn[j])) + UndoRecoveryTrackBatch(xlrec->xid, + xlrec->last_batch_lsn[j], + InvalidXLogRecPtr, + (UndoPersistenceLevel) j); + } } else if (info == XLOG_XACT_ASSIGNMENT) { diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index fd37ea4f607f8..4754d8cd9360a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -55,7 +55,10 @@ #include "access/timeline.h" #include "access/transam.h" #include "access/twophase.h" +#include "access/undolog.h" +#include "access/undo_xlog.h" #include "access/xact.h" +#include "access/xactundo.h" #include "access/xlog_internal.h" #include "access/xlogarchive.h" #include "access/xloginsert.h" @@ -6836,6 +6839,14 @@ StartupXLOG(void) Insert->fullPageWrites = lastFullPageWrites; UpdateFullPageWrites(); + /* + * Roll back the transactions the crash left in progress that wrote UNDO + * records, and WAL-log each rollback, before the end-of-recovery + * checkpoint or record below makes the rollbacks durable. + */ + if (performedWalRecovery) + UndoRecoveryRollBack(); + /* * Emit checkpoint or end-of-recovery record in XLOG, if required. */ @@ -8055,6 +8066,14 @@ CreateCheckPoint(int flags) if (!shutdown && XLogStandbyInfoActive()) LogStandbySnapshot(); + /* + * Tell crash recovery, which starts at our redo pointer, about the UNDO + * chains of transactions still in flight; it would not replay the batches + * they wrote before the redo pointer. + */ + if (!shutdown) + LogXactUndoInflight(); + START_CRIT_SECTION(); /* @@ -8231,6 +8250,7 @@ CreateCheckPoint(int flags) if (!RecoveryInProgress()) TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning()); + /* Real work is done; log and update stats. */ LogCheckpointEnd(false, flags); @@ -8971,6 +8991,26 @@ KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo) segno = unsummarized_segno; } + /* + * If UNDO-in-WAL is active, retain WAL segments that contain UNDO records + * still needed for rollback of in-progress transactions. + * + * Scan live per-backend UNDO batch LSN slots at every checkpoint rather + * than using the worker-updated cached horizon, to ensure WAL retention + * is accurate even when the UNDO worker lags. + */ + { + keep = UndoGetOldestBatchLSN(); + if (XLogRecPtrIsValid(keep)) + { + XLogSegNo undo_segno; + + XLByteToSeg(keep, undo_segno, wal_segment_size); + if (undo_segno < segno) + segno = undo_segno; + } + } + /* but, keep at least wal_keep_size if that's set */ if (wal_keep_size_mb > 0) { diff --git a/src/backend/access/undo/Makefile b/src/backend/access/undo/Makefile new file mode 100644 index 0000000000000..06a8a9eeda79a --- /dev/null +++ b/src/backend/access/undo/Makefile @@ -0,0 +1,25 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for access/undo +# +# IDENTIFICATION +# src/backend/access/undo/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/access/undo +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = \ + undo.o \ + undo_xlog.o \ + undoapply.o \ + undoinsert.o \ + undolog.o \ + undorecord.o \ + undormgr.o \ + xactundo.o + +include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/undo/README b/src/backend/access/undo/README new file mode 100644 index 0000000000000..0716d69d263a6 --- /dev/null +++ b/src/backend/access/undo/README @@ -0,0 +1,100 @@ +UNDO for PostgreSQL +=================== + +This directory implements a cluster-wide UNDO engine: a way for a +transaction to record how to reverse a change it has made, so that the +change is reversed if the transaction rolls back, or if a crash leaves it +in progress. The first consumer is FILEOPS, the transactional filesystem +operations in storage/file/fileops.c; see storage/file/README. + +Records and chains +------------------ + +A consumer writes UNDO through xactundo.c. PrepareXactUndoData() appends a +record (an UndoRecordHeader, undorecord.h, and an opaque payload) to the +transaction's UndoRecordSet for the record's persistence level, and +InsertXactUndoData() writes the set as one XLOG_UNDO_BATCH WAL record, +inside the critical section of the consumer's own WAL record. There are no +UNDO files: the records live in the WAL stream. + +Each batch carries the LSN of the transaction's previous batch +(xl_undo_batch.chain_prev), so a transaction's UNDO forms a chain from its +newest batch back to its first. Every record carries the top-level xid, +even one written by a subtransaction. urec_rmid names the UNDO resource +manager that applies the record (undormgr.h); consumers register theirs +through access/undormgrlist.h, and the engine never interprets a payload. + +A record is applied by handing it to its resource manager's rm_undo +callback. The engine keeps no "already applied" marker, and the same +record can be applied more than once, so a callback must leave alone an +object that no longer shows the record's effect. A callback must not +write WAL, since redo of XLOG_UNDO_ROLLBACK (below) calls it. + +Rollback +-------- + +AbortTransaction() calls AtAbort_XactUndo(), which walks the chain from its +newest batch, applying each batch's records newest first +(ApplyUndoChainFromWAL(), undoapply.c). A subtransaction abort applies only +the batches written since the subtransaction started, and a subtransaction +commit hands its chain to the parent. ROLLBACK PREPARED applies the chain +whose head PREPARE saved in the 2PC state. + +A consumer decides how its rollback reaches a standby. FILEOPS carries its +inverses in the abort record and applies them from there, so the +transaction-abort walk skips its records (FileOpsSkipUndo). + +Crash recovery +-------------- + +During redo, each XLOG_UNDO_BATCH record registers its transaction in an +in-memory table (undo_xlog.c), and the transaction's commit or abort +record removes it. Redo starts at the checkpoint's redo pointer, so it +replays none of the batches written before it; each online checkpoint +therefore writes an XLOG_UNDO_INFLIGHT record listing the chain heads of +the transactions in progress (LogXactUndoInflight()), which redo adds to +the table. InsertXactUndoData() holds off the checkpoint's redo-pointer +phase while it inserts a batch and publishes its chain head, so no batch +can fall between the two. + +What remains in the table at the end of redo, apart from prepared +transactions, are the transactions the crash left in progress. +StartupXLOG() calls UndoRecoveryRollBack() once WAL is writable, just +before the end-of-recovery checkpoint or record. For each such loser it +applies the UNDO records, newest first, then writes an XLOG_UNDO_ROLLBACK +record carrying the records that applied, and forgets the loser. Redo of +that record applies the same records again and forgets the loser too. So +a standby rolls the loser back at the same point in the WAL as the primary +did, without reading the loser's chain (which a restartpoint may already +have removed), and a later recovery never reverts it a second time. + +A crash before the rollback record reaches disk leaves the loser tracked, +and the next recovery rolls it back again. A record whose callback fails +is left out of the rollback record, so the change it describes stays in +place everywhere, and the summary at the end of the undo phase says how +many records could not be applied. + +Recovery applies only PERMANENT records: temporary relations do not +survive a restart, and unlogged ones are reset to their init forks. + +WAL retention +------------- + +Each backend publishes the LSN of its current transaction's first UNDO +batch in a shared slot (undolog.c), and a prepared transaction's chain +head is kept in its 2PC state. KeepLogSeg() keeps WAL from the oldest of +them onward, so a long-running or long-prepared transaction that wrote +UNDO holds back WAL removal until it ends, much as a replication slot +does. + +Files +----- + + undo.c shared memory set-up and per-backend initialization + undolog.c per-backend first-batch LSN slots for WAL retention + undorecord.c UndoRecordHeader and UndoRecordSet + undoinsert.c writing an UndoRecordSet as an XLOG_UNDO_BATCH record + xactundo.c per-transaction UNDO, the checkpoint in-flight record + undoapply.c online rollback: walking a chain and applying it + undo_xlog.c RM_UNDO_ID redo, the recovery undo phase, batch reads + undormgr.c UNDO resource manager registry diff --git a/src/backend/access/undo/meson.build b/src/backend/access/undo/meson.build new file mode 100644 index 0000000000000..e2f871f7194fd --- /dev/null +++ b/src/backend/access/undo/meson.build @@ -0,0 +1,12 @@ +# Copyright (c) 2022-2026, PostgreSQL Global Development Group + +backend_sources += files( + 'undo.c', + 'undo_xlog.c', + 'undoapply.c', + 'undoinsert.c', + 'undolog.c', + 'undorecord.c', + 'undormgr.c', + 'xactundo.c', +) diff --git a/src/backend/access/undo/undo.c b/src/backend/access/undo/undo.c new file mode 100644 index 0000000000000..d45d2c59f4d65 --- /dev/null +++ b/src/backend/access/undo/undo.c @@ -0,0 +1,212 @@ +/*------------------------------------------------------------------------- + * + * undo.c + * Common undo layer coordination + * + * The undo subsystem consists of several logically separate subsystems + * that work together to achieve a common goal. The code in this file + * provides a limited amount of common infrastructure that can be used + * by all of those various subsystems, and helps coordinate activities + * such as shared memory initialization and startup/shutdown. + * + * This file has no compile-time or link-time dependency on any specific + * table or index access method. Shared memory for AM-specific tracking + * structures (such as an optional per-tuple tracking hash an AM may register) + * is registered as its own sibling entry in storage/subsystemlist.h, not + * sized or initialized here. Likewise, the + * set of UNDO resource managers to register comes from access/undormgrlist.h + * (mirroring the subsystemlist.h idiom): a new consumer adds itself to that + * list, and never edits this file. + * + * Shared memory initialization uses the PG_SHMEM_SUBSYSTEM pattern: + * UndoShmemCallbacks is registered in subsystemlist.h, and the framework + * calls UndoShmemRequest() and UndoShmemInit() at the appropriate times + * during postmaster startup. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/backend/access/undo/undo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undo.h" +#include "access/undolog.h" +#include "access/undormgr.h" +#include "access/undormgrs.h" +#include "access/xactundo.h" +#include "miscadmin.h" +#include "storage/ipc.h" +#include "storage/shmem.h" +#include "storage/subsystems.h" +#include "utils/memutils.h" + +/* + * UndoContext is a child of TopMemoryContext which is never reset. The only + * reason for having a separate context is to make it easier to spot leaks or + * excessive memory utilization related to undo operations. + */ +MemoryContext UndoContext = NULL; + +static void AtProcExit_Undo(int code, Datum arg); +static void UndoShmemRequest_internal(void *arg); +static void UndoShmemInit_internal(void *arg); +static void UndoShmemAttach_internal(void *arg); +static void RegisterUndoRmgrs(void); + +/* + * ShmemCallbacks for the UNDO subsystem. + * + * Registered via PG_SHMEM_SUBSYSTEM(UndoShmemCallbacks) in subsystemlist.h. + * + * init_fn initializes the contents of all UNDO shared memory structures. + */ +const ShmemCallbacks UndoShmemCallbacks = { + .request_fn = UndoShmemRequest_internal, + .init_fn = UndoShmemInit_internal, + .attach_fn = UndoShmemAttach_internal, +}; + +/* + * UndoShmemRequest_internal + * Register shared memory needs for UNDO subsystems. + * + * Called during the request_fn phase of postmaster startup, before shared + * memory is allocated. + */ +static void +UndoShmemRequest_internal(void *arg) +{ + /* Shared areas of the UNDO sub-modules. */ + UndoLogShmemRequest(); + XactUndoShmemRequest(); +} + +/* + * UndoShmemInit / UndoShmemInit_internal + * Initialize undo-related shared memory. + * + * Also, perform other initialization steps that need to be done very early. + * This is called once during postmaster startup via the ShmemCallbacks + * framework. + */ +static void +UndoShmemInit_internal(void *arg) +{ + UndoShmemInit(); +} + +void +UndoShmemInit(void) +{ + /* + * Initialize the undo memory context. If it already exists (crash restart + * via reset_shared()), reset it instead. + */ + if (UndoContext) + MemoryContextReset(UndoContext); + else + UndoContext = AllocSetContextCreate(TopMemoryContext, "Undo", + ALLOCSET_DEFAULT_SIZES); + + /* Now give various undo subsystems a chance to initialize. */ + UndoLogShmemInit(); + XactUndoShmemInit(); + + /* + * Initialize the UNDO resource manager dispatch table and register the + * built-in resource managers listed in access/undormgrlist.h. + */ + RegisterUndoRmgrs(); +} + +/* + * UndoShmemAttach_internal + * Re-establish per-process UNDO state in EXEC_BACKEND children. + * + * Under EXEC_BACKEND (Windows, or --enable-exec-backend builds) a child does + * not inherit the postmaster's address space, so module-scope state populated + * by UndoShmemInit() is not present and must be rebuilt here. Two distinct + * kinds of state need re-establishing: + * + * 1. Process-local pointers derived from the requested shmem areas. Every + * UNDO shmem area is requested with ShmemRequestStruct() from + * UndoShmemRequest_internal(), so the framework has already set each + * module's base pointer before calling us; the sub-modules' *ShmemInit() + * functions initialize shared contents and must not be called here. + * + * 2. The UndoRmgrs[] dispatch table and each registered AM's undo hook + * function pointers, both rebuilt by RegisterUndoRmgrs() (idempotent: + * InitUndoRmgrs() zeroes the table first). + * + * Any AM-specific shared structure handles its own EXEC_BACKEND re-attach + * via its own PG_SHMEM_SUBSYSTEM entry (see storage/subsystemlist.h). + */ +static void +UndoShmemAttach_internal(void *arg) +{ + RegisterUndoRmgrs(); +} + +/* + * RegisterUndoRmgrs + * Initialize the UNDO resource manager dispatch table and register the + * built-in resource managers listed in access/undormgrlist.h. + * + * Called from both UndoShmemInit() (postmaster/standalone) and + * UndoShmemAttach_internal() (EXEC_BACKEND children). Idempotent: + * InitUndoRmgrs() zeroes the dispatch table before the *UndoRmgrInit() calls + * re-register, so RegisterUndoRmgr()'s double-registration guard is not + * tripped on a second invocation in the same process. + * + * This function has no compile-time knowledge of which resource managers + * exist: it just expands access/undormgrlist.h through the UNDO_RMGR_INIT + * macro. A new UNDO-writing consumer (a new index AM, table AM, or other + * subsystem) adds one line to that list; this function and the rest of + * undo.c are never modified. + */ +static void +RegisterUndoRmgrs(void) +{ + /* + * Initialize the UNDO resource manager dispatch table. + */ + InitUndoRmgrs(); + + /* + * Register every built-in resource manager listed in + * access/undormgrlist.h. + */ +#define UNDO_RMGR_INIT(initfunc) \ + initfunc(); +#include "access/undormgrlist.h" +#undef UNDO_RMGR_INIT +} + +/* + * InitializeUndo + * Per-backend initialization for the undo subsystem. + * + * Called once per backend from InitPostgres(). + */ +void +InitializeUndo(void) +{ + InitializeXactUndo(); + + on_shmem_exit(AtProcExit_Undo, 0); +} + +/* + * AtProcExit_Undo + * Shut down undo subsystems in the correct order. + * + * Higher-level stuff should be shut down first. + */ +static void +AtProcExit_Undo(int code, Datum arg) +{ + AtProcExit_XactUndo(); +} diff --git a/src/backend/access/undo/undo_xlog.c b/src/backend/access/undo/undo_xlog.c new file mode 100644 index 0000000000000..3e3a87cde67cc --- /dev/null +++ b/src/backend/access/undo/undo_xlog.c @@ -0,0 +1,878 @@ +/*------------------------------------------------------------------------- + * + * undo_xlog.c + * UNDO resource manager WAL redo routines + * + * This module implements the WAL redo callback for the RM_UNDO_ID resource + * manager. It handles replay of: + * + * XLOG_UNDO_ROLLBACK - Reapply a crash loser's rollback + * XLOG_UNDO_BATCH - Track a transaction's UNDO for recovery + * XLOG_UNDO_INFLIGHT - Track UNDO chains in flight at a checkpoint + * + * It also holds the recovery undo phase (UndoRecoveryRollBack), which rolls + * back the transactions a crash left in progress, and the code that reads + * UNDO batches back from WAL. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undo_xlog.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/transam.h" +#include "access/twophase.h" +#include "access/undo_xlog.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/xlog.h" +#include "access/xlog_internal.h" +#include "access/xloginsert.h" +#include "access/xlogreader.h" +#include "access/xlogrecovery.h" +#include "access/xlogutils.h" +#include "miscadmin.h" +#include "utils/memutils.h" + +static int undo_apply_records(TransactionId xid, const char *pos, Size len, + int *failed, StringInfo done); + +/* + * undo_redo - Replay an UNDO WAL record during crash recovery + */ +void +undo_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_UNDO_ROLLBACK: + { + xl_undo_rollback *xlrec = (xl_undo_rollback *) XLogRecGetData(record); + + (void) undo_apply_records(xlrec->xid, + XLogRecGetData(record) + SizeOfUndoRollback, + XLogRecGetDataLen(record) - SizeOfUndoRollback, + NULL, NULL); + UndoRecoveryRemoveXid(xlrec->xid); + } + break; + + case XLOG_UNDO_INFLIGHT: + { + xl_undo_inflight *xlrec = (xl_undo_inflight *) XLogRecGetData(record); + + for (uint32 i = 0; i < xlrec->nxacts; i++) + UndoRecoveryTrackInflight(&xlrec->xacts[i]); + } + break; + + case XLOG_UNDO_BATCH: + { + xl_undo_batch *xlrec = (xl_undo_batch *) XLogRecGetData(record); + + /* + * During recovery, track this batch for incomplete + * transaction detection. After redo completes, any + * transaction that wrote UNDO batches but did not commit will + * need its UNDO chain walked for rollback. + * + * The batch payload (serialized UNDO records) is part of the + * WAL record and can be re-read later via XLogReadRecord() + * during the undo phase. + */ + UndoRecoveryTrackBatch(xlrec->xid, record->ReadRecPtr, + xlrec->chain_prev, + xlrec->persistence); + + ereport(DEBUG2, + (errmsg("undo_redo: BATCH xid %u, nrecords %u, " + "total_len %u, chain_prev %X/%08X", + xlrec->xid, xlrec->nrecords, + xlrec->total_len, + LSN_FORMAT_ARGS(xlrec->chain_prev)))); + } + break; + + default: + elog(PANIC, "undo_redo: unknown op code %u", info); + } +} + +/* ---------------------------------------------------------------- + * UNDO recovery tracking + * + * During WAL redo, we track which transactions wrote UNDO batches. + * When a commit/abort record is redone, the XID is removed. + * After redo completes, remaining entries represent incomplete + * transactions that need their UNDO chains walked. + * ---------------------------------------------------------------- + */ + +/* Hash table entry for tracking incomplete UNDO transactions */ +typedef struct UndoRecoveryEntry +{ + TransactionId xid; /* hash key */ + XLogRecPtr last_batch_lsn[NUndoPersistenceLevels]; /* chain heads per + * persistence level */ + char status; /* in use */ +} UndoRecoveryEntry; + +/* Simple dynamic array for recovery tracking (used during startup only) */ +static UndoRecoveryEntry *undo_recovery_entries = NULL; +static int undo_recovery_nentries = 0; +static int undo_recovery_capacity = 0; + +/* + * Safety cap to prevent OOM during recovery. If more than this many + * distinct in-flight XIDs are found in WAL at crash time, we stop + * tracking new ones and log a warning. The untracked transactions will + * need manual resolution (e.g. via pg_resetwal or targeted UNDO apply). + * + * 1 million entries x ~40 bytes each ~ 40 MB, which is reasonable for + * a recovery-only allocation. + */ +#define UNDO_RECOVERY_MAX_ENTRIES 1048576 + +/* + * undo_recovery_entry - Find or create the recovery entry for xid + * + * Returns NULL if the entry would exceed UNDO_RECOVERY_MAX_ENTRIES. + */ +static UndoRecoveryEntry * +undo_recovery_entry(TransactionId xid) +{ + int i; + UndoRecoveryEntry *entry = NULL; + + /* Find existing entry for this XID */ + for (i = 0; i < undo_recovery_nentries; i++) + { + if (undo_recovery_entries[i].xid == xid) + return &undo_recovery_entries[i]; + } + + /* Safety cap: refuse to track more XIDs to prevent OOM */ + if (undo_recovery_nentries >= UNDO_RECOVERY_MAX_ENTRIES) + { + static bool warned = false; + + if (!warned) + { + ereport(WARNING, + (errmsg("UNDO recovery: reached maximum tracked transaction limit (%d)", + UNDO_RECOVERY_MAX_ENTRIES), + errhint("Transactions beyond this limit will not be automatically rolled back. " + "Manual intervention may be required after recovery completes."))); + warned = true; + } + return NULL; + } + + if (undo_recovery_nentries >= undo_recovery_capacity) + { + int new_capacity = (undo_recovery_capacity == 0) ? 64 : + undo_recovery_capacity * 2; + UndoRecoveryEntry *new_entries; + + /* Clamp doubling to not exceed the safety cap */ + if (new_capacity > UNDO_RECOVERY_MAX_ENTRIES) + new_capacity = UNDO_RECOVERY_MAX_ENTRIES; + + if (undo_recovery_entries == NULL) + { + new_entries = (UndoRecoveryEntry *) + palloc0(sizeof(UndoRecoveryEntry) * new_capacity); + } + else + { + new_entries = (UndoRecoveryEntry *) + repalloc(undo_recovery_entries, + sizeof(UndoRecoveryEntry) * new_capacity); + memset(&new_entries[undo_recovery_capacity], 0, + sizeof(UndoRecoveryEntry) * (new_capacity - undo_recovery_capacity)); + } + undo_recovery_entries = new_entries; + undo_recovery_capacity = new_capacity; + } + + entry = &undo_recovery_entries[undo_recovery_nentries++]; + entry->xid = xid; + for (i = 0; i < NUndoPersistenceLevels; i++) + entry->last_batch_lsn[i] = InvalidXLogRecPtr; + + return entry; +} + +/* + * UndoRecoveryTrackBatch - Record an UNDO batch during WAL redo + * + * Called from the XLOG_UNDO_BATCH redo handler to track which + * transactions have UNDO data that may need rollback. + */ +void +UndoRecoveryTrackBatch(TransactionId xid, XLogRecPtr batch_lsn, + XLogRecPtr chain_prev, + UndoPersistenceLevel persistence) +{ + UndoRecoveryEntry *entry; + + if (!TransactionIdIsValid(xid)) + return; + entry = undo_recovery_entry(xid); + if (entry == NULL) + return; + + /* Update the chain head for this persistence level */ + if (persistence < NUndoPersistenceLevels) + entry->last_batch_lsn[persistence] = batch_lsn; +} + +/* + * UndoRecoveryTrackInflight - Record a chain listed by XLOG_UNDO_INFLIGHT + * + * The checkpoint that wrote the record saw the transaction in flight. Redo + * started at its redo pointer, so batches the transaction wrote before it + * were not replayed and the entry may not exist yet. A batch already seen + * by redo is at least as new as the one listed, so keep the later head. A + * transaction whose commit or abort precedes the record was listed after it + * ended; its outcome is already in pg_xact, so it is ignored. + */ +void +UndoRecoveryTrackInflight(const xl_undo_inflight_xact *x) +{ + UndoRecoveryEntry *entry; + + if (!TransactionIdIsValid(x->xid) || + TransactionIdDidCommit(x->xid) || TransactionIdDidAbort(x->xid)) + return; + entry = undo_recovery_entry(x->xid); + if (entry == NULL) + return; + for (int i = 0; i < NUndoPersistenceLevels; i++) + { + if (XLogRecPtrIsValid(x->last_batch_lsn[i]) && + (!XLogRecPtrIsValid(entry->last_batch_lsn[i]) || + x->last_batch_lsn[i] > entry->last_batch_lsn[i])) + entry->last_batch_lsn[i] = x->last_batch_lsn[i]; + } +} + +/* + * UndoRecoveryRemoveXid - Remove an XID from recovery tracking + * + * Called when a commit or abort record is redone during recovery. + * Committed transactions don't need UNDO rollback. Aborted transactions + * that were already fully rolled back (abort record present) also don't + * need further work. + */ +void +UndoRecoveryRemoveXid(TransactionId xid) +{ + int i; + + if (!TransactionIdIsValid(xid)) + return; + + for (i = 0; i < undo_recovery_nentries; i++) + { + if (undo_recovery_entries[i].xid == xid) + { + /* Mark as removed by zeroing XID */ + undo_recovery_entries[i].xid = InvalidTransactionId; + break; + } + } +} + +/* + * UndoRecoveryNeeded - Check if there are incomplete transactions needing UNDO + * + * Returns true if any tracked transactions remain after redo is complete. + */ +static bool +UndoRecoveryNeeded(void) +{ + int i; + + for (i = 0; i < undo_recovery_nentries; i++) + { + if (TransactionIdIsValid(undo_recovery_entries[i].xid)) + return true; + } + + return false; +} + +/* + * UndoRecoveryRollingBack is true while UndoRecoveryRollBack() applies UNDO + * records. That runs after InRecovery has been cleared, but an object an + * UNDO record describes may have been recreated by redo, as during redo + * itself, so a resource manager that tests InRecovery should test this too. + */ +bool UndoRecoveryRollingBack = false; + +/* + * undo_apply_records - Apply a packed stream of UNDO records, in order + * + * Shared by the undo phase and by redo of XLOG_UNDO_ROLLBACK, so the two + * apply the same records the same way. The stream is the payload format of + * XLOG_UNDO_BATCH: each record is an UndoRecordHeader and its payload, + * urec_len bytes in all. Returns the number of records applied. If failed + * is not NULL, the number of records whose apply function reported an error + * is added to it, and if done is not NULL, each record that applied is + * appended to it. + */ +static int +undo_apply_records(TransactionId xid, const char *pos, Size len, + int *failed, StringInfo done) +{ + const char *end = pos + len; + int applied = 0; + + while ((Size) (end - pos) >= SizeOfUndoRecordHeader) + { + UndoRecordHeader header; + const UndoRmgrData *rmgr; + UndoApplyResult result; + + memcpy(&header, pos, SizeOfUndoRecordHeader); + if (header.urec_len < SizeOfUndoRecordHeader || + (Size) (end - pos) < header.urec_len || + header.urec_payload_len > header.urec_len - SizeOfUndoRecordHeader) + break; + + rmgr = GetUndoRmgr(header.urec_rmid); + if (rmgr != NULL) + { + result = rmgr->rm_undo(header.urec_rmid, header.urec_info, xid, + header.urec_reloid, + header.urec_payload_len > 0 ? + pos + SizeOfUndoRecordHeader : NULL, + header.urec_payload_len, + InvalidUndoRecPtr); + if (result == UNDO_APPLY_ERROR) + { + if (failed != NULL) + (*failed)++; + } + else + { + applied++; + if (done != NULL) + appendBinaryStringInfo(done, pos, header.urec_len); + } + } + pos += header.urec_len; + } + + return applied; +} + +/* + * undo_collect_chain - Gather one persistence level of a loser's UNDO chain + * + * Walks the chain from its newest batch back to its first and appends each + * batch's records to buf, newest batch first, so that applying buf in order + * applies every record newest first, as an online abort does. + * + * Returns false, having warned, if a batch could not be read: buf then holds + * the records of the newer batches only. + */ +static bool +undo_collect_chain(TransactionId xid, XLogRecPtr batch_lsn, StringInfo buf) +{ + int maxstarts = 64; + const char **starts = palloc_array(const char *, maxstarts); + + while (XLogRecPtrIsValid(batch_lsn)) + { + int nstarts; + UndoBatchData *batch; + XLogRecPtr next_lsn; + const char *pos; + const char *end; + + batch = UndoReadBatchFromWAL(batch_lsn); + if (batch == NULL) + { + /* + * The batch's WAL is gone or unreadable, for example because a + * restartpoint on a standby removed the segment it was in. Skip + * the rest of the transaction's rollback rather than PANIC, which + * would leave the cluster unable to start. + */ + ereport(WARNING, + (errmsg("UNDO recovery: could not read batch at %X/%08X for transaction %u; skipping rollback", + LSN_FORMAT_ARGS(batch_lsn), xid))); + pfree(starts); + return false; + } + + /* + * Find the whole records, then copy them newest first. A batch holds + * them in the order they were written. + */ + pos = batch->payload; + end = pos + batch->payload_len; + nstarts = 0; + while ((Size) (end - pos) >= SizeOfUndoRecordHeader) + { + UndoRecordHeader header; + + memcpy(&header, pos, SizeOfUndoRecordHeader); + if (header.urec_len < SizeOfUndoRecordHeader || + (Size) (end - pos) < header.urec_len) + break; + if (nstarts == maxstarts) + { + maxstarts *= 2; + starts = repalloc_array(starts, const char *, maxstarts); + } + starts[nstarts++] = pos; + pos += header.urec_len; + } + for (int i = nstarts - 1; i >= 0; i--) + { + UndoRecordHeader header; + + memcpy(&header, starts[i], SizeOfUndoRecordHeader); + appendBinaryStringInfo(buf, starts[i], header.urec_len); + } + + /* + * chain_prev must point strictly backward, or the walk would never + * end. + */ + next_lsn = batch->header.chain_prev; + if (XLogRecPtrIsValid(next_lsn) && next_lsn >= batch_lsn) + ereport(PANIC, + (errmsg("UNDO recovery: chain_prev %X/%08X >= batch_lsn %X/%08X for transaction %u; corrupt UNDO chain", + LSN_FORMAT_ARGS(next_lsn), + LSN_FORMAT_ARGS(batch_lsn), xid))); + UndoFreeBatchData(batch); + batch_lsn = next_lsn; + } + + pfree(starts); + return true; +} + +/* + * UndoRecoveryRollBack - Roll back the transactions a crash left in progress + * + * This is the undo phase. StartupXLOG() calls it once WAL can be written + * again, where resource managers may write cleanup records, just before the + * end-of-recovery checkpoint or record. For each transaction that redo found + * writing UNDO records but neither committing nor aborting, other than a + * prepared one, it applies the records, newest first, then writes an + * XLOG_UNDO_ROLLBACK record carrying the ones that applied, and forgets the + * transaction. A standby and any later recovery replay that record instead + * of reading the chain from WAL; see xl_undo_rollback. + * + * TEMP and UNLOGGED records are not applied: temporary relations do not + * survive a restart, and unlogged ones are reset to their init forks. + * + * A record whose apply function fails is left out of the rollback record, + * so the change it describes stays in place everywhere and no later replay + * retries it. The same goes for the records of a batch that could not be + * read, and of the batches older than it. The summary at the end says how + * many records could not be applied. + */ +void +UndoRecoveryRollBack(void) +{ + int total_xacts = 0; + int total_records = 0; + int failed_records = 0; + int incomplete_xacts = 0; + int pending_xacts = 0; + + if (!UndoRecoveryNeeded()) + return; + for (int i = 0; i < undo_recovery_nentries; i++) + { + if (TransactionIdIsValid(undo_recovery_entries[i].xid)) + pending_xacts++; + } + + ereport(LOG, + (errmsg("UNDO recovery: %d incomplete transaction(s) to roll back", + pending_xacts))); + + for (int i = 0; i < undo_recovery_nentries; i++) + { + UndoRecoveryEntry *entry = &undo_recovery_entries[i]; + xl_undo_rollback xlrec; + StringInfoData buf; + StringInfoData done; + int nrecords; + + if (!TransactionIdIsValid(entry->xid)) + continue; + + /* + * A prepared transaction stays prepared; COMMIT PREPARED or ROLLBACK + * PREPARED settles it later. + */ + if (RecoveryTransactionIdIsPrepared(entry->xid)) + { + ereport(LOG, + (errmsg("UNDO recovery: skipping prepared transaction %u " + "(will remain in prepared state)", + entry->xid))); + continue; + } + + total_xacts++; + ereport(LOG, + (errmsg("UNDO recovery: rolling back transaction %u", + entry->xid))); + + /* + * FILEOPS caps the UNDO a transaction can write (see + * FILEOPS_MAX_ABORT_BYTES), which keeps this record far below the WAL + * record size limit. + */ + initStringInfo(&buf); + initStringInfo(&done); + if (!undo_collect_chain(entry->xid, + entry->last_batch_lsn[UNDOPERSISTENCE_PERMANENT], + &buf)) + incomplete_xacts++; + + UndoRecoveryRollingBack = true; + nrecords = undo_apply_records(entry->xid, buf.data, buf.len, + &failed_records, &done); + UndoRecoveryRollingBack = false; + total_records += nrecords; + + /* + * Log the rollback, and only then forget the transaction. The apply + * functions made their changes durable, and a crash before this + * record reaches disk leaves the loser to be rolled back again. The + * end-of-recovery checkpoint or record flushes it. Write it even if + * nothing applied, so that a standby forgets the loser too. + */ + xlrec.xid = entry->xid; + xlrec.nrecords = nrecords; + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfUndoRollback); + XLogRegisterData(done.data, done.len); + (void) XLogInsert(RM_UNDO_ID, XLOG_UNDO_ROLLBACK); + + pfree(buf.data); + pfree(done.data); + entry->xid = InvalidTransactionId; + } + + if (failed_records > 0 || incomplete_xacts > 0) + ereport(WARNING, + (errmsg("UNDO recovery complete: %d transactions processed, " + "%d records applied, %d records could not be applied", + total_xacts, total_records, failed_records), + errdetail("The changes those records describe were not " + "rolled back."))); + else if (total_xacts > 0) + ereport(LOG, + (errmsg("UNDO recovery complete: %d transactions rolled back, " + "%d records applied", + total_xacts, total_records))); + + /* Only prepared transactions are left, and nothing tracks them now. */ + pfree(undo_recovery_entries); + undo_recovery_entries = NULL; + undo_recovery_nentries = 0; + undo_recovery_capacity = 0; +} + +/* ---------------------------------------------------------------- + * UNDO batch reading from WAL + * ---------------------------------------------------------------- + */ + +/* + * UndoReadBatchFromWAL - Read a single XLOG_UNDO_BATCH record from WAL + * + * Uses XLogReader to read the WAL record at the given LSN. + * Returns a palloc'd UndoBatchData containing the header and a copy + * of the payload. The caller must pfree via UndoFreeBatchData(). + * + * Returns NULL if the record cannot be read or is not an UNDO batch. + */ +/* + * Module-level cached XLogReader for UndoReadBatchFromWAL. + * Allocated once and reused across calls to avoid per-batch + * open/close overhead on WAL segment files during rollback. + */ +static XLogReaderState *undo_batch_reader = NULL; +static XLogReaderRoutine undo_batch_reader_routine = { + .page_read = read_local_xlog_page, + .segment_open = wal_segment_open, + .segment_close = wal_segment_close, +}; + +/* + * UndoValidateBatchLSN + * Quick check that the WAL record at batch_lsn is a valid UNDO source. + * + * Returns true if the record is a standalone RM_UNDO_ID batch. Returns + * false for any other unrecognized record type. Used by the inline UNDO + * path to avoid calling ApplyUndoChainFromWAL on a batch_lsn that points to + * the wrong record type. + */ +bool +UndoValidateBatchLSN(XLogRecPtr batch_lsn) +{ + XLogRecord *record_hdr; + char *errormsg = NULL; + uint8 rmid; + MemoryContext old_ctx; + bool result = false; + + if (!XLogRecPtrIsValid(batch_lsn)) + return false; + + /* + * Perform the entire read under TopMemoryContext. The cached reader and, + * critically, its lazily-allocated decode_buffer (xlogreader.c) are + * palloc'd in CurrentMemoryContext. Callers such as the inline-abort + * path run with a transient context that is deleted right after; + * allocating the reader state there would leave the static + * undo_batch_reader (and its decode buffer) dangling, crashing the next + * abort. TopMemoryContext makes the cache truly persistent. + */ + old_ctx = MemoryContextSwitchTo(TopMemoryContext); + + if (undo_batch_reader == NULL) + { + undo_batch_reader = XLogReaderAllocate(wal_segment_size, NULL, + &undo_batch_reader_routine, NULL); + if (undo_batch_reader == NULL) + { + MemoryContextSwitchTo(old_ctx); + return false; + } + } + + XLogBeginRead(undo_batch_reader, batch_lsn); + record_hdr = XLogReadRecord(undo_batch_reader, &errormsg); + if (record_hdr == NULL) + { + MemoryContextSwitchTo(old_ctx); + return false; + } + + rmid = XLogRecGetRmid(undo_batch_reader); + + /* Only a standalone UNDO batch is a valid inline UNDO source */ + if (rmid == RM_UNDO_ID) + result = true; + + MemoryContextSwitchTo(old_ctx); + return result; +} + +UndoBatchData * +UndoReadBatchFromWAL(XLogRecPtr batch_lsn) +{ + XLogRecord *record_hdr; + char *errormsg = NULL; + UndoBatchData *result; + xl_undo_batch *xlrec; + char *record_data; + Size record_len; + Size payload_offset; + + if (!XLogRecPtrIsValid(batch_lsn)) + return NULL; + + /* + * Safety check: verify the WAL segment containing this LSN has not been + * recycled by a checkpoint. If the LSN is behind the current redo + * pointer and the segment file doesn't exist, reading would cause SIGBUS + * (signal 10: Bus error) or SIGSEGV. Return NULL gracefully instead. + * + * Compare against GetRedoRecPtr(); if our target is well behind the redo + * pointer AND behind the last checkpoint's redo location, the segment may + * have been recycled. + */ + { + XLogRecPtr redo_ptr = GetRedoRecPtr(); + XLogSegNo target_segno; + char path[MAXPGPATH]; + TimeLineID tli; + + if (batch_lsn < redo_ptr) + { + /* + * Determine the timeline for the segment path. + * GetWALInsertionTimeLine() is only valid once recovery is DONE + * (it asserts RECOVERY_STATE_DONE); during the crash-recovery + * undo phase we must instead use the replay timeline. + * RecoveryInProgress() distinguishes the two callers + * (crash-recovery undo phase vs. online abort). + */ + if (RecoveryInProgress()) + { + (void) GetXLogReplayRecPtr(&tli); + if (tli == 0) + tli = 1; /* no replay position yet: default to TLI 1 */ + } + else + tli = GetWALInsertionTimeLine(); + + XLByteToSeg(batch_lsn, target_segno, wal_segment_size); + XLogFilePath(path, tli, target_segno, + wal_segment_size); + + if (access(path, F_OK) != 0) + { + ereport(WARNING, + (errmsg("UNDO batch at %X/%08X: WAL segment \"%s\" no longer " + "exists (recycled by checkpoint); skipping rollback", + LSN_FORMAT_ARGS(batch_lsn), path))); + return NULL; + } + } + } + + /* + * Allocate the reader and perform the read under TopMemoryContext. Both + * the cached reader and its lazily-allocated decode_buffer (xlogreader.c) + * are palloc'd in CurrentMemoryContext; if that is a transient caller + * context (e.g. the inline-abort context) it gets deleted, leaving the + * static undo_batch_reader and its decode buffer dangling and crashing + * the next abort. We switch back to the caller's context before + * allocating the returned UndoBatchData, which the caller owns. See the + * matching comment in UndoValidateBatchLSN. + */ + { + MemoryContext read_ctx = MemoryContextSwitchTo(TopMemoryContext); + + if (undo_batch_reader == NULL) + { + undo_batch_reader = XLogReaderAllocate(wal_segment_size, NULL, + &undo_batch_reader_routine, NULL); + if (undo_batch_reader == NULL) + { + MemoryContextSwitchTo(read_ctx); + ereport(WARNING, + (errmsg("could not allocate XLogReader for UNDO batch read"))); + return NULL; + } + } + + /* Position the reader at the target LSN, then read */ + XLogBeginRead(undo_batch_reader, batch_lsn); + record_hdr = XLogReadRecord(undo_batch_reader, &errormsg); + MemoryContextSwitchTo(read_ctx); + } + if (record_hdr == NULL) + { + if (errormsg) + ereport(WARNING, + (errmsg("could not read WAL record at %X/%08X: %s", + LSN_FORMAT_ARGS(batch_lsn), errormsg))); + return NULL; + } + + /* + * Determine record format: a standalone XLOG_UNDO_BATCH record carrying + * the UNDO payload. + */ + record_data = XLogRecGetData(undo_batch_reader); + record_len = XLogRecGetDataLen(undo_batch_reader); + + if (XLogRecGetRmid(undo_batch_reader) == RM_UNDO_ID && + (XLogRecGetInfo(undo_batch_reader) & ~XLR_INFO_MASK) == XLOG_UNDO_BATCH) + { + /* Standalone XLOG_UNDO_BATCH record */ + if (record_len < SizeOfUndoBatch) + { + ereport(WARNING, + (errmsg("UNDO batch record at %X/%08X too short: %zu bytes", + LSN_FORMAT_ARGS(batch_lsn), record_len))); + return NULL; + } + + xlrec = (xl_undo_batch *) record_data; + payload_offset = SizeOfUndoBatch; + } + else + { + ereport(WARNING, + (errmsg("WAL record at %X/%08X is not an UNDO batch (rmid=%u, info=0x%02x)", + LSN_FORMAT_ARGS(batch_lsn), + XLogRecGetRmid(undo_batch_reader), + XLogRecGetInfo(undo_batch_reader) & ~XLR_INFO_MASK))); + return NULL; + } + + /* + * Validate that the claimed payload length fits within the WAL record. A + * mismatch here means the WAL segment was recycled and overwritten (the + * LSN now points to a different record), or the record is corrupt. + * Without this check, the memcpy below would read past the XLogReader's + * internal buffer, potentially accessing unmapped memory + * (SIGBUS/SIGSEGV). + */ + if (payload_offset + (Size) xlrec->total_len > record_len) + { + ereport(WARNING, + (errmsg("UNDO batch at %X/%08X: payload length %u exceeds " + "record data (offset %zu, record_len %zu)", + LSN_FORMAT_ARGS(batch_lsn), + xlrec->total_len, payload_offset, record_len))); + return NULL; + } + + /* + * Allocate UndoBatchData. We use palloc (CurrentMemoryContext) because + * this structure is only needed until ApplyUndoChainFromWAL processes the + * batch. We intentionally do NOT pfree in UndoFreeBatchData() because + * calling pfree on BumpContext memory would ERROR. The memory will be + * reclaimed when the current memory context is reset. + */ + result = (UndoBatchData *) palloc(sizeof(UndoBatchData)); + memcpy(&result->header, xlrec, SizeOfUndoBatch); + result->payload_len = (Size) xlrec->total_len; + if (result->payload_len > 0) + { + result->payload = (char *) palloc(result->payload_len); + memcpy(result->payload, record_data + payload_offset, + result->payload_len); + } + else + { + result->payload = NULL; + } + + /* Do not free reader -- it is cached for reuse. */ + return result; +} + +/* + * UndoFreeBatchData - Release a UndoBatchData structure + * + * This is a no-op function. We don't actually pfree the batch or payload + * because they were allocated with palloc() from CurrentMemoryContext, which + * may be a BumpContext. Calling pfree on BumpContext memory would ERROR. + * The memory will be automatically reclaimed when the current memory context + * is reset (e.g., at end of query, transaction, or subtransaction). + * + * This function exists to maintain API compatibility and to serve as a + * clear marker in the code where batch data is no longer needed. + */ +void +UndoFreeBatchData(UndoBatchData *batch) +{ + /* Intentionally empty - memory reclaimed by context reset */ + (void) batch; +} diff --git a/src/backend/access/undo/undoapply.c b/src/backend/access/undo/undoapply.c new file mode 100644 index 0000000000000..772ec11431ca2 --- /dev/null +++ b/src/backend/access/undo/undoapply.c @@ -0,0 +1,312 @@ +/*------------------------------------------------------------------------- + * + * undoapply.c + * Generic UNDO record application during transaction rollback + * + * When a transaction aborts, this module walks the UNDO chain backward + * from the most recent record to the first. For each record, it + * dispatches to the appropriate resource manager's rm_undo callback + * based on the urec_rmid field in the record header. + * + * This module is AM-agnostic: it contains no AM- or subsystem-specific + * code. All such UNDO application logic lives in the respective resource + * managers, each registered via RegisterUndoRmgr() (see access/undormgr.h). + * + * The dispatch pattern is analogous to WAL resource managers: each RM + * registers its callbacks via RegisterUndoRmgr(), and this module + * routes UNDO records to the correct handler. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undoapply.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/undo_xlog.h" +#include "miscadmin.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" + +/* + * ApplyOneUndoRecord - Apply a single UNDO record via RM dispatch + * + * UNDO records carry no "already applied" marker, so a record can be applied + * more than once; each resource manager's rm_undo callback must leave alone + * an object that no longer shows the record's effect. + * + * Returns true if successfully applied, false if skipped. + */ +static bool +ApplyOneUndoRecord(UndoRecordHeader *header, char *payload, + UndoRecPtr urec_ptr) +{ + const UndoRmgrData *rmgr; + UndoApplyResult result; + + /* + * Look up the resource manager for this record. + */ + rmgr = GetUndoRmgr(header->urec_rmid); + if (rmgr == NULL) + { + ereport(WARNING, + (errmsg("UNDO rollback: unknown RM ID %u for record at %llu, skipping", + header->urec_rmid, + (unsigned long long) urec_ptr))); + return false; + } + + /* + * Dispatch to the RM's undo-apply callback, which does all of the + * consumer-specific work (see UndoRmgrApplyFunc). + */ + result = rmgr->rm_undo(header->urec_rmid, + header->urec_info, + header->urec_xid, + header->urec_reloid, + payload, + header->urec_payload_len, + urec_ptr); + + if (result == UNDO_APPLY_SUCCESS) + { + ereport(DEBUG2, + (errmsg("UNDO rollback: applied %s record at %llu", + rmgr->rm_name, + (unsigned long long) urec_ptr))); + return true; + } + else if (result == UNDO_APPLY_SKIPPED) + { + ereport(DEBUG2, + (errmsg("UNDO rollback: skipped %s record at %llu", + rmgr->rm_name, + (unsigned long long) urec_ptr))); + return false; + } + else + { + ereport(WARNING, + (errmsg("UNDO rollback: error applying %s record at %llu", + rmgr->rm_name, + (unsigned long long) urec_ptr))); + return false; + } +} + +/* + * ApplyUndoChainFromWAL - Walk and apply an UNDO chain from WAL + * + * Reads XLOG_UNDO_BATCH WAL records via UndoReadBatchFromWAL() and + * iterates through the serialized records within each batch. + * + * The chain is walked backward via the xl_undo_batch.chain_prev LSN + * from the most recent batch to the first. Within each batch, records + * are applied in reverse order (newest to oldest) as required by + * ARIES-style rollback semantics. This is achieved by first scanning + * forward through the serialized records to collect their start offsets, + * then iterating the collected offsets in reverse to apply each record. + */ +bool +ApplyUndoChainFromWAL(XLogRecPtr last_batch_lsn) +{ + return ApplyUndoChainFromWALBounded(last_batch_lsn, InvalidXLogRecPtr); +} + +/* + * ApplyUndoChainFromWALBounded - Apply an UNDO chain, stopping at a boundary + * + * Identical to ApplyUndoChainFromWAL() except that the chain walk halts before + * processing any batch whose LSN is at or below stop_at_lsn. This applies only + * the batches strictly newer than stop_at_lsn, which is what subtransaction + * abort needs: stop_at_lsn is the parent's saved chain head, so the parent's + * (and earlier subtransactions') batches are left intact while the aborting + * subtransaction's batches are reverted. + * + * Pass InvalidXLogRecPtr as stop_at_lsn to walk the entire chain (top-level + * rollback / recovery semantics). + */ +bool +ApplyUndoChainFromWALBounded(XLogRecPtr last_batch_lsn, XLogRecPtr stop_at_lsn) +{ + XLogRecPtr batch_lsn; + int records_applied = 0; + int records_skipped = 0; + int batches_processed = 0; + + if (!XLogRecPtrIsValid(last_batch_lsn)) + return false; + + ereport(DEBUG1, + (errmsg("applying UNDO chain from WAL starting at %X/%08X " + "(stop boundary %X/%08X)", + LSN_FORMAT_ARGS(last_batch_lsn), + LSN_FORMAT_ARGS(stop_at_lsn)))); + + batch_lsn = last_batch_lsn; + + while (XLogRecPtrIsValid(batch_lsn)) + { + UndoBatchData *batch; + char *pos; + char *end; + + /* + * Bounded walk: stop once we reach a batch that belongs to the parent + * (or an earlier subtransaction). chain_prev LSNs decrease strictly + * as we walk backward, so the first batch at or below the boundary + * marks the start of the region we must preserve. + */ + if (XLogRecPtrIsValid(stop_at_lsn) && batch_lsn <= stop_at_lsn) + break; + + INJECTION_POINT("undo-apply-before-batch", NULL); + + batch = UndoReadBatchFromWAL(batch_lsn); + if (batch == NULL) + { + ereport(WARNING, + (errmsg("UNDO rollback: could not read batch at %X/%08X, " + "stopping chain walk", + LSN_FORMAT_ARGS(batch_lsn)))); + break; + } + + batches_processed++; + + /* + * Walk through records within this batch in reverse order. + * + * ARIES requires that UNDO records within a batch be applied + * newest-first (reverse of serialization order). We first scan + * forward to collect pointers to each record start, then iterate the + * collected pointers in reverse to apply them. + */ + pos = batch->payload; + end = pos + batch->payload_len; + + { + char **record_starts; + int nrecords_in_batch = 0; + int max_records = 1024; + int i; + MemoryContext batchctx; + MemoryContext oldctx; + + /* + * record_starts must grow to hold EVERY record in the batch -- + * silently truncating rollback is a data-integrity bug (the + * dropped records are never applied, so a partially-applied + * in-place UPDATE/DELETE is left on the page). We therefore + * repalloc() to double the array whenever it fills. + * + * The caller's CurrentMemoryContext may be a BumpContext + * (executor abort path), which supports neither repalloc() nor + * pfree(). So we do all growth in a private AllocSet child + * context and delete it whole at the end of the batch -- + * context-agnostic, leak-free, and unbounded. + */ + batchctx = AllocSetContextCreate(CurrentMemoryContext, + "UNDO batch record_starts", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(batchctx); + + record_starts = (char **) palloc(max_records * sizeof(char *)); + + /* First pass: collect record start pointers by scanning forward */ + while (pos < end) + { + UndoRecordHeader hdr; + + if ((Size) (end - pos) < SizeOfUndoRecordHeader) + { + ereport(WARNING, + (errmsg("UNDO rollback: truncated record in batch at %X/%08X", + LSN_FORMAT_ARGS(batch_lsn)))); + break; + } + + memcpy(&hdr, pos, SizeOfUndoRecordHeader); + + if (hdr.urec_len < SizeOfUndoRecordHeader || + (Size) (end - pos) < hdr.urec_len || + hdr.urec_payload_len > hdr.urec_len - SizeOfUndoRecordHeader) + { + ereport(WARNING, + (errmsg("UNDO rollback: invalid record size %u in batch at %X/%08X", + hdr.urec_len, LSN_FORMAT_ARGS(batch_lsn)))); + break; + } + + /* + * Grow the array (double it) rather than truncate the + * rollback + */ + if (nrecords_in_batch >= max_records) + { + max_records *= 2; + record_starts = (char **) repalloc(record_starts, + max_records * sizeof(char *)); + } + + record_starts[nrecords_in_batch++] = pos; + pos += hdr.urec_len; + } + + /* + * Second pass: apply records in reverse order (newest first). + * Even if there was a scan error, apply whatever records we + * successfully collected. + */ + for (i = nrecords_in_batch - 1; i >= 0; i--) + { + UndoRecordHeader header; + char *payload = NULL; + + memcpy(&header, record_starts[i], SizeOfUndoRecordHeader); + + if (header.urec_payload_len > 0) + payload = record_starts[i] + SizeOfUndoRecordHeader; + + if (ApplyOneUndoRecord(&header, payload, InvalidUndoRecPtr)) + records_applied++; + else + records_skipped++; + } + + MemoryContextSwitchTo(oldctx); + MemoryContextDelete(batchctx); + } + + INJECTION_POINT("undo-apply-after-batch", NULL); + + /* Follow chain to previous batch */ + batch_lsn = batch->header.chain_prev; + UndoFreeBatchData(batch); + } + + /* Report results */ + if (records_skipped > 0) + { + ereport(WARNING, + (errmsg("UNDO rollback from WAL: %d batches, %d records applied, " + "%d skipped", + batches_processed, records_applied, records_skipped))); + } + else + { + ereport(DEBUG1, + (errmsg("UNDO rollback from WAL complete: %d batches, " + "%d records applied", + batches_processed, records_applied))); + } + + return (batches_processed > 0); +} diff --git a/src/backend/access/undo/undoinsert.c b/src/backend/access/undo/undoinsert.c new file mode 100644 index 0000000000000..1c7a728a463a7 --- /dev/null +++ b/src/backend/access/undo/undoinsert.c @@ -0,0 +1,127 @@ +/*------------------------------------------------------------------------- + * + * undoinsert.c + * UNDO record batch insertion operations + * + * This file implements batch insertion of UNDO records into the WAL + * stream. Records are accumulated in an UndoRecordSet and then + * written as a single XLOG_UNDO_BATCH WAL record. + * + * UNDO-IN-WAL ARCHITECTURE + * ------------------------ + * All UNDO record data flows through the standard WAL pipeline: + * UndoRecordSetInsert() -> XLogBeginInsert() + * -> XLogRegisterData(batch_header) + * -> XLogRegisterData(uset->buffer, uset->buffer_size) + * -> XLogInsert(RM_UNDO_ID, XLOG_UNDO_BATCH) + * + * Coalescing: The existing UndoRecordSet mechanism batches records. + * This batch becomes one WAL record. A 1000-row INSERT produces ~1 + * WAL record containing 1000 UNDO records. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undoinsert.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undo_xlog.h" +#include "access/xloginsert.h" +#include "access/xlog.h" +#include "utils/injection_point.h" + +/* + * UndoRecordSetInsert - Insert accumulated UNDO records into WAL + * + * This function writes all UNDO records in the set as a single + * XLOG_UNDO_BATCH WAL record. The batch payload is the serialized + * content of uset->buffer (concatenated UndoRecordHeader+payload). + * + * Returns the (legacy) UndoRecPtr for backward compatibility. + * The actual record location is the XLogRecPtr stored in + * uset->last_batch_lsn after this call. + */ +UndoRecPtr +UndoRecordSetInsert(UndoRecordSet *uset) +{ + xl_undo_batch xlrec; + XLogRecPtr batch_lsn; + Oid primary_reloid = InvalidOid; + + if (uset == NULL || uset->nrecords == 0) + return InvalidUndoRecPtr; + + /* + * Extract the primary relation OID from the first record in the batch as + * an optimization hint. Most batches contain records for a single + * relation. + */ + if (uset->buffer_size >= SizeOfUndoRecordHeader) + { + UndoRecordHeader *first_hdr = (UndoRecordHeader *) uset->buffer; + + primary_reloid = first_hdr->urec_reloid; + } + + /* Build the batch header */ + xlrec.xid = uset->xid; + xlrec.chain_prev = uset->last_batch_lsn; + xlrec.nrecords = (uint32) uset->nrecords; + xlrec.total_len = (uint32) uset->buffer_size; + xlrec.primary_reloid = primary_reloid; + xlrec.persistence = uset->persistence; + + /* + * Write the UNDO batch as a single WAL record. + * + * XLogRegisterData has no size limit on main data (tracked as uint64 in + * xloginsert.c), so even a 256KB batch is fine. The WAL insertion lock + * will be held for the duration of the record write, which is acceptable + * for batch sizes up to a few hundred KB. + */ + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfUndoBatch); + XLogRegisterData(uset->buffer, uset->buffer_size); + + /* + * Use the CACHED variant: UndoRecordSetInsert() always runs inside the + * caller's critical section (it emits a WAL record), and + * InjectionPointRun() may palloc (dlopen the callback library, or a + * wait-mode callback allocating to suspend the backend), which is + * forbidden in a critical section. The matching INJECTION_POINT_LOAD is + * issued pre-crit in PrepareXactUndoData(); INJECTION_POINT_CACHED is + * palloc-free. + */ + INJECTION_POINT_CACHED("undo-batch-before-wal-insert", NULL); + + (void) XLogInsert(RM_UNDO_ID, XLOG_UNDO_BATCH); + + /* + * XLogInsert() returns the end+1 position of the record, but the rollback + * path must re-read the batch by its START LSN. ProcLastRecPtr holds the + * start of the record we just inserted; use it so UndoReadBatchFromWAL + * lands on this XLOG_UNDO_BATCH instead of the following record. + */ + batch_lsn = ProcLastRecPtr; + + INJECTION_POINT_CACHED("undo-batch-after-wal-insert", NULL); + + /* Update the record set's chain pointer for subsequent batches */ + uset->last_batch_lsn = batch_lsn; + + /* + * Register the batch LSN for WAL retention tracking. Only the first call + * per transaction takes effect (UndoRegisterBatchLSN is a no-op if the + * slot is already occupied), so this records the oldest batch for this + * transaction without additional bookkeeping. + */ + UndoRegisterBatchLSN(batch_lsn); + + return (UndoRecPtr) batch_lsn; +} diff --git a/src/backend/access/undo/undolog.c b/src/backend/access/undo/undolog.c new file mode 100644 index 0000000000000..d1af41a6ae6cb --- /dev/null +++ b/src/backend/access/undo/undolog.c @@ -0,0 +1,153 @@ +/*------------------------------------------------------------------------- + * + * undolog.c + * WAL retention for UNDO records + * + * See access/undolog.h. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undolog.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/twophase.h" +#include "access/undolog.h" +#include "miscadmin.h" +#include "storage/procnumber.h" +#include "storage/shmem.h" + +pg_atomic_uint64 *UndoBatchLSNs = NULL; + +/* + * UndoLogShmemRequest + * Request the per-backend slots from the shared memory framework. + * + * Called from the UNDO subsystem's request_fn. The framework sets + * UndoBatchLSNs, in the postmaster and, under EXEC_BACKEND, in every child. + */ +void +UndoLogShmemRequest(void) +{ + ShmemRequestStruct(.name = "UNDO Log Control", + .size = mul_size(MaxBackends, sizeof(pg_atomic_uint64)), + .ptr = (void **) &UndoBatchLSNs); +} + +/* + * UndoLogShmemInit + * Initialize the slots requested by UndoLogShmemRequest(). + */ +void +UndoLogShmemInit(void) +{ + for (int i = 0; i < MaxBackends; i++) + pg_atomic_init_u64(&UndoBatchLSNs[i], + InvalidXLogRecPtr); +} + +/* + * UndoRegisterBatchLSN + * Register the first UNDO batch LSN for the current backend. + * + * Called from UndoRecordSetInsert() each time a transaction writes an UNDO + * batch. Stores the LSN in the backend's slot, which KeepLogSeg() reads + * through UndoGetOldestBatchLSN(). + * + * Only the FIRST call per transaction takes effect (we want the oldest, + * i.e., smallest, LSN). Subsequent calls for the same transaction are + * no-ops because the slot is already occupied. + */ +void +UndoRegisterBatchLSN(XLogRecPtr batch_lsn) +{ + pg_atomic_uint64 *slot; + uint64 expected; + + if (UndoBatchLSNs == NULL || !XLogRecPtrIsValid(batch_lsn)) + return; + if (MyProcNumber < 0 || MyProcNumber >= MaxBackends) + return; + + slot = &UndoBatchLSNs[MyProcNumber]; + expected = InvalidXLogRecPtr; + + /* + * Only set if the slot is currently empty. This records the first + * (oldest) batch for this transaction; later batches have larger LSNs and + * should not overwrite the stored value. + */ + (void) pg_atomic_compare_exchange_u64(slot, &expected, (uint64) batch_lsn); +} + +/* + * UndoClearBatchLSN + * Clear the per-backend UNDO batch LSN registration. + * + * Called at transaction commit or abort to release the WAL retention + * hold that was established by UndoRegisterBatchLSN(). + */ +void +UndoClearBatchLSN(void) +{ + if (UndoBatchLSNs == NULL) + return; + if (MyProcNumber < 0 || MyProcNumber >= MaxBackends) + return; + + pg_atomic_write_u64(&UndoBatchLSNs[MyProcNumber], + (uint64) InvalidXLogRecPtr); +} + +/* + * UndoGetOldestBatchLSN + * Return the oldest UNDO batch LSN that must be retained in WAL. + * + * Considers both: + * 1. Per-backend slots (in-flight transactions with UNDO data) + * 2. Prepared transactions, whose chain ROLLBACK PREPARED still reads + * + * Returns InvalidXLogRecPtr if no WAL retention is needed for UNDO. + */ +XLogRecPtr +UndoGetOldestBatchLSN(void) +{ + XLogRecPtr oldest = InvalidXLogRecPtr; + int i; + + if (UndoBatchLSNs == NULL) + return InvalidXLogRecPtr; + + /* Check per-backend slots for in-flight transactions */ + for (i = 0; i < MaxBackends; i++) + { + XLogRecPtr lsn = (XLogRecPtr) + pg_atomic_read_u64(&UndoBatchLSNs[i]); + + if (XLogRecPtrIsValid(lsn)) + { + if (!XLogRecPtrIsValid(oldest) || lsn < oldest) + oldest = lsn; + } + } + + /* + * Check prepared (2PC) transactions. A xact can sit PREPARED + * indefinitely; its UNDO-batch WAL must survive until ROLLBACK PREPARED + * reads it. The per-backend slot is cleared when the preparing backend + * exits, so it does not cover this. + */ + { + XLogRecPtr prep_oldest = TwoPhaseGetOldestUndoBatchLSN(); + + if (XLogRecPtrIsValid(prep_oldest) && + (!XLogRecPtrIsValid(oldest) || prep_oldest < oldest)) + oldest = prep_oldest; + } + + return oldest; +} diff --git a/src/backend/access/undo/undorecord.c b/src/backend/access/undo/undorecord.c new file mode 100644 index 0000000000000..6cde66ce2899b --- /dev/null +++ b/src/backend/access/undo/undorecord.c @@ -0,0 +1,360 @@ +/*------------------------------------------------------------------------- + * + * undorecord.c + * UNDO record assembly and serialization + * + * This file implements the AM-agnostic UNDO record format and provides + * functions for creating, serializing, and deserializing UNDO records. + * All AM-specific knowledge is kept out of this module; records carry + * opaque payloads whose interpretation is delegated to the owning RM. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undorecord.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undorecord.h" +#include "utils/memutils.h" + +/* + * Per-backend recycled memory context for UndoRecordSet. + * + * Instead of creating and destroying a MemoryContext for every + * UndoRecordSet, we recycle one context across operations within a + * transaction. This avoids the overhead of repeated + * AllocSetContextCreate/MemoryContextDelete for high-frequency + * operations (e.g., 1000-row INSERTs). The cached context is cleaned + * up at transaction end by UndoRecordSetResetCache(). + */ +static MemoryContext UndoRecordReusableContext = NULL; + +/* + * UndoRecordGetPayloadSize - Calculate size needed for an UNDO record + * + * This includes the fixed header plus the RM-specific payload, rounded up to + * MAXALIGN so that each record in a packed stream begins at a MAXALIGN'd + * offset. The writers build the header in place through an + * (UndoRecordHeader *) cast into the packed buffer; without this rounding a record whose length is + * not a multiple of MAXIMUM_ALIGNOF would leave the next record's header at a + * mis-aligned address, which is undefined behavior (and traps under + * -fsanitize=alignment / crashes on alignment-strict hardware). The padded + * size is stored in urec_len, so readers advance by the same stride; the true + * payload length travels separately in urec_payload_len, so the padding never + * affects payload extraction. + */ +Size +UndoRecordGetPayloadSize(Size payload_len) +{ + return MAXALIGN(SizeOfUndoRecordHeader + payload_len); +} + +/* + * UndoRecordSetCreate - Create a new UNDO record set + * + * A record set accumulates multiple UNDO records before writing them + * to the UNDO log in a batch. This improves performance by reducing + * I/O operations. + */ +UndoRecordSet * +UndoRecordSetCreate(TransactionId xid) +{ + UndoRecordSet *uset; + MemoryContext mctx; + MemoryContext parent; + + /* + * Use TopTransactionContext as the parent so the record set survives + * across SPI statement boundaries. When called from PL/pgSQL DO blocks, + * CurrentMemoryContext is the executor's per-query context + * (es_query_cxt), which is destroyed in FreeExecutorState() after each + * SPI_execute call. Since xactundo.c stores the uset pointer in the + * static XactUndo.record_set[] and reuses it across multiple statements + * within a transaction, the context must outlive any single query. + * TopTransactionContext is ideal: it survives until transaction + * commit/abort, and AtAbort cleanup will free the uset via + * UndoRecordSetFree(). + */ + parent = TopTransactionContext; + + /* + * Reuse a previously recycled memory context if available. This avoids + * the overhead of AllocSetContextCreate/MemoryContextDelete for every + * UndoRecordSet within a transaction. MemoryContextReset clears all + * allocations but keeps the context's memory blocks for reuse. + */ + if (UndoRecordReusableContext != NULL) + { + mctx = UndoRecordReusableContext; + UndoRecordReusableContext = NULL; /* take ownership */ + MemoryContextReset(mctx); + MemoryContextSetParent(mctx, parent); + } + else + { + mctx = AllocSetContextCreate(parent, + "UNDO record set", + ALLOCSET_DEFAULT_SIZES); + } + + /* + * Allocate everything in the uset's memory context using direct + * MemoryContextAlloc to avoid MemoryContextSwitchTo overhead. + */ + uset = (UndoRecordSet *) MemoryContextAllocZero(mctx, sizeof(UndoRecordSet)); + uset->xid = xid; + uset->persistence = UNDOPERSISTENCE_PERMANENT; + + /* + * Allocate initial buffer. 512 bytes holds a typical UNDO record; the + * buffer grows as needed via UndoRecordEnsureCapacity. + */ + uset->buffer_capacity = 512; + uset->buffer = (char *) MemoryContextAlloc(mctx, uset->buffer_capacity); + uset->buffer_size = 0; + + uset->last_batch_lsn = InvalidXLogRecPtr; + uset->mctx = mctx; + + return uset; +} + +/* + * UndoRecordSetFree - Free an UNDO record set + * + * Recycles the memory context for later reuse if possible, otherwise + * destroys it. We keep at most one recycled context to bound memory. + */ +void +UndoRecordSetFree(UndoRecordSet *uset) +{ + MemoryContext mctx; + + if (uset == NULL || uset->mctx == NULL) + return; + + mctx = uset->mctx; + + if (UndoRecordReusableContext == NULL) + { + /* + * Recycle this context for the next UndoRecordSetCreate call. + * + * Re-parent to TopMemoryContext so the cached context is not + * destroyed if its original parent is cleaned up before + * UndoRecordSetResetCache() runs. This can happen when the UNDO + * record set was created inside an SPI execution context (e.g., DO $$ + * ... $$ blocks): SPI_finish() deletes its procCxt/execCxt, which + * would recursively destroy this child context, leaving + * UndoRecordReusableContext as a dangling pointer. + * UndoRecordSetCreate() will re-parent it to the caller's + * CurrentMemoryContext on reuse. + */ + MemoryContextSetParent(mctx, TopMemoryContext); + UndoRecordReusableContext = mctx; + } + else + { + /* Already have one recycled context; destroy this one */ + MemoryContextDelete(mctx); + } +} + +/* + * UndoRecordEnsureCapacity - Ensure the uset buffer can hold additional bytes + * + * Grows the buffer (using the uset's memory context) if needed. + * Avoids MemoryContextSwitchTo overhead by using MemoryContextAlloc directly. + */ +static void +UndoRecordEnsureCapacity(UndoRecordSet *uset, Size additional) +{ + if (uset->buffer_size + additional > uset->buffer_capacity) + { + Size new_capacity = uset->buffer_capacity * 2; + char *newbuf; + + while (new_capacity < uset->buffer_size + additional) + new_capacity *= 2; + + newbuf = (char *) MemoryContextAlloc(uset->mctx, new_capacity); + if (uset->buffer_size > 0) + memcpy(newbuf, uset->buffer, uset->buffer_size); + pfree(uset->buffer); + uset->buffer = newbuf; + uset->buffer_capacity = new_capacity; + } +} + +/* + * UndoRecordSetReset - Reset a record set for reuse + * + * Resets the buffer position and record count without freeing the memory + * context or reallocating the buffer. This is much cheaper than + * UndoRecordSetCreate/Free (~5 cycles vs ~300 cycles) because it avoids + * MemoryContextReset/AllocSetContextCreate overhead entirely. + */ +void +UndoRecordSetReset(UndoRecordSet *uset) +{ + if (uset == NULL) + return; + + uset->buffer_size = 0; + uset->nrecords = 0; +} + +/* + * UndoRecordSetResetCache - Release the recycled memory context. + * + * Called at transaction end (commit or abort) to ensure the cached + * context does not outlive the transaction. + */ +void +UndoRecordSetResetCache(void) +{ + if (UndoRecordReusableContext != NULL) + { + MemoryContextDelete(UndoRecordReusableContext); + UndoRecordReusableContext = NULL; + } +} + +/* + * UndoRecordAddPayload - Add an UNDO record with opaque payload to the set + * + * This is the main API for adding UNDO records. The caller provides an + * RM ID, RM-specific info flags, a relation OID, and an opaque payload. + * The payload's interpretation is entirely RM-specific. + */ +void +UndoRecordAddPayload(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *payload, + Size payload_len) +{ + UndoRecordHeader *header; + Size record_size; + char *dest; + + if (uset == NULL) + elog(ERROR, "cannot add UNDO record to NULL set"); + + record_size = UndoRecordGetPayloadSize(payload_len); + + /* Expand buffer if needed (allocate in the uset's memory context) */ + UndoRecordEnsureCapacity(uset, record_size); + + /* + * Build the header directly in the buffer, avoiding a separate stack + * variable, memset, and memcpy. We zero the header in-place to avoid + * uninitialized padding bytes in the on-disk format. + */ + dest = uset->buffer + uset->buffer_size; + header = (UndoRecordHeader *) dest; + memset(header, 0, SizeOfUndoRecordHeader); + header->urec_rmid = rmid; + header->urec_flags = UNDO_INFO_XID_VALID; + if (payload_len > 0) + header->urec_flags |= UNDO_INFO_HAS_PAYLOAD; + header->urec_info = info; + header->urec_len = (uint32) record_size; + header->urec_xid = uset->xid; + header->urec_reloid = reloid; + header->urec_payload_len = (uint32) payload_len; + + /* Copy payload directly after header */ + if (payload_len > 0 && payload != NULL) + memcpy(dest + SizeOfUndoRecordHeader, payload, payload_len); + + /* + * Zero the alignment padding between the payload and the next record so + * the serialized stream is deterministic (the padding is skipped by + * readers, which advance by urec_len and read urec_payload_len bytes of + * payload, but deterministic bytes keep WAL consistency checking happy). + */ + if (record_size > SizeOfUndoRecordHeader + payload_len) + memset(dest + SizeOfUndoRecordHeader + payload_len, 0, + record_size - SizeOfUndoRecordHeader - payload_len); + + uset->buffer_size += record_size; + uset->nrecords++; +} + +/* + * UndoRecordAddPayloadParts - Add an UNDO record with scatter-gather payload + * + * Like UndoRecordAddPayload, but takes the payload as two parts that are + * concatenated directly into the uset buffer. This avoids allocating an + * intermediate payload buffer when the caller has the data in separate + * pieces (e.g., a fixed header struct + variable-length tuple data). + */ +void +UndoRecordAddPayloadParts(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *part1, + Size part1_len, + const char *part2, + Size part2_len) +{ + UndoRecordHeader *header; + Size payload_len = part1_len + part2_len; + Size record_size; + char *dest; + + if (uset == NULL) + elog(ERROR, "cannot add UNDO record to NULL set"); + + record_size = UndoRecordGetPayloadSize(payload_len); + + UndoRecordEnsureCapacity(uset, record_size); + + /* Build header directly in the buffer */ + dest = uset->buffer + uset->buffer_size; + header = (UndoRecordHeader *) dest; + memset(header, 0, SizeOfUndoRecordHeader); + header->urec_rmid = rmid; + header->urec_flags = UNDO_INFO_XID_VALID; + if (payload_len > 0) + header->urec_flags |= UNDO_INFO_HAS_PAYLOAD; + header->urec_info = info; + header->urec_len = (uint32) record_size; + header->urec_xid = uset->xid; + header->urec_reloid = reloid; + header->urec_payload_len = (uint32) payload_len; + + /* Copy payload parts directly after header */ + dest += SizeOfUndoRecordHeader; + if (part1_len > 0 && part1 != NULL) + { + memcpy(dest, part1, part1_len); + dest += part1_len; + } + if (part2_len > 0 && part2 != NULL) + { + memcpy(dest, part2, part2_len); + dest += part2_len; + } + + /* + * Zero the alignment padding between the payload and the next record (see + * UndoRecordAddPayload); dest now points just past the payload. + */ + { + char *rec_start = uset->buffer + uset->buffer_size; + + if ((Size) (dest - rec_start) < record_size) + memset(dest, 0, record_size - (Size) (dest - rec_start)); + } + + uset->buffer_size += record_size; + uset->nrecords++; +} diff --git a/src/backend/access/undo/undormgr.c b/src/backend/access/undo/undormgr.c new file mode 100644 index 0000000000000..851ef61ae8d4f --- /dev/null +++ b/src/backend/access/undo/undormgr.c @@ -0,0 +1,70 @@ +/*------------------------------------------------------------------------- + * + * undormgr.c + * UNDO resource manager registration and dispatch + * + * This module manages the registration table for UNDO resource managers. + * Each access method or subsystem that writes UNDO records registers + * its callbacks here. The generic UNDO infrastructure dispatches to + * the appropriate callback based on the urec_rmid in the record header. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undormgr.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undormgr.h" + +/* Global registration table, indexed by RM ID */ +const UndoRmgrData *UndoRmgrs[MAX_UNDO_RMGRS]; + +/* + * RegisterUndoRmgr - Register an UNDO resource manager + * + * Called by each AM/subsystem during initialization to register its + * UNDO apply and describe callbacks. + */ +void +RegisterUndoRmgr(uint8 rmid, const UndoRmgrData *rmgr) +{ + if (rmid == UNDO_RMID_INVALID) + elog(ERROR, "cannot register UNDO RM with invalid ID 0"); + + if (UndoRmgrs[rmid] != NULL) + elog(ERROR, "UNDO RM ID %u already registered as \"%s\"", + rmid, UndoRmgrs[rmid]->rm_name); + + if (rmgr->rm_undo == NULL) + elog(ERROR, "UNDO RM \"%s\" must provide an rm_undo callback", + rmgr->rm_name ? rmgr->rm_name : "(null)"); + + UndoRmgrs[rmid] = rmgr; +} + +/* + * GetUndoRmgr - Look up an UNDO resource manager by ID + * + * Returns the registration entry, or NULL if not registered. + */ +const UndoRmgrData * +GetUndoRmgr(uint8 rmid) +{ + return UndoRmgrs[rmid]; +} + +/* + * InitUndoRmgrs - Initialize the UNDO resource manager table + * + * Called during postmaster startup. Individual RMs register themselves + * via RegisterUndoRmgr() during their initialization. + */ +void +InitUndoRmgrs(void) +{ + MemSet(UndoRmgrs, 0, sizeof(UndoRmgrs)); +} diff --git a/src/backend/access/undo/xactundo.c b/src/backend/access/undo/xactundo.c new file mode 100644 index 0000000000000..3c7e22b499b15 --- /dev/null +++ b/src/backend/access/undo/xactundo.c @@ -0,0 +1,1149 @@ +/*------------------------------------------------------------------------- + * + * xactundo.c + * Management of undo record sets for transactions + * + * Undo records that need to be applied after a transaction or + * subtransaction abort should be inserted using the functions defined + * in this file; every consumer that wants undo for post-abort cleanup should + * use these interfaces. + * + * The reason for this design is that we want to pack all of the undo + * records for a single transaction into one place, regardless of the + * AM which generated them. That way, we can apply the undo actions + * which pertain to that transaction in the correct order; namely, + * backwards as compared with the order in which the records were + * generated. + * + * We may use up to three undo record sets per transaction, one per + * persistence level (permanent, unlogged, temporary). We assume that + * it's OK to apply the undo records for each persistence level + * independently of the others. This is safe since the modifications + * must necessarily touch disjoint sets of pages. + * + * All of a transaction's UNDO records at one persistence level go into the + * same UndoRecordSet in the order they were written, whatever objects they + * describe, and rollback applies them newest first. + * + * SUBTRANSACTION TRACKING: + * + * Subtransaction state is tracked using a dynamically-grown array allocated + * in TopMemoryContext. The array starts at INITIAL_SUBXACT_CAPACITY (64) + * slots and doubles when needed via repalloc(). The array persists across + * transactions within the same backend to avoid repeated allocation for + * steady-state workloads. It lives in TopMemoryContext rather than + * CurTransactionContext so that growing it in a SubXactCallback allocates + * nothing per subtransaction. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/backend/access/undo/xactundo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undo.h" +#include "access/undo_xlog.h" +#include "access/xlog.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/xact.h" +#include "access/xactundo.h" +#include "access/xlogdefs.h" +#include "access/xloginsert.h" +#include "access/table.h" +#include "catalog/pg_class.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/ipc.h" +#include "utils/injection_point.h" +#include "storage/lmgr.h" +#include "storage/proc.h" +#include "storage/shmem.h" +#include "storage/spin.h" +#include "utils/memutils.h" +#include "utils/rel.h" + +/* + * Initial capacity for the dynamically-grown subtransaction stack. + * Covers 99.9% of workloads without needing reallocation. The stack + * doubles when needed, so there is no artificial upper limit. + */ +#define INITIAL_SUBXACT_CAPACITY 64 + +/* Per-subtransaction backend-private undo state (array element). */ +typedef struct XactUndoSubTransactionState +{ + SubTransactionId nestingLevel; + UndoRecPtr start_location[NUndoPersistenceLevels]; + + /* + * Snapshot of the parent's last_batch_lsn at the time this subtransaction + * started. On subtransaction abort, after applying this subtransaction's + * UNDO chain, we restore XactUndo.last_batch_lsn to these saved values so + * the parent's subsequent abort (or further subtransactions) only covers + * the parent's own records and does not double-apply already-reversed + * batches. + */ + XLogRecPtr last_batch_lsn[NUndoPersistenceLevels]; +} XactUndoSubTransactionState; + +/* Backend-private undo state. */ +typedef struct XactUndoData +{ + bool has_undo; /* has this xact generated any undo? */ + int subxact_depth; /* 0 = top-level, 1+ = savepoints */ + int subxact_capacity; /* allocated slots in subxact_stack */ + + /* Dynamically-grown subtransaction stack (TopMemoryContext). */ + XactUndoSubTransactionState *subxact_stack; + + /* + * Per-persistence-level record sets. These are created lazily on first + * use and destroyed at transaction end. + */ + UndoRecordSet *record_set[NUndoPersistenceLevels]; + + /* Tracking for the most recent undo insertion per persistence level. */ + UndoRecPtr last_location[NUndoPersistenceLevels]; + + /* + * WAL-based UNDO chain heads. When UNDO records are routed through WAL + * via XLOG_UNDO_BATCH, this tracks the LSN of the most recent batch per + * persistence level. Used for rollback chain walking. + */ + XLogRecPtr last_batch_lsn[NUndoPersistenceLevels]; +} XactUndoData; + +static XactUndoData XactUndo; +static bool subxact_callback_registered = false; + +/* + * Per-backend copy of the current transaction's UNDO chain heads, for the + * checkpointer (see LogXactUndoInflight). Only the owning backend writes its + * slot, under the slot's spinlock; xid is invalid when it has none. + */ +typedef struct XactUndoInflightSlot +{ + slock_t mutex; + xl_undo_inflight_xact chain; +} XactUndoInflightSlot; + +static XactUndoInflightSlot *XactUndoInflight = NULL; + +static void PublishXactUndoChain(TransactionId xid); + +/* + * Compile-time guard: xl_xact_prepare.last_batch_lsn[3] must match + * NUndoPersistenceLevels. Both headers are available here; xact.h avoids + * including undodefs.h to keep its include footprint minimal. + */ +StaticAssertDecl(NUndoPersistenceLevels == 3, + "xl_xact_prepare.last_batch_lsn array size (3) must match NUndoPersistenceLevels"); + +static void ResetXactUndo(void); +static void CollapseXactUndoSubTransactions(void); +static UndoPersistenceLevel GetUndoPersistenceLevel(char relpersistence); +static void EnsureSubxactStackCapacity(void); +static void XactUndo_SubXactCallback(SubXactEvent event, SubTransactionId mySubid, + SubTransactionId parentSubid, void *arg); + +/* Convenience macro: pointer to current subtransaction state. */ +#define CURRENT_SUBXACT() (&XactUndo.subxact_stack[XactUndo.subxact_depth]) + +/* + * XactUndoShmemSize + * How much shared memory do we need for transaction undo state? + * + * One XactUndoInflightSlot per backend. + */ +Size +XactUndoShmemSize(void) +{ + return mul_size(MaxBackends, sizeof(XactUndoInflightSlot)); +} + +/* + * XactUndoShmemRequest + * Register the in-flight chain slots with the shared memory framework. + */ +void +XactUndoShmemRequest(void) +{ + ShmemRequestStruct(.name = "UNDO In-flight Chains", + .size = XactUndoShmemSize(), + .ptr = (void **) &XactUndoInflight); +} + +/* + * XactUndoShmemInit + * Initialize shared memory for transaction undo state. + */ +void +XactUndoShmemInit(void) +{ + for (int i = 0; i < MaxBackends; i++) + { + SpinLockInit(&XactUndoInflight[i].mutex); + XactUndoInflight[i].chain.xid = InvalidTransactionId; + } +} + +/* + * PublishXactUndoChain + * Make this backend's chain heads visible to LogXactUndoInflight(). + * + * xid invalid clears the slot. Called in critical sections: no palloc. + */ +static void +PublishXactUndoChain(TransactionId xid) +{ + XactUndoInflightSlot *slot; + + if (XactUndoInflight == NULL || MyProcNumber < 0 || + MyProcNumber >= MaxBackends) + return; + slot = &XactUndoInflight[MyProcNumber]; + if (!TransactionIdIsValid(xid) && !TransactionIdIsValid(slot->chain.xid)) + return; + + SpinLockAcquire(&slot->mutex); + slot->chain.xid = xid; + for (int i = 0; i < NUndoPersistenceLevels; i++) + slot->chain.last_batch_lsn[i] = XactUndo.last_batch_lsn[i]; + SpinLockRelease(&slot->mutex); +} + +/* + * LogXactUndoInflight + * WAL-log the UNDO chains of every transaction in flight. + * + * Called by the checkpointer after the checkpoint's redo pointer has been + * determined. Redo starts at that pointer and so does not see the batches a + * transaction wrote before it; this record tells recovery about them. A + * transaction that commits or aborts after we read its slot is still listed; + * its commit or abort record, which follows this one, removes the entry + * again during redo. + */ +void +LogXactUndoInflight(void) +{ + xl_undo_inflight *rec; + uint32 n = 0; + + if (XactUndoInflight == NULL) + return; + + rec = palloc(SizeOfUndoInflight + + MaxBackends * sizeof(xl_undo_inflight_xact)); + for (int i = 0; i < MaxBackends; i++) + { + XactUndoInflightSlot *slot = &XactUndoInflight[i]; + + SpinLockAcquire(&slot->mutex); + if (TransactionIdIsValid(slot->chain.xid)) + rec->xacts[n++] = slot->chain; + SpinLockRelease(&slot->mutex); + } + + /* Unimportant, so that an idle system still skips checkpoints. */ + if (n > 0) + { + rec->nxacts = n; + XLogBeginInsert(); + XLogSetRecordFlags(XLOG_MARK_UNIMPORTANT); + XLogRegisterData(rec, SizeOfUndoInflight + + n * sizeof(xl_undo_inflight_xact)); + (void) XLogInsert(RM_UNDO_ID, XLOG_UNDO_INFLIGHT); + } + pfree(rec); +} + +/* + * InitializeXactUndo + * Per-backend initialization for transaction undo. + */ +void +InitializeXactUndo(void) +{ + /* Ensure the dynamic subxact stack is allocated */ + EnsureSubxactStackCapacity(); + + ResetXactUndo(); + + /* + * Register callback to track subtransaction lifecycle. Do this lazily on + * first transaction to ensure it's registered for the backend that will + * actually use UNDO. + */ + if (!subxact_callback_registered) + { + RegisterSubXactCallback(XactUndo_SubXactCallback, NULL); + subxact_callback_registered = true; + } +} + +/* + * GetUndoPersistenceLevel + * Map relation persistence character to UndoPersistenceLevel. + */ +static UndoPersistenceLevel +GetUndoPersistenceLevel(char relpersistence) +{ + switch (relpersistence) + { + case RELPERSISTENCE_PERMANENT: + return UNDOPERSISTENCE_PERMANENT; + case RELPERSISTENCE_UNLOGGED: + return UNDOPERSISTENCE_UNLOGGED; + case RELPERSISTENCE_TEMP: + return UNDOPERSISTENCE_TEMP; + default: + elog(ERROR, "unrecognized relpersistence: %c", relpersistence); + return UNDOPERSISTENCE_PERMANENT; /* keep compiler quiet */ + } +} + +/* + * PrepareXactUndoData + * Prepare to insert a transactional undo record. + * + * Finds or creates the appropriate per-persistence-level UndoRecordSet + * for the current transaction and adds the record to it. + * + * The API is AM-agnostic: callers pass an RM ID, RM-specific info, + * a relation OID, and an opaque payload. + * + * Returns the UndoRecPtr where the record will be inserted (or + * InvalidUndoRecPtr if undo is disabled). + */ +UndoRecPtr +PrepareXactUndoData(XactUndoContext *ctx, char persistence, + uint8 rmid, uint16 info, Oid reloid, + const char *payload, Size payload_len) +{ + int nestingLevel = GetCurrentTransactionNestLevel(); + UndoPersistenceLevel plevel = GetUndoPersistenceLevel(persistence); + UndoRecordSet *uset; + XactUndoSubTransactionState *cur; + UndoRecPtr *sub_start_location; + + /* Remember that we've done something undo-related. */ + XactUndo.has_undo = true; + + /* The current (sub)transaction must have an xid; see below. */ + (void) GetCurrentTransactionId(); + + /* + * Load the UndoRecordSetInsert() injection points into this backend's + * local cache now, while we are guaranteed not to be inside a critical + * section (PrepareXactUndoData may itself palloc). UndoRecordSetInsert() + * runs inside the caller's crit section and can only fire these via the + * palloc-free INJECTION_POINT_CACHED variant. + */ + INJECTION_POINT_LOAD("undo-batch-before-wal-insert"); + INJECTION_POINT_LOAD("undo-batch-after-wal-insert"); + + /* + * If we've entered a subtransaction deeper than what's currently tracked, + * push a new entry onto the subxact_stack. This handles the case where + * PrepareXactUndoData is called for the first time in a subtransaction + * that was started before the SubXactCallback fired (e.g., if the + * callback hadn't been registered yet when the subtransaction began). + */ + cur = CURRENT_SUBXACT(); + if (nestingLevel > (int) cur->nestingLevel) + { + int i; + + XactUndo.subxact_depth++; + EnsureSubxactStackCapacity(); + + cur = CURRENT_SUBXACT(); + cur->nestingLevel = nestingLevel; + for (i = 0; i < NUndoPersistenceLevels; ++i) + { + cur->start_location[i] = InvalidUndoRecPtr; + cur->last_batch_lsn[i] = XactUndo.last_batch_lsn[i]; + } + } + + /* + * Make sure we have an UndoRecordSet of the appropriate type open for + * this persistence level. These record sets are always associated with + * the toplevel transaction, not a subtransaction, to avoid fragmentation. + * + * The set, and so every batch it writes, is stamped with the top-level + * xid even when the first record comes from a subtransaction. Crash + * recovery tracks loser chains by that xid and forgets them at the top + * level's commit or abort record; a subxid stamp would outlive the commit + * and get a committed transaction's UNDO applied. + */ + uset = XactUndo.record_set[plevel]; + if (uset == NULL) + { + uset = UndoRecordSetCreate(GetTopTransactionId()); + XactUndo.record_set[plevel] = uset; + } + + /* Remember persistence level for InsertXactUndoData. */ + ctx->plevel = plevel; + ctx->uset = uset; + + /* Add the record to the record set using generic payload API. */ + UndoRecordAddPayload(uset, rmid, info, reloid, payload, payload_len); + + /* + * If this is the first undo for this persistence level in this + * subtransaction, record the start location. The actual UndoRecPtr is not + * known until insertion, so we use a sentinel for now and the caller will + * update it after InsertXactUndoData. + */ + sub_start_location = &cur->start_location[plevel]; + if (!UndoRecPtrIsValid(*sub_start_location)) + *sub_start_location = (UndoRecPtr) 1; /* will be set properly */ + + return InvalidUndoRecPtr; /* actual ptr assigned during insert */ +} + +/* + * InsertXactUndoData + * Insert the prepared undo data into the undo log. + * + * This performs the actual write of the accumulated records. + */ +void +InsertXactUndoData(XactUndoContext *ctx) +{ + UndoRecordSet *uset = ctx->uset; + UndoRecPtr ptr; + int saved_delay = 0; + + Assert(uset != NULL); + + /* + * Hold off a checkpoint's redo-pointer phase while the batch is inserted + * and published for LogXactUndoInflight(): a batch that precedes the redo + * pointer must be visible to the checkpointer when it reads the slots. + */ + if (MyProc != NULL) + { + saved_delay = MyProc->delayChkptFlags; + MyProc->delayChkptFlags |= DELAY_CHKPT_START; + } + + ptr = UndoRecordSetInsert(uset); + if (UndoRecPtrIsValid(ptr)) + { + XactUndoSubTransactionState *cur = CURRENT_SUBXACT(); + + XactUndo.last_location[ctx->plevel] = ptr; + + /* + * Track the WAL LSN of the most recent UNDO batch for this + * persistence level. This is used during rollback to walk the UNDO + * chain backward through WAL. + * + * The pointer must be a valid LSN; the caller must have produced an + * RM_UNDO_ID XLOG_UNDO_BATCH record via UndoRecordSetInsert. We do + * not validate the rmid here because UndoValidateBatchLSN reads from + * the on-disk WAL via read_local_xlog_page, which waits on flush and + * cannot be safely called from the WAL-insertion hot path. The + * rollback path at AtAbort_XactUndo validates before applying. + */ + Assert(XLogRecPtrIsValid(uset->last_batch_lsn)); + XactUndo.last_batch_lsn[ctx->plevel] = uset->last_batch_lsn; + PublishXactUndoChain(uset->xid); + + /* Fix up subtransaction start location if needed */ + if (cur->start_location[ctx->plevel] == (UndoRecPtr) 1) + cur->start_location[ctx->plevel] = ptr; + } + else + { + XactUndoSubTransactionState *cur = CURRENT_SUBXACT(); + + /* + * The batch was empty or elided, so no record pointer exists to fix + * up the sentinel with. Leaving the (UndoRecPtr) 1 sentinel in place + * would make UndoRecPtrIsValid() treat it as a real start location + * and leak it into subxact->parent merges and rollback chain walks; + * reset it so this level records no start location. + */ + if (cur->start_location[ctx->plevel] == (UndoRecPtr) 1) + cur->start_location[ctx->plevel] = InvalidUndoRecPtr; + } + + if (MyProc != NULL) + MyProc->delayChkptFlags = saved_delay; +} + +/* + * CleanupXactUndoInsertion + * Clean up after an undo insertion cycle. + * + * Resets the record set's buffer position and record count so it can + * accumulate more records. Does NOT free the record set -- that + * happens at transaction end (AtCommit_XactUndo / AtAbort_XactUndo). + */ +void +CleanupXactUndoInsertion(XactUndoContext *ctx) +{ + if (ctx->uset != NULL) + UndoRecordSetReset(ctx->uset); +} + +/* + * GetCurrentXactLastBatchLSN + * Get the WAL LSN of the most recent UNDO batch for a persistence level. + * + * Used during transaction abort to start the WAL-based UNDO chain walk. + */ +XLogRecPtr +GetCurrentXactLastBatchLSN(UndoPersistenceLevel plevel) +{ + return XactUndo.last_batch_lsn[plevel]; +} + +/* + * AtCommit_XactUndo + * Post-commit cleanup of the undo state. + * + * On commit, undo records are no longer needed for rollback. + * Free all record sets and reset state. + * + * The records are already in WAL, made durable by the commit record's + * flush, so nothing needs writing here. + * + * NB: This code MUST NOT FAIL, since it is run as a post-commit step. + */ +void +AtCommit_XactUndo(void) +{ + int i; + + + if (!XactUndo.has_undo) + return; + + /* + * Free all per-persistence-level record sets. + * + * We can safely call UndoRecordSetFree() during commit because we're in + * CurTransactionContext, not BumpContext (which is only used during + * abort). The record sets are allocated in CurTransactionContext and will + * be freed when that context is destroyed at transaction end. + */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (XactUndo.record_set[i] != NULL) + { + UndoRecordSetFree(XactUndo.record_set[i]); + XactUndo.record_set[i] = NULL; + } + } + + /* Release WAL retention hold acquired in UndoRecordSetInsert(). */ + UndoClearBatchLSN(); + + ResetXactUndo(); +} + +/* + * AtPrepare_XactUndo + * End-of-PREPARE cleanup of the backend-private undo state. + * + * PREPARE TRANSACTION hands ownership of this transaction's cluster-wide UNDO + * to the 2PC state: the permanent chain-head LSN was durably saved in + * xl_xact_prepare (twophase.c gxact->undo_batch_lsn), its WAL is pinned while + * the xact stays prepared, and ROLLBACK PREPARED applies that LSN + * synchronously via ApplyUndoChainFromWAL(). + * + * The in-memory UndoRecordSets, however, live in TopTransactionContext, which + * PrepareTransaction() destroys (AtCommit_Memory) right after this. We must + * therefore drop the backend-private references to them here, or the dangling + * XactUndo.record_set[] pointers get freed at backend exit (AtProcExit_XactUndo + * -> UndoRecordSetFree on already-freed memory) and crash. We only NULL the + * pointers (via ResetXactUndo); the record sets themselves are freed by + * TopTransactionContext destruction. + * + * Unlike AtCommit_XactUndo, we do NOT call UndoClearBatchLSN(): the WAL + * retention hold must persist for as long as the transaction stays prepared. + */ +void +AtPrepare_XactUndo(void) +{ + /* + * Drop the backend-private record-set references (memory freed with + * TopTransactionContext); keep the WAL retention hold. + */ + ResetXactUndo(); +} + +/* + * AtAbort_XactUndo + * Post-abort cleanup of the undo state. + * + * Apply the transaction's UNDO chain and clean up the record sets. The + * batches are already in WAL, so a crash during the rollback leaves them for + * the recovery undo phase. + */ +void +AtAbort_XactUndo(void) +{ + int i; + bool lsn_safely_held = false; /* true once inline UNDO completed */ + + + /* Always clean up the recycled context; see AtCommit_XactUndo. */ + UndoRecordSetResetCache(); + + if (!XactUndo.has_undo) + return; + + INJECTION_POINT("undo-xact-abort-before-apply", NULL); + + /* Collapse all subtransaction state. */ + CollapseXactUndoSubTransactions(); + + /* + * UNDO application strategy: apply the transaction's cluster-wide UNDO + * chain synchronously in this backend, here at abort. + * + * The BumpContext issue (pfree crashes during abort) is avoided by: - + * Creating a temporary AllocSetContext for inline UNDO application - + * ApplyUndoChainFromWAL already avoids pfree on its allocations - + * Switching back to the abort context afterward + */ + { + XLogRecPtr perm_lsn = + XactUndo.last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; + + if (XLogRecPtrIsValid(perm_lsn)) + { + /* + * The UNDO batch records were inserted into the WAL buffers + * during this transaction but may not yet be flushed to disk. The + * inline apply path reads them back via the local XLog reader, + * which would otherwise busy-wait (pg_usleep) for the walwriter + * to flush past the batch LSN -- adding multi-second latency to + * every rollback. Flush WAL up to the end of the last record this + * backend wrote so the batch is immediately readable. We must not + * flush to GetXLogInsertRecPtr(): under concurrency that global + * insert position can point into the middle of a record another + * backend has reserved but not finished copying, so XLogFlush + * would fail with "xlog flush request is not satisfied". + * XactLastRecEnd is always a valid record boundary and, since + * this backend wrote the UNDO batch, is guaranteed to be at or + * past the batch end. This mirrors the commit-path flush in + * RecordTransactionCommit. + */ + XLogFlush(XactLastRecEnd); + + { + /* + * Apply UNDO inline. Use a dedicated AllocSetContext to + * avoid BumpContext pfree issues. + */ + MemoryContext undo_ctx; + MemoryContext old_ctx; + int saved_trans_state; + bool undo_applied = false; + bool valid_batch; + + undo_ctx = AllocSetContextCreate(TopMemoryContext, + "Inline UNDO Apply", + ALLOCSET_DEFAULT_SIZES); + old_ctx = MemoryContextSwitchTo(undo_ctx); + + /* + * Validate the batch LSN points to an actual UNDO record + * before attempting inline application. Stale LSNs from + * chain_prev tracking anomalies can point to non-UNDO WAL + * records, which would cause "not an UNDO batch" warnings. + */ + valid_batch = UndoValidateBatchLSN(perm_lsn); +#ifdef USE_INJECTION_POINTS + /* Force the invalid-batch early exit for testing. */ + if (IS_INJECTION_POINT_ATTACHED("inline-undo-invalid-batch")) + valid_batch = false; +#endif + if (!valid_batch) + { + elog(DEBUG1, "inline UNDO: last_batch_lsn %X/%08X is not " + "a valid UNDO batch", + LSN_FORMAT_ARGS(perm_lsn)); + undo_applied = false; + MemoryContextSwitchTo(old_ctx); + MemoryContextDelete(undo_ctx); + goto inline_undo_done; + } + + /* + * AbortTransaction() has already advanced the transaction + * state to TRANS_ABORT, but the relcache, locks, and resource + * owner are all still live. UNDO appliers open relations, + * which asserts IsTransactionState(); temporarily present + * TRANS_INPROGRESS for the duration of the inline apply and + * always restore the real state afterward. + */ + saved_trans_state = EnterInlineUndoApplyState(); + PG_TRY(); + { + undo_applied = ApplyUndoChainFromWAL(perm_lsn); + } + PG_CATCH(); + { + /* + * If inline UNDO throws an error, release any content + * locks the failed chain left held before unwinding. + */ + BufferLockReleaseAll(); + LeaveInlineUndoApplyState(saved_trans_state); + FlushErrorState(); + undo_applied = false; + } + PG_END_TRY(); + LeaveInlineUndoApplyState(saved_trans_state); + + MemoryContextSwitchTo(old_ctx); + MemoryContextDelete(undo_ctx); + + inline_undo_done: + if (!undo_applied) + { + /* + * Inline UNDO failed (WAL recycled, wrong record type, or + * chain walk aborted). Retain the UNDO WAL retention + * slot so the WAL segment holding our UNDO data is not + * recycled; the slot is cleared at backend exit. + * + * The error that is aborting us may have been raised + * while this backend held buffer content locks, which + * resource-owner cleanup releases only later. The + * appliers lock pages, possibly those very ones, so + * release the stragglers first. LWLockReleaseAll does + * not cover them. + */ + BufferLockReleaseAll(); + elog(WARNING, "inline UNDO failed for xid %u", + GetCurrentTransactionId()); + } + else + { + lsn_safely_held = true; /* UNDO fully applied, WAL can be + * recycled */ + ereport(DEBUG2, + (errmsg("inline UNDO applied for xid %u", + GetCurrentTransactionId()))); + } + } + } + } + + INJECTION_POINT("undo-xact-abort-after-apply", NULL); + + /* Free all per-persistence-level record sets. */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (XactUndo.record_set[i] != NULL) + { + UndoRecordSetFree(XactUndo.record_set[i]); + XactUndo.record_set[i] = NULL; + } + } + + /* + * Release the WAL retention hold if inline UNDO completed: nothing needs + * the UNDO records any more. + * + * Otherwise the slot keeps the hold, but only until this backend's next + * transaction commits or aborts, or the backend exits; both clear it. + * Nothing re-applies the chain in the meantime. The abort record is + * already written, so crash recovery does not treat the transaction as a + * loser either. + */ + if (lsn_safely_held) + UndoClearBatchLSN(); + else + elog(DEBUG1, "retaining UNDO WAL retention slot (inline UNDO failed)"); + + ResetXactUndo(); +} + +/* + * AtSubCommit_XactUndo + * Subtransaction commit: merge sub undo state into parent. + */ +void +AtSubCommit_XactUndo(int level) +{ + XactUndoSubTransactionState *cur; + XactUndoSubTransactionState *parent; + int i; + + + if (XactUndo.subxact_depth <= 0) + return; + + cur = CURRENT_SUBXACT(); + if ((int) cur->nestingLevel != level) + return; + + parent = &XactUndo.subxact_stack[XactUndo.subxact_depth - 1]; + + /* + * Merge start locations into parent. + * + * Invariant: all UNDO records for this transaction, regardless of nesting + * level, are stored in a single chain per persistence level (one + * UndoRecordSet). start_location tracks the earliest record the + * subtransaction generated. Since records are strictly append-only, the + * parent's start location is always earlier than the subtransaction's if + * it exists. We only update the parent's start when it is not yet set + * (the parent wrote no UNDO before this subtransaction). + */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (UndoRecPtrIsValid(cur->start_location[i]) && + !UndoRecPtrIsValid(parent->start_location[i])) + { + parent->start_location[i] = cur->start_location[i]; + } + } + + XactUndo.subxact_depth--; +} + +/* + * AtSubAbort_XactUndo + * Subtransaction abort: apply undo for this sub-level, clean up. + * + * We apply the batches the subtransaction wrote synchronously, before + * control returns to the caller; the parent's batches are left alone. + */ +void +AtSubAbort_XactUndo(int level) +{ + XactUndoSubTransactionState *cur; + XactUndoSubTransactionState *parent; + int i; + + + if (XactUndo.subxact_depth <= 0) + return; + + cur = CURRENT_SUBXACT(); + if ((int) cur->nestingLevel != level) + return; + + parent = &XactUndo.subxact_stack[XactUndo.subxact_depth - 1]; + + /* + * For each persistence level where this subtransaction wrote UNDO, apply + * the batches newer than the parent's chain head; the parent's records + * are preserved. + */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + UndoRecPtr sub_start = cur->start_location[i]; + + if (!UndoRecPtrIsValid(sub_start)) + continue; + + /* + * Eagerly roll back the cluster-wide UNDO this subtransaction wrote. + * + * If the subtransaction advanced XactUndo.last_batch_lsn[i] beyond + * the value saved at subtransaction start, it wrote UNDO batches that + * must now be reversed. We must apply them here rather than leave + * them to the top-level abort: a subtransaction can abort while its + * parent goes on to COMMIT, in which case the top level never aborts + * and the UNDO would never run, leaking the aborted subtransaction's + * writes as if committed. + * + * Apply only the batches strictly newer than the parent's saved head + * (cur->last_batch_lsn[i]) via the bounded chain walk, so the + * parent's and earlier subtransactions' batches are preserved. + * Afterward restore last_batch_lsn[i] to the parent's value so a + * later parent abort walks only the parent's batches and does not + * double-apply ours. + * + * BumpContext safety: subtransaction abort may run under a + * BumpContext that does not support pfree(), so apply inside a + * dedicated AllocSetContext and present an in-progress transaction + * state for the relation opens, mirroring the inline path in + * AtAbort_XactUndo(). + */ + if (XLogRecPtrIsValid(XactUndo.last_batch_lsn[i]) && + XactUndo.last_batch_lsn[i] != cur->last_batch_lsn[i]) + { + XLogRecPtr sub_head = XactUndo.last_batch_lsn[i]; + XLogRecPtr parent_head = cur->last_batch_lsn[i]; + + /* + * Invariant: the current head must be strictly newer (larger) + * than the parent's saved head. If this fires, a sibling + * subtransaction improperly modified last_batch_lsn after its own + * abort, which would cause us to skip or double-apply batches. + */ + Assert(!XLogRecPtrIsValid(parent_head) || + sub_head > parent_head); + + /* + * Flush WAL so the subtransaction's batches are readable by the + * local XLog reader without busy-waiting on the walwriter. See + * the matching flush in AtAbort_XactUndo() for why XactLastRecEnd + * is the correct, always-valid record boundary to flush to. + */ + XLogFlush(XactLastRecEnd); + + if (UndoValidateBatchLSN(sub_head)) + { + MemoryContext undo_ctx; + MemoryContext old_ctx; + int saved_trans_state; + + undo_ctx = AllocSetContextCreate(TopMemoryContext, + "Subxact UNDO Apply", + ALLOCSET_DEFAULT_SIZES); + old_ctx = MemoryContextSwitchTo(undo_ctx); + + /* + * Release buffer content locks the failed subtransaction may + * still hold -- an error raised mid-split leaves both halves + * locked -- before the appliers lock pages again. See + * AtAbort_XactUndo. + */ + BufferLockReleaseAll(); + saved_trans_state = EnterInlineUndoApplyState(); + PG_TRY(); + { + ApplyUndoChainFromWALBounded(sub_head, parent_head); + } + PG_CATCH(); + { + /* + * Release any content locks the failed subxact UNDO chain + * left held before unwinding. + */ + BufferLockReleaseAll(); + LeaveInlineUndoApplyState(saved_trans_state); + FlushErrorState(); + } + PG_END_TRY(); + LeaveInlineUndoApplyState(saved_trans_state); + + MemoryContextSwitchTo(old_ctx); + MemoryContextDelete(undo_ctx); + } + else + elog(DEBUG1, "subxact UNDO: head %X/%08X is not a valid UNDO " + "batch, skipping eager rollback", + LSN_FORMAT_ARGS(sub_head)); + + XactUndo.last_batch_lsn[i] = parent_head; + PublishXactUndoChain(GetTopTransactionIdIfAny()); + } + + /* + * Reset the last_location to what it was before this subtransaction, + * so that if the parent transaction continues and then aborts, only + * the parent's records are applied (the subtransaction's records have + * already been applied). + */ + if (UndoRecPtrIsValid(parent->start_location[i])) + XactUndo.last_location[i] = parent->start_location[i]; + } + + XactUndo.subxact_depth--; +} + +/* + * AtProcExit_XactUndo + * Process exit cleanup for transaction undo. + */ +void +AtProcExit_XactUndo(void) +{ + int i; + + /* Free any lingering record sets. */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (XactUndo.record_set[i] != NULL) + { + UndoRecordSetFree(XactUndo.record_set[i]); + XactUndo.record_set[i] = NULL; + } + } + + /* Release any WAL retention hold (in case process exits mid-transaction). */ + UndoClearBatchLSN(); + + ResetXactUndo(); +} + +/* + * XactUndo_SubXactCallback + * Subtransaction callback to manage UNDO subtransaction state. + * + * This ensures the UNDO subsystem properly tracks all subtransactions, + * including those created by ROLLBACK TO SAVEPOINT. + * + * The subtransaction state is stored in a dynamically-grown array in + * TopMemoryContext. In the common case (depth < capacity), no allocation + * occurs. Growth via repalloc is safe here because it operates on a + * TopMemoryContext allocation, not a per-subtransaction context. + */ +static void +XactUndo_SubXactCallback(SubXactEvent event, SubTransactionId mySubid, + SubTransactionId parentSubid, void *arg) +{ + int i; + + /* These parameters are mandated by the callback signature. */ + (void) parentSubid; + (void) arg; + + switch (event) + { + case SUBXACT_EVENT_START_SUB: + + + /* + * A new subtransaction is starting. Push an entry onto the + * dynamically-grown stack, extending it if needed. + */ + XactUndo.subxact_depth++; + EnsureSubxactStackCapacity(); + { + XactUndoSubTransactionState *s = CURRENT_SUBXACT(); + + s->nestingLevel = mySubid; + for (i = 0; i < NUndoPersistenceLevels; ++i) + { + s->start_location[i] = InvalidUndoRecPtr; + /* Save parent's last_batch_lsn for restore on abort */ + s->last_batch_lsn[i] = XactUndo.last_batch_lsn[i]; + } + } + break; + + case SUBXACT_EVENT_COMMIT_SUB: + + /* + * Subtransaction is committing. Merge its UNDO state into parent. + */ + AtSubCommit_XactUndo(mySubid); + break; + + case SUBXACT_EVENT_ABORT_SUB: + + /* + * Subtransaction is aborting. Apply UNDO and clean up. + */ + AtSubAbort_XactUndo(mySubid); + break; + + case SUBXACT_EVENT_PRE_COMMIT_SUB: + /* Nothing to do at pre-commit */ + break; + } +} + +/* + * EnsureSubxactStackCapacity + * Ensure the subxact_stack has room for the current depth. + * + * If the stack hasn't been allocated yet, allocate it with the initial + * capacity. If we've exceeded the current capacity, double it. + * The stack lives in TopMemoryContext so it persists across transactions + * within the same backend. + */ +static void +EnsureSubxactStackCapacity(void) +{ + if (XactUndo.subxact_stack == NULL) + { + /* First-time allocation */ + XactUndo.subxact_capacity = INITIAL_SUBXACT_CAPACITY; + XactUndo.subxact_stack = (XactUndoSubTransactionState *) + MemoryContextAllocZero(TopMemoryContext, + XactUndo.subxact_capacity * + sizeof(XactUndoSubTransactionState)); + } + else if (XactUndo.subxact_depth >= XactUndo.subxact_capacity) + { + int new_capacity = XactUndo.subxact_capacity * 2; + + XactUndo.subxact_stack = (XactUndoSubTransactionState *) + repalloc(XactUndo.subxact_stack, + new_capacity * sizeof(XactUndoSubTransactionState)); + /* Zero the newly-allocated portion */ + memset(&XactUndo.subxact_stack[XactUndo.subxact_capacity], 0, + (new_capacity - XactUndo.subxact_capacity) * + sizeof(XactUndoSubTransactionState)); + XactUndo.subxact_capacity = new_capacity; + } +} + +/* + * ResetXactUndo + * Reset all backend-private undo state for the next transaction. + */ +static void +ResetXactUndo(void) +{ + int i; + + XactUndo.has_undo = false; + XactUndo.subxact_depth = 0; + + /* + * The subxact_stack allocation persists across transactions (it's in + * TopMemoryContext). We just reset the depth and initialize slot 0. + */ + if (XactUndo.subxact_stack != NULL) + { + XactUndo.subxact_stack[0].nestingLevel = 1; + for (i = 0; i < NUndoPersistenceLevels; i++) + { + XactUndo.subxact_stack[0].start_location[i] = InvalidUndoRecPtr; + XactUndo.subxact_stack[0].last_batch_lsn[i] = InvalidXLogRecPtr; + } + } + + for (i = 0; i < NUndoPersistenceLevels; i++) + { + XactUndo.record_set[i] = NULL; + XactUndo.last_location[i] = InvalidUndoRecPtr; + XactUndo.last_batch_lsn[i] = InvalidXLogRecPtr; + } + + /* + * The chain is now the commit's, the abort's or the 2PC state's; none of + * those needs the in-flight record. + */ + PublishXactUndoChain(InvalidTransactionId); +} + +/* + * CollapseXactUndoSubTransactions + * Collapse all subtransaction state into the top level. + */ +static void +CollapseXactUndoSubTransactions(void) +{ + while (XactUndo.subxact_depth > 0) + { + /* + * Merge current level into parent by calling AtSubCommit_XactUndo + * with the current level's nestingLevel. + */ + AtSubCommit_XactUndo( + XactUndo.subxact_stack[XactUndo.subxact_depth].nestingLevel); + } +} diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 15b93512e4bf5..3b0a8c4e1d5c4 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -57,6 +57,7 @@ #include "replication/slot.h" #include "storage/copydir.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/ipc.h" #include "storage/lmgr.h" #include "storage/md.h" @@ -222,6 +223,8 @@ CreateDatabaseUsingWalLog(Oid src_dboid, Oid dst_dboid, CreateAndCopyRelationData(srcrlocator, dstrlocator, relinfo->permanent, relinfo->mainfork_page_std); + INJECTION_POINT("createdb-wal-log-after-copy-relation", NULL); + /* Release the relation locks. */ UnlockRelationId(&srcrelid, AccessShareLock); UnlockRelationId(&dstrelid, AccessShareLock); @@ -488,56 +491,95 @@ CreateDirAndVersionFile(char *dbpath, Oid dbid, Oid tsid, bool isRedo) sprintf(buf, "%s\n", PG_MAJORVERSION); nbytes = strlen(PG_MAJORVERSION) + 1; - /* Create database directory. */ - if (MakePGDirectory(dbpath) < 0) + /* + * Create database directory. The copy fills it through smgr, so an abort + * or crash before commit must remove the whole tree. + */ + if (!isRedo) { - /* Failure other than already exists or not in WAL replay? */ - if (errno != EEXIST || !isRedo) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create directory \"%s\": %m", dbpath))); + FileOpsMkdirTree(dbpath, pg_dir_create_mode, dbid); + } + else + { + if (MakePGDirectory(dbpath) < 0) + { + /* Failure other than already exists or not in WAL replay? */ + if (errno != EEXIST) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", dbpath))); + } } /* - * Create PG_VERSION file in the database path. If the file already - * exists and we are in WAL replay then try again to open it in write - * mode. + * Create PG_VERSION file in the database path. */ snprintf(versionfile, sizeof(versionfile), "%s/%s", dbpath, "PG_VERSION"); - fd = OpenTransientFile(versionfile, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY); - if (fd < 0 && errno == EEXIST && isRedo) - fd = OpenTransientFile(versionfile, O_WRONLY | O_TRUNC | PG_BINARY); + if (!isRedo) + { + /* + * Use FileOpsCreate + FileOpsWrite for transactional version file + * creation with crash-safe rollback support. + */ + fd = FileOpsCreate(versionfile, O_WRONLY | PG_BINARY, + pg_file_create_mode, true); + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_WRITE); + errno = 0; + if ((int) write(fd, buf, nbytes) != nbytes) + { + if (errno == 0) + errno = ENOSPC; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", versionfile))); + } + pgstat_report_wait_end(); - if (fd < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create file \"%s\": %m", versionfile))); + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_SYNC); + if (pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", versionfile))); + fsync_fname(dbpath, true); + pgstat_report_wait_end(); - /* Write PG_MAJORVERSION in the PG_VERSION file. */ - pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_WRITE); - errno = 0; - if (write(fd, buf, nbytes) != nbytes) - { - /* If write didn't set errno, assume problem is no disk space. */ - if (errno == 0) - errno = ENOSPC; - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not write to file \"%s\": %m", versionfile))); + CloseTransientFile(fd); } - pgstat_report_wait_end(); + else + { + /* WAL replay: use raw file operations */ + fd = OpenTransientFile(versionfile, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY); + if (fd < 0 && errno == EEXIST) + fd = OpenTransientFile(versionfile, O_WRONLY | O_TRUNC | PG_BINARY); - pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_SYNC); - if (pg_fsync(fd) != 0) - ereport(data_sync_elevel(ERROR), - (errcode_for_file_access(), - errmsg("could not fsync file \"%s\": %m", versionfile))); - fsync_fname(dbpath, true); - pgstat_report_wait_end(); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", versionfile))); - /* Close the version file. */ - CloseTransientFile(fd); + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_WRITE); + errno = 0; + if ((int) write(fd, buf, nbytes) != nbytes) + { + if (errno == 0) + errno = ENOSPC; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", versionfile))); + } + pgstat_report_wait_end(); + + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_SYNC); + if (pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", versionfile))); + fsync_fname(dbpath, true); + pgstat_report_wait_end(); + + CloseTransientFile(fd); + } /* If we are not in WAL replay then write the WAL. */ if (!isRedo) @@ -649,9 +691,13 @@ CreateDatabaseUsingFileCopy(Oid src_dboid, Oid dst_dboid, Oid src_tsid, /* * Copy this subdirectory to the new location * - * We don't need to copy subdirectories + * We don't need to copy subdirectories. Register the destination for + * delete-on-abort so a transaction abort recursively removes the + * partial copy. */ - copydir(srcpath, dstpath, false); + copydir(srcpath, dstpath, false, true); + + INJECTION_POINT("createdb-file-copy-after-copydir", NULL); /* Record the filesystem change in XLOG */ { @@ -2237,8 +2283,17 @@ movedb(const char *dbname, const char *tblspcname) FreeDir(dstdir); /* - * The directory exists but is empty. We must remove it before using + * The directory exists but is empty. We must remove it before using * the copydir function. + * + * NB: this rmdir is deliberately NOT routed through FILEOPS. The + * destination tablespace path is not yet referenced by any catalog + * row (the pg_database.dattablespace update happens later, and is + * what binds dst_dbpath to the database on commit), so an abort that + * leaves dst_dbpath missing does not orphan any catalog state -- the + * database remains in src_tblspcoid. copydir() below recreates + * dst_dbpath with FileOpsMkdirTree(), so an abort or a crash before + * commit removes the partial copy. */ if (rmdir(dst_dbpath) != 0) elog(ERROR, "could not remove directory \"%s\": %m", @@ -2261,9 +2316,13 @@ movedb(const char *dbname, const char *tblspcname) bool new_record_repl[Natts_pg_database] = {0}; /* - * Copy files from the old tablespace to the new one + * Copy files from the old tablespace to the new one. Register the + * destination for delete-on-abort so a transaction abort recursively + * removes the partial copy (closing the long-standing crash-asymmetry + * with the source tree, which is preserved until commit by movedb's + * post-commit cleanup). */ - copydir(src_dbpath, dst_dbpath, false); + copydir(src_dbpath, dst_dbpath, false, true); /* * Record the filesystem change in XLOG @@ -3423,9 +3482,12 @@ dbase_redo(XLogReaderState *record) /* * Copy this subdirectory to the new location * - * We don't need to copy subdirectories + * We don't need to copy subdirectories. Pass false for + * register_for_abort_cleanup: this is a WAL replay path with no + * surrounding transaction; the redo handler is itself the recovery + * action. */ - copydir(src_path, dst_path, false); + copydir(src_path, dst_path, false, false); pfree(src_path); pfree(dst_path); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index b6700929459f8..f01593bf38151 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -974,6 +974,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, break; default: (void) heap_reloptions(relkind, reloptions, true); + break; } if (stmt->ofTypename) diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c index 7c40176b7b9d2..0ba99f635afc3 100644 --- a/src/backend/commands/tablespace.c +++ b/src/backend/commands/tablespace.c @@ -70,6 +70,7 @@ #include "miscadmin.h" #include "postmaster/bgwriter.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/lmgr.h" #include "storage/lwlock.h" #include "storage/procsignal.h" @@ -600,11 +601,18 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) if (in_place) { - if (MakePGDirectory(linkloc) < 0 && errno != EEXIST) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create directory \"%s\": %m", - linkloc))); + if (!InRecovery) + { + FileOpsMkdir(linkloc, pg_dir_create_mode); + } + else + { + if (MakePGDirectory(linkloc) < 0 && errno != EEXIST) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", + linkloc))); + } } location_with_version_dir = psprintf("%s/%s", in_place ? linkloc : location, @@ -615,20 +623,36 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) * it doesn't exist or has the wrong owner. Not needed for in-place mode, * because in that case we created the directory with the desired * permissions. + * + * Outside recovery, route through FileOpsChmod so a transaction abort + * restores the prior mode via the FILEOPS UNDO record. In recovery + * (where there is no surrounding transaction) issue chmod directly. On + * Windows, chmod is partial (only the read-only bit is meaningful); + * FileOpsChmod handles platform differences. */ - if (!in_place && chmod(location, pg_dir_create_mode) != 0) + if (!in_place) { - if (errno == ENOENT) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_FILE), - errmsg("directory \"%s\" does not exist", location), - InRecovery ? errhint("Create this directory for the tablespace before " - "restarting the server.") : 0)); + int rc; + + if (InRecovery) + rc = chmod(location, pg_dir_create_mode); else - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not set permissions on directory \"%s\": %m", - location))); + rc = FileOpsChmod(location, pg_dir_create_mode); + + if (rc != 0) + { + if (errno == ENOENT) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FILE), + errmsg("directory \"%s\" does not exist", location), + InRecovery ? errhint("Create this directory for the tablespace before " + "restarting the server.") : 0)); + else + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set permissions on directory \"%s\": %m", + location))); + } } /* @@ -645,6 +669,8 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) (errcode_for_file_access(), errmsg("could not stat directory \"%s\": %m", location_with_version_dir))); + else if (!InRecovery) + FileOpsMkdir(location_with_version_dir, pg_dir_create_mode); else if (MakePGDirectory(location_with_version_dir) < 0) ereport(ERROR, (errcode_for_file_access(), @@ -671,11 +697,16 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) /* * Create the symlink under PGDATA */ - if (!in_place && symlink(location, linkloc) < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create symbolic link \"%s\": %m", - linkloc))); + if (!in_place) + { + if (!InRecovery) + FileOpsSymlink(location, linkloc); + else if (symlink(location, linkloc) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create symbolic link \"%s\": %m", + linkloc))); + } pfree(linkloc); pfree(location_with_version_dir); @@ -773,27 +804,50 @@ destroy_tablespace_directories(Oid tablespaceoid, bool redo) return false; } - /* remove empty directory */ - if (rmdir(subfile) < 0) - ereport(redo ? LOG : ERROR, - (errcode_for_file_access(), - errmsg("could not remove directory \"%s\": %m", - subfile))); + /* + * Remove empty directory. Outside recovery, defer to commit time so + * a transaction abort leaves the per-DB tablespace subdirectory + * intact. In recovery, issue rmdir directly because the redo handler + * is itself the recovery action. + */ + if (redo) + { + if (rmdir(subfile) < 0) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + subfile))); + } + else + { + FileOpsRmdir(subfile, true /* at_commit */ ); + } pfree(subfile); } FreeDir(dirdesc); - /* remove version directory */ - if (rmdir(linkloc_with_version_dir) < 0) + /* + * Remove version directory. Same forward / redo split as above: defer to + * commit outside recovery so abort restores the version dir; issue rmdir + * directly during redo. + */ + if (redo) { - ereport(redo ? LOG : ERROR, - (errcode_for_file_access(), - errmsg("could not remove directory \"%s\": %m", - linkloc_with_version_dir))); - pfree(linkloc_with_version_dir); - return false; + if (rmdir(linkloc_with_version_dir) < 0) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + linkloc_with_version_dir))); + pfree(linkloc_with_version_dir); + return false; + } + } + else + { + FileOpsRmdir(linkloc_with_version_dir, true /* at_commit */ ); } /* @@ -819,26 +873,53 @@ destroy_tablespace_directories(Oid tablespaceoid, bool redo) } else if (S_ISDIR(st.st_mode)) { - if (rmdir(linkloc) < 0) + /* + * Junction-style entry (Windows) or stale directory (any platform). + * In redo, remove now; outside redo, defer to commit so abort + * restores the entry. + */ + if (redo) { - int saved_errno = errno; + if (rmdir(linkloc) < 0) + { + int saved_errno = errno; - ereport(redo ? LOG : (saved_errno == ENOENT ? WARNING : ERROR), - (errcode_for_file_access(), - errmsg("could not remove directory \"%s\": %m", - linkloc))); + ereport(saved_errno == ENOENT ? LOG : LOG, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + linkloc))); + } + } + else + { + FileOpsRmdir(linkloc, true /* at_commit */ ); } } else if (S_ISLNK(st.st_mode)) { - if (unlink(linkloc) < 0) + /* + * Tablespace symlink under PGDATA/pg_tblspc/. Symmetric with + * FileOpsSymlink on the create path: defer the unlink to commit so + * abort restores the symlink, closing the long-standing + * crash-asymmetry between CREATE TABLESPACE and DROP TABLESPACE. In + * redo, issue unlink directly because the redo handler is itself the + * recovery action. + */ + if (redo) { - int saved_errno = errno; + if (unlink(linkloc) < 0) + { + int saved_errno = errno; - ereport(redo ? LOG : (saved_errno == ENOENT ? WARNING : ERROR), - (errcode_for_file_access(), - errmsg("could not remove symbolic link \"%s\": %m", - linkloc))); + ereport(saved_errno == ENOENT ? LOG : LOG, + (errcode_for_file_access(), + errmsg("could not remove symbolic link \"%s\": %m", + linkloc))); + } + } + else + { + FileOpsDelete(linkloc, true /* at_commit */ ); } } else diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 4dbc25c6c5054..3dbd1245d4090 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -5928,6 +5928,79 @@ UnlockBuffers(void) } } +/* + * BufferLockReleaseAll -- release any buffer content locks still held by this + * backend. + * + * Normally, buffer content locks are released as part of resource-owner + * cleanup (see ResOwnerReleaseBuffer, which calls BufferLockUnlock on any + * entry whose data.lockmode != BUFFER_LOCK_UNLOCK). However, that cleanup + * runs in ResourceOwnerRelease(RESOURCE_RELEASE_LOCKS), which is *after* + * AtAbort_XactUndo() in AbortTransaction(). Inline UNDO application may + * itself take buffer content locks, and BufferLockAcquire asserts that no + * lock is already held. If a statement errors out while holding a share + * or exclusive content lock, that assert would trip on any re-lock during + * UNDO application. + * + * This helper releases such stragglers early, before inline UNDO runs. + * It mirrors the still-locked branch of ResOwnerReleaseBuffer (matching + * the HOLD_INTERRUPTS/BufferLockUnlock discipline). Callers must be in + * an error-recovery path -- releasing content locks under a live statement + * would violate correctness. + * + * The subsequent ResourceOwnerRelease pass at RESOURCE_RELEASE_LOCKS still + * runs, but ResOwnerReleaseBuffer only calls BufferLockUnlock when + * data.lockmode != BUFFER_LOCK_UNLOCK, so already-released buffers are + * skipped -- no double-unlock. + */ +void +BufferLockReleaseAll(void) +{ + PrivateRefCountEntry *res; + int i; + + /* Walk the small array first (fast path, covers the usual case). */ + for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) + { + Buffer buffer = PrivateRefCountArrayKeys[i]; + + if (buffer == InvalidBuffer) + continue; + + res = &PrivateRefCountArray[i]; + if (res->data.lockmode == BUFFER_LOCK_UNLOCK) + continue; + + /* Local buffers do not have content locks. */ + if (BufferIsLocal(buffer)) + continue; + + HOLD_INTERRUPTS(); /* matched by RESUME_INTERRUPTS in + * BufferLockUnlock */ + BufferLockUnlock(buffer, GetBufferDescriptor(buffer - 1)); + } + + /* Then the overflow hash, if any. */ + if (PrivateRefCountOverflowed) + { + refcount_iterator iter; + + refcount_start_iterate(PrivateRefCountHash, &iter); + while ((res = refcount_iterate(PrivateRefCountHash, &iter)) != NULL) + { + Buffer buffer = res->buffer; + + if (res->data.lockmode == BUFFER_LOCK_UNLOCK) + continue; + if (BufferIsLocal(buffer)) + continue; + + HOLD_INTERRUPTS(); + BufferLockUnlock(buffer, GetBufferDescriptor(buffer - 1)); + } + } +} + /* * Acquire the buffer content lock in the specified mode * diff --git a/src/backend/storage/file/Makefile b/src/backend/storage/file/Makefile index 660ac51807e79..8bcfafec7cf7c 100644 --- a/src/backend/storage/file/Makefile +++ b/src/backend/storage/file/Makefile @@ -16,6 +16,8 @@ OBJS = \ buffile.o \ copydir.o \ fd.o \ + fileops.o \ + fileops_undo.o \ fileset.o \ reinit.o \ sharedfileset.o diff --git a/src/backend/storage/file/README b/src/backend/storage/file/README new file mode 100644 index 0000000000000..ece2768ac0af9 --- /dev/null +++ b/src/backend/storage/file/README @@ -0,0 +1,272 @@ +src/backend/storage/file/README + +Transactional File Operations (FILEOPS) +======================================= + +This directory's fileops.c implements FILEOPS: a set of filesystem +operations that participate in PostgreSQL transactions. An ordinary +unlink() or rename() issued in the middle of a transaction is not covered by +WAL and is not reversed if the transaction aborts or the server crashes. +FILEOPS closes that gap for the filesystem mutations the server performs +outside the buffer manager. + +FILEOPS is NOT used for relation files. Creating and dropping tables, +indexes, TRUNCATE, and CREATE/DROP DATABASE|TABLESPACE at the catalog/smgr +level go through smgr and catalog/storage.c, which have zero FileOps calls. +The only in-core callers of the FileOps* API are: + + - dbcommands.c (CREATE/ALTER/DROP DATABASE: directory + PG_VERSION, + ALTER DATABASE ... SET TABLESPACE) + - tablespace.c (CREATE/DROP TABLESPACE: the per-tablespace directory + and symlink; the LOCATION paths are absolute by design) + - copydir.c (the destination tree of a directory copy) + + +When each operation takes effect, and what undoes it +---------------------------------------------------- + +The rule: everything that CREATES or CHANGES an object takes effect at once +and is undone if the transaction aborts; DELETING an object is DEFERRED to +transaction end, so nothing is ever destroyed that a rollback would need. + + operation takes effect undone on abort by (*) + --------------------- ---------------- ------------------------------ + FileOpsCreate now unlink the file + FileOpsMkdir now rmdir the directory + FileOpsMkdirTree now remove the whole tree + FileOpsChmod / Chown now restore the old mode / owner + FileOpsTruncate now restore length and removed bytes + FileOpsWrite now restore the overwritten bytes + FileOpsSymlink / Link now unlink the link + FileOpsRename now rename back + FileOpsSetXattr / now restore the old value + FileOpsRemoveXattr + FileOpsDelete at commit (**) nothing to undo: not done yet + FileOpsRmdir at commit (**) nothing to undo: not done yet + + (*) at wal_level >= replica, by the operation's UNDO record: on ROLLBACK, + ROLLBACK TO SAVEPOINT and ROLLBACK PREPARED (applied with the abort + record, which carries the inverses), on a standby replaying that + abort record, and by the startup undo phase for a transaction a crash + left in flight. At wal_level = minimal, by an abort-time pending op + on ROLLBACK and ROLLBACK TO SAVEPOINT only (see below). + (**) or at abort, with at_commit = false. + +Each undo reverts only an object that still shows the operation's effect +(fileops_undo_apply() in fileops_undo.c), so a name reused in the meantime is +left alone. Undo runs newest first, so an operation on a renamed file's new +name is undone before the rename. + + +Immediate operations +-------------------- + +These run when the API is called, so the file or directory can be used +within the same transaction, and each registers an abort-time undo action +that reverses it if the transaction rolls back: + + FileOpsCreate -- create() now, an ERROR if the file exists; + UNDO/pending-delete on abort + FileOpsMkdir -- mkdir() now; UNDO rmdir on abort + FileOpsMkdirTree -- mkdir() now; UNDO removes the whole tree on abort + FileOpsChmod -- chmod() now; UNDO restore old mode on abort + FileOpsChown -- chown() now; UNDO restore old uid/gid on abort + FileOpsTruncate -- ftruncate() now; UNDO restore old length on abort + FileOpsSymlink -- symlink() now; UNDO unlink on abort + FileOpsLink -- link() now; UNDO unlink on abort + FileOpsRename -- rename() now; UNDO rename back on abort + FileOpsWrite -- pwrite() now (WAL-logged, bounded before-image UNDO) + FileOpsSetXattr / FileOpsRemoveXattr -- now; UNDO restore old value on abort + +FileOpsRename(oldpath, newpath) renames a file or directory. It never +replaces newpath: an existing newpath is an ERROR (ERRCODE_DUPLICATE_FILE), +as are a missing oldpath and a newpath on another filesystem, and each +changes nothing. Undoing a replacement would need the replaced object kept +somewhere until the transaction ends, and no caller needs one. The rename +is reversed only while newpath exists and oldpath is free. Like the undo of +FileOpsCreate and FileOpsMkdir, that test looks at the paths alone: the +device and inode of the object are not stable across a restore, a promoted +standby, or redo that makes the object again, so ROLLBACK PREPARED on a +promoted standby could not use them. +Unlike the other immediate operations it writes and flushes its WAL and UNDO +records BEFORE the rename() call, so a crash cannot lose the old name with +nothing on record: a crash before the call is harmless, since redo makes the +rename and the loser's UNDO reverses it. + + +Deferred operations +------------------- + +The deleting operations only record a PendingFileOp, and the filesystem is +touched at transaction end: + + FileOpsDelete -- deferred; at_commit chooses commit vs abort + FileOpsRmdir -- deferred; at_commit chooses commit vs abort + +FileOpsDoPendingOps(isCommit) runs the pending list at transaction end: the +at_commit=true ops fire on commit, the at_commit=false ops fire on abort. + + +When FILEOPS is durable: only at wal_level >= replica +----------------------------------------------------- + +FILEOPS writes WAL and UNDO records ONLY when XLogIsNeeded() is true, i.e. +wal_level >= replica. At wal_level = minimal it emits no WAL and no UNDO: + + - There is NO crash safety at minimal. The immediate constructive ops + mutate the filesystem with no WAL record to redo, and the only reversal + for a crash-mid-transaction is the in-memory pendingFileOps list, which + does not survive a crash. + + - There is NO 2PC safety at minimal. With nothing serialized, a PREPARE'd + transaction's file operations cannot be reproduced after a crash. + +Every FILEOPS + crash-recovery + 2PC guarantee below assumes wal_level >= +replica, which is what any deployment relying on those guarantees must set. + + +Revert and recovery model (NO fileops_cleanup) +---------------------------------------------- + +There is NO fileops_cleanup() function. The RM_FILEOPS_ID WAL resource +manager registers only rm_redo/rm_desc/rm_identify; its rm_startup and +rm_cleanup slots are NULL (see access/rmgrlist.h). Reversal rides two +mechanisms, both on the COMMON WAL stream: + + 1. Live-transaction abort (in-memory). ROLLBACK runs + FileOpsDoPendingOps(false), executing each constructive op's registered + abort-time undo (delete the created file, restore the old + mode/length/xattr, rmdir the created directory, ...). No WAL needed; + the in-process list has the state. + + 2. Crash / async revert (common-WAL UNDO records). Each constructive op + also prepares an UNDO record (PrepareXactUndoData(UNDO_RMID_FILEOPS,...)) + and inserts it (InsertXactUndoData) in the same critical section as its + forward WAL record. On crash, a loser transaction's UNDO chain is + applied by the cluster-wide UNDO engine's undo phase at the end of + recovery (UndoRecoveryRollBack), which logs the records it applied in + an XLOG_UNDO_ROLLBACK record for standbys and later recoveries to + replay; the apply side is fileops_undo_apply() in fileops_undo.c, + registered through the UNDO rmgr registry (FileopsUndoRmgrInit), NOT + through a WAL rm_cleanup callback. + +Deferred ops emit their forward WAL record at commit-EXECUTION time, inside +FileOpsDoPendingOps() when the op actually happens on the primary, so a +physical standby reproduces it by redo. Their redo handlers are idempotent +(delete of an already-absent file, rmdir of an already-absent directory both +count as success). Only the commit path is logged; an at_commit=false +delete/rmdir run during abort emits no WAL, because a standby must never +apply an aborted transaction's effect. + +Redo replays each record onto the filesystem as it is now, which after a +crash may already include later operations, and so it cannot tell a name +reused later from the original. This is a known limit of FILEOPS redo: +making FILEOPS fully crash-atomic is future work (a resource manager that +logs enough to replay against the state the record was made in). Rename +redo converges on the primary's state as far as the paths allow: + + - oldpath absent: the rename was made already; nothing to do. + - newpath absent: rename oldpath to newpath. + - both present: oldpath was made again by the replay of an earlier record + (the CREATE, MKDIR or LINK that made it since the checkpoint), or by a + later operation, and newpath is the later state. Remove oldpath (a + directory only if it is empty) and keep newpath. If oldpath is a + non-empty directory, leave both, with a WARNING. + +Renaming over newpath instead would lose data. After a -> b, then b -> c, +then a new b, with a older than the checkpoint, redo of b -> c finds the new +b and c both present, and would put the new b in place of c. + +One deliberate exception: + + SYMLINK redo is an intentional no-op. FileOpsSymlink is used only for + tablespace symlinks; standby/crash-recovery replay of those is owned by + RM_TBLSPC, which applies the recovery-time layout remap. + + +Two-phase commit +---------------- + +PREPARE TRANSACTION serializes the transaction's pending FileOps into the +2PC state file so they survive to COMMIT/ROLLBACK PREPARED, including from a +different backend after a crash between PREPARE and the final decision: + + - FileOpsAtPrepare(), called from EndPrepare() between StartPrepare() and + the END sentinel, writes one TWOPHASE_RM_FILEOPS_ID record per pending + at_commit=true op into the durable 2PC state. + + - The FILEOPS 2PC post-commit callback performs those deferred commit-time + ops (delete/rmdir), reusing the same per-op executor as + FileOpsDoPendingOps(). COMMIT PREPARED thus applies them; a crash before + it re-drives them from the durable 2PC state. + + - The immediate ops' inverses are serialized separately, as one + FILEOPS_TWOPHASE_INVERSES record. ROLLBACK PREPARED carries them in its + abort record and applies them before it releases the prepared + transaction's locks, as an ordinary abort does; COMMIT PREPARED ignores + them. The FILEOPS post-abort callback is therefore effectively a no-op. + + - PostPrepare_FileOps() then discards this backend's in-memory list; the + durable 2PC state (for commit-time ops) and the UNDO chain (for abort-time + reversal), not this backend, drive the eventual work. + +Before this, PostPrepare_FileOps() freed the pending list without recording +it, so deferred commit-time ops were lost across PREPARE. Adding the +TWOPHASE_RM_FILEOPS_ID record type bumped TWOPHASE_MAGIC (twophase.c). + + +How to use the API +------------------ + +The public API is declared in src/include/storage/fileops.h. Call these in +place of the raw libc/syscall equivalents whenever the operation must be +transactional. + + register_delete (FileOpsCreate). When true, the newly created file is + registered for deletion if the transaction aborts. + + at_commit (FileOpsDelete, FileOpsRmdir). + at_commit = true removes on commit (dropping state the transaction is + logically deleting); at_commit = false removes only on abort. + +FileOpsMkdirTree creates a directory that the transaction then fills by +other means (smgr for CREATE DATABASE, copydir() for FILE_COPY and ALTER +DATABASE SET TABLESPACE). Its UNDO record and its abort-time pending op +remove the whole tree, whatever ended up in it, where FileOpsMkdir's only +rmdir() an empty directory. For a database directory the UNDO first drops +that database's shared buffers and pending fsync requests. + +The API hides platform differences: PG_O_DIRECT for unbuffered I/O, +pg_fsync() for the correct per-platform flush, and fsync_parent_path() for +directory durability on Unix. + + +Transaction lifecycle integration +---------------------------------- + +The deferred operations are driven by AM-neutral hooks that xact.c calls +(wired to the FileOps* functions in fileops_undo.c): + + FileOpsDoPendingOps(isCommit) -- executes the pending list at end of xact. + AtSubCommit_FileOps() -- reparents a subxact's pending ops. + AtSubAbort_FileOps() -- runs a subxact's abort-time ops. + FileOpsAtPrepare() -- serializes pending ops into 2PC state. + PostPrepare_FileOps() -- discards the in-memory list after PREPARE. + + +WAL replay +---------- + +fileops_redo() dispatches each XLOG_FILEOPS_* record to the matching action +during recovery; fileops_desc()/fileops_identify() render records for +pg_waldump. Because the redo handler is itself the recovery action, code +paths that run under WAL replay (the "redo" arms in dbcommands.c, +tablespace.c, and copydir.c) call the raw syscalls directly rather than +re-entering the FileOps* API. + +Tests: src/test/recovery/t/063_fileops_recovery.pl (crash recovery), +064_fileops_2pc.pl (constructive-op 2PC via chmod), +065_fileops_standby_divergence.pl (physical-standby convergence), +062_fileops_2pc_deferred.pl (deferred commit-time ops through PREPARE + +crash), and the test_fileops module's regression test and TAP tests +(018_fileops_rename.pl covers FileOpsRename). diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c index ee42c796f773a..4eb46d3498a41 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -24,11 +24,14 @@ #include #include +#include "access/xlog.h" +#include "common/file_perm.h" #include "common/file_utils.h" #include "miscadmin.h" #include "pgstat.h" #include "storage/copydir.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "utils/wait_event.h" /* GUCs */ @@ -44,16 +47,24 @@ static void clone_file(const char *fromfile, const char *tofile); * * This function uses the file_copy_method GUC. New uses of this function must * be documented in doc/src/sgml/config.sgml. + * + * If register_for_abort_cleanup is true and we are not in WAL replay, the + * destination is created with FileOpsMkdirTree(), so that a transaction + * abort, or a crash before commit, removes the partial copy. Recovery + * callers pass false because they are themselves the recovery action. */ void -copydir(const char *fromdir, const char *todir, bool recurse) +copydir(const char *fromdir, const char *todir, bool recurse, + bool register_for_abort_cleanup) { DIR *xldir; struct dirent *xlde; char fromfile[MAXPGPATH * 2]; char tofile[MAXPGPATH * 2]; - if (MakePGDirectory(todir) != 0) + if (register_for_abort_cleanup && !RecoveryInProgress()) + FileOpsMkdirTree(todir, pg_dir_create_mode, InvalidOid); + else if (MakePGDirectory(todir) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not create directory \"%s\": %m", todir))); @@ -78,9 +89,13 @@ copydir(const char *fromdir, const char *todir, bool recurse) if (xlde_type == PGFILETYPE_DIR) { - /* recurse to handle subdirectories */ + /* + * Recurse to handle subdirectories. Pass false for + * register_for_abort_cleanup: removing the top-level destination + * tree removes them too. + */ if (recurse) - copydir(fromfile, tofile, true); + copydir(fromfile, tofile, true, false); } else if (xlde_type == PGFILETYPE_REG) { diff --git a/src/backend/storage/file/fileops.c b/src/backend/storage/file/fileops.c new file mode 100644 index 0000000000000..045557dccf072 --- /dev/null +++ b/src/backend/storage/file/fileops.c @@ -0,0 +1,2471 @@ +/*------------------------------------------------------------------------- + * + * fileops.c + * Transactional file operations with WAL logging + * + * This module provides transactional filesystem operations that integrate + * with PostgreSQL's WAL and transaction management. It is NOT used for + * relation files (smgr/catalog handle those); the only in-core callers are + * dbcommands.c, tablespace.c, and copydir.c. + * + * Two op classes: + * - IMMEDIATE ops (create, mkdir, chmod, chown, truncate, symlink, link, + * rename, write, setxattr) take effect at once, and each registers an + * abort-time undo action (a common-WAL UNDO record plus, where relevant, + * a pending op) that reverses it on rollback. + * - DESTRUCTIVE ops (delete, rmdir, rmtree) are DEFERRED: a PendingFileOp + * is recorded now and the filesystem is touched at transaction end by + * FileOpsDoPendingOps(). + * + * WAL and UNDO are written only when XLogIsNeeded() (wal_level >= replica); + * at wal_level = minimal there is no crash or 2PC safety. + * + * An immediate op performs its system call FIRST, outside any critical + * section, and WAL-logs itself only once the call has succeeded: the forward + * WAL record and its UNDO record are inserted and flushed together in a short + * critical section (fileops_log()). So an ordinary filesystem error + * (permission denied, no space, ...) is a plain ERROR, as it is for the same + * call without FILEOPS, and a failed op leaves nothing in the WAL for redo or + * a standby to replay. + * + * This is NOT atomic. Between the system call and the WAL flush there is a + * window in which the change is on disk with no record of it: a crash there + * leaves the change in place, unknown to redo, to a standby and to rollback, + * just as the bare system call would. Nothing reverts it. The + * opposite order (log, then act inside a critical section) closes that window + * only by turning every filesystem error into a PANIC. + * + * FileOpsRename() is the exception: there the window would lose the old name + * for good, so it logs first and renames after, outside the critical section. + * A failed rename() is still a plain ERROR, because its inverse renames back + * only while the new name exists and the old one does not (see there). + * + * Reverting checks the object at the recorded path before touching it (see + * fileops_undo.c), but a path, its type and, where recorded, its inode are + * evidence that the object is the one this transaction made, not proof: + * something else can have replaced it in between. + * + * The deferred-operations pattern works as follows: + * 1. A destructive API call records a PendingFileOp entry in a linked list + * 2. At commit/abort time, FileOpsDoPendingOps() executes or discards + * the pending operations based on transaction outcome (a commit-time + * destructive op emits its WAL record at execution time) + * 3. At PREPARE, FileOpsAtPrepare() serializes the pending ops into the + * 2PC state file so COMMIT/ROLLBACK PREPARED can perform them + * + * Subtransaction support: + * - At subtransaction commit, entries are reassigned to the parent level + * - At subtransaction abort, abort-time actions execute immediately + * + * Platform-specific handling: + * - O_DIRECT: Uses PG_O_DIRECT abstraction (Linux native O_DIRECT, + * macOS F_NOCACHE via fcntl, Windows FILE_FLAG_NO_BUFFERING) + * - fsync: Uses pg_fsync() which selects the appropriate mechanism + * (Linux fdatasync, macOS F_FULLFSYNC, Windows FlushFileBuffers, + * BSD fsync) + * - Directory sync: Uses fsync_fname()/fsync_parent_path() which + * handle directory fsync on Unix platforms (not needed on Windows) + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/file/fileops.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#ifdef HAVE_SYS_FCNTL_H +#include +#endif + +/* + * ENODATA is Linux-specific; FreeBSD and other BSDs don't define it. + * When removing extended attributes, ENODATA means "attribute does not + * exist" -- equivalent to ENOATTR on BSDs. + */ +#ifndef ENODATA +#ifdef ENOATTR +#define ENODATA ENOATTR +#else +#define ENODATA ENOENT +#endif +#endif + +#include "access/fileops_xlog.h" +#include "access/rmgr.h" +#include "access/twophase_rmgr.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/xact.h" +#include "access/xactundo.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "catalog/pg_class.h" +#include "miscadmin.h" +#include "port/pg_xattr.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "storage/proc.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" + + +/* Head of the pending file operations linked list */ +static PendingFileOp *pendingFileOps = NULL; +static Size inverseBytes = 0; + +static void FileOpsExecOne(PendingFileOp *pending, bool isCommit, + XLogRecPtr *max_lsn); + +/* + * fileops_fsync_parent -- fsync the parent directory of a file path + * + * This ensures that directory entry changes (create, delete, rename) + * are durable. On Windows, directory fsync is not needed because NTFS + * journals directory entries; fsync_fname_ext() handles this by being + * a no-op for directories on Windows. + */ +static void +fileops_fsync_parent(const char *fname, int elevel) +{ + char parentpath[MAXPGPATH]; + char *sep; + + strlcpy(parentpath, fname, MAXPGPATH); + + sep = strrchr(parentpath, '/'); + if (sep != NULL) + { + /* Got a path component, fsync the directory portion */ + if (sep == parentpath) + parentpath[1] = '\0'; /* root directory */ + else + *sep = '\0'; + + fsync_fname_ext(parentpath, true, true, elevel); + } +} + +/* + * fileops_pathlen - Length of a path including its terminating NUL + * + * WAL and UNDO records store path lengths in a uint16 and the UNDO payload + * buffers are sized for MAXPGPATH, so refuse a longer path rather than + * truncate it (or overrun a buffer). Called before the system call, so a + * refused path changes nothing. + */ +static int +fileops_pathlen(const char *path) +{ + size_t len = strlen(path) + 1; + + /* Reserve room before any mutation; abort itself must never exceed WAL. */ + if (inverseBytes + 4 * MAXPGPATH + 2 * FILEOPS_MAX_DATA_BYTES > + FILEOPS_MAX_ABORT_BYTES) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("too much transactional file operation undo data"))); + + if (len > MAXPGPATH) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("file operation path length %zu exceeds maximum %d", + len, MAXPGPATH))); + return (int) len; +} + +/* + * fileops_log - WAL-log an immediate operation that has already succeeded + * + * The caller has made the filesystem change, prepared its UNDO record (if + * any, undo_ctx != NULL) with PrepareXactUndoData(), and registered the + * forward record's data after XLogBeginInsert(). Insert both and flush them + * through the end of the UNDO batch, so that a crash leaves either both or + * neither. Only this is a critical section; see the file header comment. + */ +static void +fileops_log(uint8 info, XactUndoContext *undo_ctx) +{ + START_CRIT_SECTION(); + (void) XLogInsert(RM_FILEOPS_ID, info); + if (undo_ctx != NULL) + { + InsertXactUndoData(undo_ctx); + CleanupXactUndoInsertion(undo_ctx); + } + XLogFlush(XactLastRecEnd); + END_CRIT_SECTION(); +} + +/* + * AddPendingFileOp - Add a new pending file operation to the list + * + * All fields are deep-copied into TopMemoryContext to survive + * until transaction end, following the PendingRelDelete pattern. + */ +static void +AddPendingFileOp(PendingFileOpType type, const char *path, + const char *newpath, off_t length, bool at_commit) +{ + PendingFileOp *pending; + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(TopMemoryContext); + + pending = (PendingFileOp *) palloc(sizeof(PendingFileOp)); + pending->type = type; + pending->path = pstrdup(path); + pending->newpath = newpath ? pstrdup(newpath) : NULL; + pending->length = length; + pending->data = NULL; + pending->data_len = 0; + pending->at_commit = at_commit; + pending->nestLevel = GetCurrentTransactionNestLevel(); + pending->next = pendingFileOps; + pendingFileOps = pending; + + MemoryContextSwitchTo(oldcxt); +} + +/* + * AddPendingFileOpWithData - Add a pending file operation with extra data + * + * Like AddPendingFileOp but also stores arbitrary data (e.g., original + * xattr value for restore on abort). + */ +static void +AddPendingFileOpWithData(PendingFileOpType type, const char *path, + const char *newpath, off_t length, + const void *data, size_t data_len, + bool at_commit) +{ + PendingFileOp *pending; + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(TopMemoryContext); + + pending = (PendingFileOp *) palloc(sizeof(PendingFileOp)); + pending->type = type; + pending->path = pstrdup(path); + pending->newpath = newpath ? pstrdup(newpath) : NULL; + pending->length = length; + if (data && data_len > 0) + { + pending->data = palloc(data_len); + memcpy(pending->data, data, data_len); + pending->data_len = data_len; + } + else + { + pending->data = NULL; + pending->data_len = 0; + } + pending->at_commit = at_commit; + pending->nestLevel = GetCurrentTransactionNestLevel(); + pending->next = pendingFileOps; + pendingFileOps = pending; + + MemoryContextSwitchTo(oldcxt); +} + +/* Keep the same inverse for online abort, abort redo, and crash losers. */ +static void +fileops_prepare_undo(XactUndoContext *ctx, char persistence, uint8 rmid, + uint16 info, Oid reloid, const char *payload, Size len) +{ + AddPendingFileOpWithData(PENDING_FILEOP_INVERSE, "", NULL, + info, payload, len, false); + inverseBytes += len + 2 * sizeof(uint32); + FileOpsReserveInverseSyncs(); + PrepareXactUndoData(ctx, persistence, rmid, info, reloid, payload, len); +} + +/* Serialize in execution (newest-first) order, without alignment assumptions. */ +void +FileOpsGetAbortData(char **data, uint32 *len) +{ + PendingFileOp *p; + StringInfoData buf; + int level = GetCurrentTransactionNestLevel(); + + *data = NULL; + *len = 0; + if (inverseBytes == 0) + return; + initStringInfo(&buf); + for (p = pendingFileOps; p; p = p->next) + { + uint32 info = p->length; + uint32 size = p->data_len; + + if (p->type != PENDING_FILEOP_INVERSE || p->nestLevel < level) + continue; + appendBinaryStringInfo(&buf, &info, sizeof(info)); + appendBinaryStringInfo(&buf, &size, sizeof(size)); + appendBinaryStringInfo(&buf, p->data, size); + } + *data = buf.data; + *len = buf.len; +} + +/* + * CreateCheckPoint waits for this flag after choosing its redo pointer. Even + * though inverses ride in abort WAL, a checkpoint must not omit that record + * until the corresponding filesystem changes have been fsynced. A database + * directory tree can make this interval long; there is no retry loop here. + */ +void +FileOpsBeginAbort(void) +{ + MyProc->delayChkptFlags |= DELAY_CHKPT_START; +} + +void +FileOpsEndAbort(void) +{ + MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; +} + +/* Apply the self-contained abort payload, also used during WAL replay. */ +void +FileOpsApplyAbort(const char *data, uint32 len) +{ + while (len > 0) + { + uint32 info; + uint32 size; + + if (len < 2 * sizeof(uint32)) + elog(PANIC, "invalid FILEOPS abort payload"); + memcpy(&info, data, sizeof(info)); + memcpy(&size, data + sizeof(info), sizeof(size)); + data += 2 * sizeof(uint32); + len -= 2 * sizeof(uint32); + if (size > len) + elog(PANIC, "invalid FILEOPS inverse length"); + FileOpsApplyInverse(info, data, size); + data += size; + len -= size; + } + FileOpsSyncInverses(PANIC); +} + +/* + * FreePendingFileOp - Free a pending file operation entry + */ +static void +FreePendingFileOp(PendingFileOp *pending) +{ + if (pending->type == PENDING_FILEOP_INVERSE) + inverseBytes -= pending->data_len + 2 * sizeof(uint32); + if (pending->path) + pfree(pending->path); + if (pending->newpath) + pfree(pending->newpath); + if (pending->data) + pfree(pending->data); + pfree(pending); +} + +/* + * FileOpsCreate - Create a file within a transaction + * + * Creates the file immediately (so it can be used within the transaction) + * and logs the creation to WAL. If register_delete is true, the file will + * be deleted if the transaction aborts. + * + * A path that already exists is an ERROR (ERRCODE_DUPLICATE_FILE), as for + * FileOpsMkdir(), FileOpsLink() and FileOpsRename(). The UNDO record + * unlinks the path, so the path must name a file this call created: opening + * an existing file here would let a rollback remove a committed one. + * + * Platform handling: + * - Linux/FreeBSD: O_DIRECT passed directly to open() + * - macOS: F_NOCACHE fcntl applied after open() + * - Windows: FILE_FLAG_NO_BUFFERING (handled by port layer) + * + * Returns the file descriptor; failing to create the file is an ERROR. + */ +int +FileOpsCreate(const char *path, int flags, mode_t mode, bool register_delete) +{ + int fd; + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + fd = OpenTransientFilePerm(path, flags | O_CREAT | O_EXCL, mode); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", path))); + + if (register_delete && !XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_DELETE, path, NULL, 0, false); + + if (enableFsync) + { + if (pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", path))); + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + + if (XLogIsNeeded()) + { + xl_fileops_create xlrec; + char undo_payload[sizeof(FileopsUndoCreate) + MAXPGPATH]; + FileopsUndoCreate *hdr = (FileopsUndoCreate *) undo_payload; + XactUndoContext undo_ctx; + + xlrec.flags = flags; + xlrec.mode = mode; + xlrec.register_delete = register_delete; + + hdr->path_len = (uint16) pathlen; + memcpy(undo_payload + sizeof(FileopsUndoCreate), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CREATE, + InvalidOid, undo_payload, + sizeof(FileopsUndoCreate) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsCreate); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_CREATE, &undo_ctx); + + INJECTION_POINT("fileops-create-after-wal-flush", NULL); + } + + return fd; +} + +/* + * FileOpsDelete - Schedule a file deletion within a transaction + * + * The file is not deleted immediately. Instead, the deletion is deferred + * to transaction commit (if at_commit is true) or abort (if false). + * On Unix: unlink() with parent dir fsync. + * On Windows: pgunlink() with retry on EACCES. + */ +void +FileOpsDelete(const char *path, bool at_commit) +{ + Assert(!IsInParallelMode()); + + /* + * no WAL is emitted at registration time. The deferred delete is + * WAL-logged in FileOpsDoPendingOps() at commit-execution time, when it + * actually happens on the primary, so a physical standby reproduces it + * via redo. Registration-time logging would be a no-op on redo (the + * standby never runs FileOpsDoPendingOps()) and, for at_commit=false + * delete-on-abort registrations, would wrongly apply an aborted + * transaction's effect on the standby. + */ + AddPendingFileOp(PENDING_FILEOP_DELETE, path, NULL, 0, at_commit); +} + +/* + * FileOpsRename - Rename a file or directory within a transaction + * + * Immediate execution: when this returns, newpath exists and oldpath does + * not, and the rename is reversed if the transaction or subtransaction + * aborts (rename back, while newpath exists and oldpath is free). Unlike + * rename(2), it never replaces newpath: rolling back a + * replacement would need the replaced object kept somewhere, and no caller + * needs one. An existing newpath, a missing oldpath and a newpath on another + * filesystem are each an ERROR that changes nothing. The check that newpath + * is free is not atomic with the rename; callers serialize the use of a path, + * as they must for any FILEOPS operation. + * + * Unlike the other immediate operations, the WAL and UNDO records are + * written and flushed BEFORE the system call. The other order would let a + * crash between rename() and the flush lose the old name for good, with + * nothing on record to undo it. In this order a crash before rename() is + * harmless: redo performs the rename and the loser's UNDO reverses it. If + * rename() fails after the WAL, it is an ERROR: the abort's inverse finds + * oldpath still in place and leaves it, and a standby that replayed the + * forward record reverses it when it replays the abort record. + * + * At wal_level = minimal nothing is logged, and an abort-time pending op + * renames the object back. + * + * On Windows, pgrename() retries MoveFileEx() while the file is in use. + * + * Returns 0; any failure is an ERROR. + */ +int +FileOpsRename(const char *oldpath, const char *newpath) +{ + int oldpathlen = fileops_pathlen(oldpath); + int newpathlen = fileops_pathlen(newpath); + struct stat st; + struct stat dst; + char parent[MAXPGPATH]; + + Assert(!IsInParallelMode()); + + if (lstat(oldpath, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", oldpath))); + if (lstat(newpath, &dst) == 0) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_FILE), + errmsg("could not rename file \"%s\" to \"%s\": file already exists", + oldpath, newpath))); + if (errno != ENOENT) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", newpath))); + strlcpy(parent, newpath, sizeof(parent)); + get_parent_directory(parent); + if (parent[0] == '\0') + strlcpy(parent, ".", sizeof(parent)); + if (stat(parent, &dst) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat directory \"%s\": %m", parent))); + if (dst.st_dev != st.st_dev) + { + errno = EXDEV; + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("could not rename file \"%s\" to \"%s\": %m", + oldpath, newpath))); + } + + if (XLogIsNeeded()) + { + xl_fileops_rename xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoRename) + 2 * MAXPGPATH]; + FileopsUndoRename *hdr = (FileopsUndoRename *) payload; + + xlrec.oldpath_len = (uint16) oldpathlen; + + hdr->oldpath_len = (uint16) oldpathlen; + hdr->newpath_len = (uint16) newpathlen; + memcpy(payload + sizeof(FileopsUndoRename), oldpath, oldpathlen); + memcpy(payload + sizeof(FileopsUndoRename) + oldpathlen, newpath, + newpathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_RENAME, + InvalidOid, payload, + sizeof(FileopsUndoRename) + oldpathlen + + newpathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRename); + XLogRegisterData(oldpath, oldpathlen); + XLogRegisterData(newpath, newpathlen); + fileops_log(XLOG_FILEOPS_RENAME, &undo_ctx); + + INJECTION_POINT("fileops-rename-after-wal-flush", NULL); + } + else + { + /* Registered first: it does nothing if the rename does not happen. */ + AddPendingFileOp(PENDING_FILEOP_RENAME, newpath, oldpath, 0, false); + } + + if (rename(oldpath, newpath) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", + oldpath, newpath))); + + if (enableFsync) + { + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + fileops_fsync_parent(oldpath, data_sync_elevel(ERROR)); + } + + return 0; +} + +/* Capture a bounded range before changing it, including shrink tails. */ +static char * +fileops_before_image(int fd, const char *path, off_t offset, Size len) +{ + char *data; + + if (len > FILEOPS_MAX_DATA_BYTES) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("file operation before-image exceeds %d bytes", + FILEOPS_MAX_DATA_BYTES))); + data = palloc(len ? len : 1); + if (len && pg_pread(fd, data, len, offset) != len) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read before-image of file \"%s\": %m", path))); + return data; +} + +/* + * FileOpsWrite - Write data to a file at a specific offset + * + * Immediate execution, WAL-logged for crash recovery replay. + * Uses pwrite() on POSIX. On Windows: SetFilePointerEx + WriteFile. + * + * Bounded before-images restore overwritten bytes and the original length. + * + * Returns 0; a failed write is an ERROR. + */ +int +FileOpsWrite(const char *path, off_t offset, const void *data, uint32 len) +{ + int fd; + ssize_t nbytes; + int pathlen = fileops_pathlen(path); + struct stat st; + char *before; + uint32 before_len; + char *payload; + Size payload_len; + FileopsUndoWrite *hdr; + XactUndoContext undo_ctx; + + Assert(!IsInParallelMode()); + + fd = OpenTransientFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for writing: %m", path))); + + if (offset < 0 || len > FILEOPS_MAX_DATA_BYTES || + offset > PG_INT64_MAX - len) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("invalid or excessive transactional file write range"))); + if (fstat(fd, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", path))); + before_len = offset < st.st_size ? Min((off_t) len, st.st_size - offset) : 0; + before = fileops_before_image(fd, path, offset, before_len); + payload_len = sizeof(FileopsUndoWrite) + pathlen + before_len + len; + payload = palloc0(payload_len); + hdr = (FileopsUndoWrite *) payload; + hdr->orig_length = st.st_size; + hdr->new_length = Max(st.st_size, offset + len); + hdr->offset = offset; + hdr->before_len = before_len; + hdr->after_len = len; + hdr->path_len = pathlen; + memcpy(payload + sizeof(*hdr), path, pathlen); + memcpy(payload + sizeof(*hdr) + pathlen, before, before_len); + memcpy(payload + sizeof(*hdr) + pathlen + before_len, data, len); + + /* Keep an inverse even if pwrite changes only part of the range. */ + if (XLogIsNeeded()) + { + (void) GetCurrentTransactionId(); + AddPendingFileOpWithData(PENDING_FILEOP_INVERSE, "", NULL, + FILEOPS_UNDO_WRITE, payload, payload_len, false); + inverseBytes += payload_len + 2 * sizeof(uint32); + } + + nbytes = pg_pwrite(fd, data, + IS_INJECTION_POINT_ATTACHED("fileops-write-short") ? len / 2 : len, + offset); + if (nbytes != (ssize_t) len) + { + int save_errno = errno; + + /* Match the actual after-image, not the requested complete write. */ + if (XLogIsNeeded()) + { + FileopsUndoWrite *inverse = (FileopsUndoWrite *) pendingFileOps->data; + + inverse->after_len = nbytes > 0 ? nbytes : 0; + inverse->new_length = nbytes > 0 ? + Max(st.st_size, offset + nbytes) : st.st_size; + pendingFileOps->data_len -= len - inverse->after_len; + } + CloseTransientFile(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write %u bytes to file \"%s\" at offset %lld: %m", + len, path, (long long) offset))); + } + + if (enableFsync && pg_fsync(fd) != 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after write: %m", path))); + } + + if (CloseTransientFile(fd) != 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + + if (XLogIsNeeded()) + { + xl_fileops_write xlrec; + + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_WRITE, + InvalidOid, payload, payload_len); + xlrec.offset = offset; + xlrec.len = len; + xlrec.path_len = (uint16) pathlen; + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsWrite); + XLogRegisterData(path, pathlen); + XLogRegisterData(data, len); + fileops_log(XLOG_FILEOPS_WRITE, &undo_ctx); + } + + pfree(before); + pfree(payload); + return 0; +} + +/* + * FileOpsTruncate - Truncate a file within a transaction + * + * Executed immediately and WAL-logged. Uses ftruncate() on POSIX, + * SetEndOfFile() on Windows. File is fsynced after truncation. The + * original size is restored if the transaction aborts. + */ +void +FileOpsTruncate(const char *path, off_t length) +{ + int fd; + struct stat st; + off_t orig_length = 0; + int pathlen = fileops_pathlen(path); + char *before; + uint32 before_len; + + Assert(!IsInParallelMode()); + + /* Capture original length for UNDO before truncation */ + if (stat(path, &st) == 0) + orig_length = st.st_size; + + fd = OpenTransientFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for truncation: %m", path))); + + if (length < 0) + ereport(ERROR, (errmsg("negative file truncation length"))); + before_len = orig_length > length ? + Min(orig_length - length, (off_t) FILEOPS_MAX_DATA_BYTES + 1) : 0; + before = fileops_before_image(fd, path, length, before_len); + + if (ftruncate(fd, length) < 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not truncate file \"%s\" to %lld bytes: %m", + path, (long long) length))); + } + + /* + * At wal_level = minimal there is no UNDO record, so register an + * abort-time pending op that restores the original size. + */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_TRUNCATE, path, NULL, orig_length, false); + + if (enableFsync && pg_fsync(fd) != 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after truncation: %m", + path))); + } + + if (CloseTransientFile(fd) != 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + + if (XLogIsNeeded()) + { + xl_fileops_truncate xlrec; + XactUndoContext undo_ctx; + char *payload = palloc0(sizeof(FileopsUndoTruncate) + pathlen + before_len); + FileopsUndoTruncate *hdr = (FileopsUndoTruncate *) payload; + + xlrec.length = length; + + hdr->before_len = before_len; + hdr->orig_length = orig_length; + hdr->new_length = length; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoTruncate), path, pathlen); + memcpy(payload + sizeof(FileopsUndoTruncate) + pathlen, before, before_len); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_TRUNCATE, + InvalidOid, payload, + sizeof(FileopsUndoTruncate) + pathlen + before_len); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsTruncate); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_TRUNCATE, &undo_ctx); + pfree(payload); + } + pfree(before); +} + +/* + * FileOpsChmod - Change file permissions within a transaction + * + * Immediate execution, WAL-logged. + * On POSIX: chmod(). On Windows: _chmod() with limited mode bits + * (only _S_IREAD/_S_IWRITE; no group/other). Logs WARNING for + * unsupported mode bits on Windows. + * + * Returns 0 on success, -1 on failure with errno left set, so the caller + * can tell ENOENT from other errors and report it as it would a plain + * chmod() failure (tablespace.c does). Nothing is logged or registered + * when it fails, because nothing changed. + */ +int +FileOpsChmod(const char *path, mode_t mode) +{ + struct stat st; + mode_t orig_mode; + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + /* Capture original mode for UNDO before chmod */ + if (stat(path, &st) != 0) + return -1; /* leave errno set by stat() for the caller */ + orig_mode = st.st_mode & 07777; + + if (chmod(path, mode) < 0) + return -1; /* leave errno set by chmod() for the caller */ + + /* + * Register abort-time pending op to restore original mode on rollback. + * Store mode in the length field (mode_t fits in off_t). + */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_CHMOD, path, NULL, (off_t) orig_mode, false); + + if (XLogIsNeeded()) + { + xl_fileops_chmod xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoChmod) + MAXPGPATH]; + FileopsUndoChmod *hdr = (FileopsUndoChmod *) payload; + + xlrec.mode = mode; + xlrec.orig_mode = orig_mode; + + hdr->orig_mode = orig_mode; + hdr->new_mode = mode & 07777; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoChmod), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CHMOD, + InvalidOid, payload, + sizeof(FileopsUndoChmod) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsChmod); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_CHMOD, &undo_ctx); + + INJECTION_POINT("fileops-chmod-after-wal-flush", NULL); + } + + return 0; +} + +/* + * FileOpsChown - Change file ownership within a transaction + * + * Immediate execution, WAL-logged. + * On POSIX: chown(). On Windows: no-op with WARNING (Windows uses + * ACLs for ownership, not uid/gid). + * + * Returns 0; a failed chown() is an ERROR. + */ +int +FileOpsChown(const char *path, uid_t uid, gid_t gid) +{ + struct stat st; + uid_t orig_uid = (uid_t) -1; + gid_t orig_gid = (gid_t) -1; + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + /* Capture original ownership for UNDO before chown */ + if (stat(path, &st) == 0) + { + orig_uid = st.st_uid; + orig_gid = st.st_gid; + } + +#ifndef WIN32 + if (chown(path, uid, gid) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not chown file \"%s\" to %d:%d: %m", + path, (int) uid, (int) gid))); +#else + ereport(WARNING, + (errmsg("chown is not supported on Windows, skipping for \"%s\"", + path))); + return 0; /* nothing changed, so nothing to log */ +#endif + + if (XLogIsNeeded()) + { + xl_fileops_chown xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoChown) + MAXPGPATH]; + FileopsUndoChown *hdr = (FileopsUndoChown *) payload; + + xlrec.uid = uid; + xlrec.gid = gid; + xlrec.orig_uid = orig_uid; + xlrec.orig_gid = orig_gid; + + hdr->orig_uid = orig_uid; + hdr->orig_gid = orig_gid; + /* chown() leaves an id of -1 unchanged */ + hdr->new_uid = (uid == (uid_t) -1) ? orig_uid : uid; + hdr->new_gid = (gid == (gid_t) -1) ? orig_gid : gid; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoChown), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CHOWN, + InvalidOid, payload, + sizeof(FileopsUndoChown) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsChown); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_CHOWN, &undo_ctx); + } + + return 0; +} + +/* + * fileops_mkdir - Create a directory within a transaction + * + * Shared by FileOpsMkdir() and FileOpsMkdirTree(); see there. + */ +static int +fileops_mkdir(const char *path, mode_t mode, uint16 flags, Oid dboid) +{ + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + if (MakePGDirectory(path) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", path))); + + /* Remove the directory (or tree) if the transaction aborts */ + if (!XLogIsNeeded()) + AddPendingFileOp((flags & FILEOPS_MKDIR_TREE) ? + PENDING_FILEOP_RMTREE : PENDING_FILEOP_RMDIR, + path, NULL, 0, false); + + if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + + if (XLogIsNeeded()) + { + xl_fileops_mkdir xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoMkdir) + MAXPGPATH]; + FileopsUndoMkdir *hdr = (FileopsUndoMkdir *) payload; + struct stat st; + + xlrec.mode = mode; + xlrec.flags = flags; + + if (stat(path, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat directory \"%s\": %m", path))); + hdr->dev = st.st_dev; + hdr->ino = st.st_ino; + hdr->dboid = dboid; + hdr->flags = flags; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoMkdir), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_MKDIR, + InvalidOid, payload, + sizeof(FileopsUndoMkdir) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsMkdir); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_MKDIR, &undo_ctx); + } + + return 0; +} + +/* + * FileOpsMkdir - Create a directory within a transaction + * + * Immediate execution; the directory is removed (rmdir) if the transaction + * aborts, so it must then be empty. + * Uses MakePGDirectory() pattern (mkdir with pg_dir_create_mode). + * On Windows: _mkdir() (no mode parameter, permissions from parent). + * + * Returns 0; a failed mkdir() is an ERROR. + */ +int +FileOpsMkdir(const char *path, mode_t mode) +{ + return fileops_mkdir(path, mode, 0, InvalidOid); +} + +/* + * FileOpsMkdirTree - Create a directory the transaction will fill + * + * Like FileOpsMkdir(), but if the transaction aborts the whole tree is + * removed, including files created in it by other means (smgr, copydir()). + * For a database directory pass its OID as dboid: before the tree is removed + * by the UNDO record, that database's shared buffers and pending fsync + * requests are dropped, as they would be by DROP DATABASE. + */ +int +FileOpsMkdirTree(const char *path, mode_t mode, Oid dboid) +{ + return fileops_mkdir(path, mode, FILEOPS_MKDIR_TREE, dboid); +} + +/* + * FileOpsRmdir - Remove a directory within a transaction + * + * Deferred to commit time (like DELETE). Uses rmdir() on all platforms. + * On Windows: _rmdir(). + */ +void +FileOpsRmdir(const char *path, bool at_commit) +{ + Assert(!IsInParallelMode()); + + /* + * WAL is emitted at commit-execution time in FileOpsDoPendingOps(), not + * at registration, so the standby reproduces the rmdir via redo. + */ + AddPendingFileOp(PENDING_FILEOP_RMDIR, path, NULL, 0, at_commit); +} + +/* + * FileOpsSymlink - Create a symbolic link within a transaction + * + * Immediate execution. Registers delete-on-abort for cleanup. + * On POSIX: symlink(). On Windows: pgsymlink() which creates NTFS + * junction points via DeviceIoControl(). Note: junction points only + * work for directories on Windows. + * + * Returns 0; a failed symlink() is an ERROR. + */ +int +FileOpsSymlink(const char *target, const char *linkpath) +{ + int targetlen = fileops_pathlen(target); + int linkpathlen = fileops_pathlen(linkpath); + + Assert(!IsInParallelMode()); + + if (symlink(target, linkpath) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create symbolic link \"%s\" -> \"%s\": %m", + linkpath, target))); + + /* Register delete-on-abort to clean up the symlink on rollback */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_DELETE, linkpath, NULL, 0, false); + + if (enableFsync) + fileops_fsync_parent(linkpath, data_sync_elevel(ERROR)); + + if (XLogIsNeeded()) + { + xl_fileops_symlink xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoSymlink) + 2 * MAXPGPATH]; + FileopsUndoSymlink *hdr = (FileopsUndoSymlink *) payload; + + xlrec.target_len = (uint16) targetlen; + + /* the target lets the UNDO recognize the link it would remove */ + hdr->linkpath_len = (uint16) linkpathlen; + hdr->target_len = (uint16) targetlen; + memcpy(payload + sizeof(FileopsUndoSymlink), linkpath, linkpathlen); + memcpy(payload + sizeof(FileopsUndoSymlink) + linkpathlen, target, + targetlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_SYMLINK, + InvalidOid, payload, + sizeof(FileopsUndoSymlink) + linkpathlen + + targetlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsSymlink); + XLogRegisterData(target, targetlen); + XLogRegisterData(linkpath, linkpathlen); + fileops_log(XLOG_FILEOPS_SYMLINK, &undo_ctx); + } + + return 0; +} + +/* + * FileOpsLink - Create a hard link within a transaction + * + * Immediate execution. Registers delete-on-abort for cleanup. + * On POSIX: link(). On Windows: CreateHardLinkA() (NTFS only). + * + * Returns 0; a failed link() is an ERROR. + */ +int +FileOpsLink(const char *oldpath, const char *newpath) +{ + int oldpathlen = fileops_pathlen(oldpath); + int newpathlen = fileops_pathlen(newpath); + struct stat st; + + Assert(!IsInParallelMode()); + + /* + * The new link will be this inode; the UNDO removes newpath only while it + * still is. + */ + if (stat(oldpath, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", oldpath))); + + if (link(oldpath, newpath) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create hard link \"%s\" -> \"%s\": %m", + newpath, oldpath))); + + /* Register delete-on-abort to clean up the link on rollback */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_DELETE, newpath, NULL, 0, false); + + if (enableFsync) + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + + if (XLogIsNeeded()) + { + xl_fileops_link xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoLink) + MAXPGPATH]; + FileopsUndoLink *hdr = (FileopsUndoLink *) payload; + + xlrec.oldpath_len = (uint16) oldpathlen; + + hdr->dev = st.st_dev; + hdr->ino = st.st_ino; + hdr->newpath_len = (uint16) newpathlen; + memcpy(payload + sizeof(FileopsUndoLink), newpath, newpathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_LINK, + InvalidOid, payload, + sizeof(FileopsUndoLink) + newpathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsLink); + XLogRegisterData(oldpath, oldpathlen); + XLogRegisterData(newpath, newpathlen); + fileops_log(XLOG_FILEOPS_LINK, &undo_ctx); + } + + return 0; +} + +/* + * FileOpsSetXattr - Set an extended attribute on a file + * + * Immediate execution, WAL-logged. + * Uses pg_setxattr() portability layer which handles: + * - Linux: setxattr() + * - macOS: setxattr() with extra options parameter + * - FreeBSD: extattr_set_file() + * - Windows: NTFS Alternate Data Streams + * - Other: returns ENOTSUP + * + * Returns 0 on success. Returns -1 with errno set if the filesystem does + * not support extended attributes (ENOTSUP/EOPNOTSUPP) or refuses this one + * (EPERM/EACCES); any other failure is an ERROR. Nothing is logged or + * registered when it fails, because nothing changed. + */ +int +FileOpsSetXattr(const char *path, const char *name, + const void *value, size_t len) +{ + bool had_value = false; + char *orig_value = NULL; + ssize_t orig_value_len = 0; + int pathlen = fileops_pathlen(path); + ssize_t vlen; + + Assert(!IsInParallelMode()); + + /* + * Capture existing xattr value before overwriting, for the abort-time + * pending op and the UNDO record. If the attribute doesn't exist, + * had_value stays false. + */ + vlen = pg_getxattr(path, name, NULL, 0); + if (vlen > 0) + { + orig_value = (char *) palloc(vlen); + orig_value_len = pg_getxattr(path, name, orig_value, vlen); + if (orig_value_len >= 0) + had_value = true; + else + { + pfree(orig_value); + orig_value = NULL; + orig_value_len = 0; + } + } + else if (vlen == 0) + { + /* Attribute exists but has zero-length value */ + had_value = true; + } + + if (pg_setxattr(path, name, value, len) < 0) + { + int save_errno = errno; + + if (orig_value) + pfree(orig_value); + errno = save_errno; + if (errno == ENOTSUP || errno == EOPNOTSUPP || + errno == EPERM || errno == EACCES) + return -1; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set extended attribute \"%s\" on \"%s\": %m", + name, path))); + } + + /* + * Register abort-time pending op to restore/remove xattr on rollback. If + * had_value: restore original value. If !had_value: remove the xattr. + */ + if (!XLogIsNeeded() && had_value) + AddPendingFileOpWithData(PENDING_FILEOP_SETXATTR, path, name, + 0, orig_value, (size_t) orig_value_len, false); + else if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_REMOVEXATTR, path, name, 0, false); + + if (XLogIsNeeded()) + { + xl_fileops_setxattr xlrec; + int namelen = (int) strlen(name) + 1; + XactUndoContext undo_ctx; + Size payload_len; + char *payload; + FileopsUndoSetxattr *hdr; + + xlrec.path_len = (uint16) pathlen; + xlrec.name_len = (uint16) namelen; + xlrec.value_len = (uint32) len; + xlrec.had_value = had_value; + xlrec.orig_value_len = (uint32) (had_value ? orig_value_len : 0); + + payload_len = sizeof(FileopsUndoSetxattr) + pathlen + namelen + + (had_value ? orig_value_len : 0) + len; + payload = (char *) palloc(payload_len); + hdr = (FileopsUndoSetxattr *) payload; + + hdr->path_len = (uint16) pathlen; + hdr->name_len = (uint16) namelen; + hdr->orig_value_len = (uint32) (had_value ? orig_value_len : 0); + hdr->new_value_len = (uint32) len; + hdr->had_value = had_value; + + memcpy(payload + sizeof(FileopsUndoSetxattr), path, pathlen); + memcpy(payload + sizeof(FileopsUndoSetxattr) + pathlen, name, namelen); + if (had_value && orig_value_len > 0) + memcpy(payload + sizeof(FileopsUndoSetxattr) + pathlen + namelen, + orig_value, orig_value_len); + if (len > 0) + memcpy(payload + sizeof(FileopsUndoSetxattr) + pathlen + namelen + + hdr->orig_value_len, value, len); + + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_SETXATTR, + InvalidOid, payload, payload_len); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsSetxattr); + XLogRegisterData(path, pathlen); + XLogRegisterData(name, namelen); + XLogRegisterData(value, (uint32) len); + if (had_value && orig_value_len > 0) + XLogRegisterData(orig_value, (uint32) orig_value_len); + fileops_log(XLOG_FILEOPS_SETXATTR, &undo_ctx); + pfree(payload); + } + + if (orig_value) + pfree(orig_value); + + return 0; +} + +/* + * FileOpsRemoveXattr - Remove an extended attribute from a file + * + * Immediate execution, WAL-logged. + * Uses pg_removexattr() portability layer. Removing an attribute that does + * not exist (PG_ENOATTR) succeeds. + * + * Returns 0 on success, -1 if extended attributes are not supported on + * this filesystem/platform (errno left as ENOTSUP/EOPNOTSUPP); any other + * failure is an ERROR. Nothing is logged or registered when it fails. + */ +int +FileOpsRemoveXattr(const char *path, const char *name) +{ + char *saved_value = NULL; + ssize_t saved_value_len = 0; + int pathlen = fileops_pathlen(path); + ssize_t vlen; + + Assert(!IsInParallelMode()); + + /* + * Capture existing xattr value before removal, to restore it on rollback. + * A NULL saved_value means there was none. + */ + vlen = pg_getxattr(path, name, NULL, 0); + if (vlen > 0) + { + saved_value = (char *) palloc(vlen); + saved_value_len = pg_getxattr(path, name, saved_value, vlen); + if (saved_value_len < 0) + { + pfree(saved_value); + saved_value = NULL; + saved_value_len = 0; + } + } + else if (vlen == 0) + { + /* Attribute exists but has zero-length value */ + saved_value_len = 0; + saved_value = (char *) palloc(1); /* non-NULL marker */ + } + + if (pg_removexattr(path, name) < 0 && errno != PG_ENOATTR) + { + int save_errno = errno; + + if (saved_value) + pfree(saved_value); + errno = save_errno; + if (errno == ENOTSUP || errno == EOPNOTSUPP) + return -1; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\": %m", + name, path))); + } + + /* + * Register abort-time pending op to restore the removed xattr on + * rollback. Use SETXATTR type: on abort, set the xattr back to its saved + * value. + */ + if (!XLogIsNeeded() && saved_value != NULL) + AddPendingFileOpWithData(PENDING_FILEOP_SETXATTR, path, name, + 0, saved_value, (size_t) saved_value_len, false); + + if (XLogIsNeeded()) + { + xl_fileops_removexattr xlrec; + int namelen = (int) strlen(name) + 1; + XactUndoContext undo_ctx; + char *payload = NULL; + bool had_value = (saved_value != NULL); + + xlrec.path_len = (uint16) pathlen; + xlrec.name_len = (uint16) namelen; + xlrec.had_value = had_value; + xlrec.orig_value_len = (uint32) (had_value ? saved_value_len : 0); + + if (had_value) + { + Size payload_len; + FileopsUndoRemovexattr *hdr; + + payload_len = sizeof(FileopsUndoRemovexattr) + pathlen + namelen + + saved_value_len; + payload = (char *) palloc(payload_len); + hdr = (FileopsUndoRemovexattr *) payload; + + hdr->path_len = (uint16) pathlen; + hdr->name_len = (uint16) namelen; + hdr->value_len = (uint32) saved_value_len; + + memcpy(payload + sizeof(FileopsUndoRemovexattr), path, pathlen); + memcpy(payload + sizeof(FileopsUndoRemovexattr) + pathlen, + name, namelen); + if (saved_value_len > 0) + memcpy(payload + sizeof(FileopsUndoRemovexattr) + pathlen + namelen, + saved_value, saved_value_len); + + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_REMOVEXATTR, + InvalidOid, payload, payload_len); + } + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRemovexattr); + XLogRegisterData(path, pathlen); + XLogRegisterData(name, namelen); + if (had_value && saved_value_len > 0) + XLogRegisterData(saved_value, (uint32) saved_value_len); + fileops_log(XLOG_FILEOPS_REMOVEXATTR, had_value ? &undo_ctx : NULL); + if (payload) + pfree(payload); + } + + if (saved_value) + pfree(saved_value); + + return 0; +} + +/* + * FileOpsDoPendingOps - Execute pending file operations at transaction end + * + * At commit, operations with at_commit=true are executed. + * At abort, operations with at_commit=false are executed. + * + * This is called from xact.c at transaction commit/abort, analogous + * to smgrDoPendingDeletes(). + */ +void +FileOpsDoPendingOps(bool isCommit) +{ + int nestLevel = GetCurrentTransactionNestLevel(); + PendingFileOp *pending; + PendingFileOp *prev; + PendingFileOp *next; + PendingFileOp *exec_head = NULL; + PendingFileOp *exec_tail = NULL; + XLogRecPtr max_lsn = InvalidXLogRecPtr; + + /* + * Pass 1: detach the entries belonging to this (or an inner) nesting + * level from the pending list. Those matching the transaction outcome + * are queued for execution; the rest are discarded. Outer-level entries + * stay on the list for a later call. The queue is built in pending-list + * order, which is newest-first because AddPendingFileOp prepends. + */ + prev = NULL; + for (pending = pendingFileOps; pending != NULL; pending = next) + { + next = pending->next; + + if (pending->nestLevel < nestLevel) + { + /* outer-level entries should not be processed yet */ + prev = pending; + continue; + } + + /* unlink from list first, so we don't retry on failure */ + if (prev) + prev->next = next; + else + pendingFileOps = next; + + if (pending->at_commit == isCommit) + { + /* + * queue for execution, preserving pending-list (newest-first) + * order + */ + pending->next = NULL; + if (exec_tail != NULL) + exec_tail->next = pending; + else + exec_head = pending; + exec_tail = pending; + } + else + FreePendingFileOp(pending); + } + + /* + * Commit-time ops must run in registration order (oldest-first) so that + * nested directory removals proceed deepest-first. The pending list is + * LIFO, so the queue collected above is newest-first; reverse it for + * commit. Abort-time ops must keep LIFO order, which the collected queue + * already preserves, so they are left as-is. + */ + if (isCommit) + { + PendingFileOp *rev = NULL; + + while (exec_head != NULL) + { + next = exec_head->next; + exec_head->next = rev; + rev = exec_head; + exec_head = next; + } + exec_head = rev; + } + + /* Commit record written, nothing executed yet (for tests) */ + if (isCommit && exec_head != NULL) + INJECTION_POINT("fileops-commit-before-pending-ops", NULL); + + /* Pass 2: execute the queued ops in order, freeing each when done. */ + for (pending = exec_head; pending != NULL; pending = next) + { + next = pending->next; + + FileOpsExecOne(pending, isCommit, &max_lsn); + + FreePendingFileOp(pending); + } + + /* + * Each destructive commit op above XLogFlush()es its own record BEFORE + * performing the filesystem mutation (WAL-before-mutation), so no primary + * FS change can outrun its WAL record. This trailing flush is therefore + * redundant, kept only as a no-op safety backstop (max_lsn is already + * durable). Only the commit path ever sets max_lsn. + */ + if (!XLogRecPtrIsInvalid(max_lsn)) + XLogFlush(max_lsn); +} + +/* + * FileOpsExecOne - Execute one pending file operation + * + * Performs a single deferred op (commit- or abort-time) exactly as it would + * run in an ordinary transaction end. Shared by FileOpsDoPendingOps() and by + * the 2PC post-commit/post-abort callbacks, so an op prepared with PREPARE + * TRANSACTION runs identically to one that never left this backend. + * + * *max_lsn accumulates the highest WAL LSN written by a commit-time op so the + * caller can flush it. Each destructive commit op also flushes its own record + * before mutating the filesystem (WAL-before-mutation), so *max_lsn is only a + * backstop. + */ +static void +FileOpsExecOne(PendingFileOp *pending, bool isCommit, XLogRecPtr *max_lsn) +{ + { + switch (pending->type) + { + case PENDING_FILEOP_INVERSE: + break; + case PENDING_FILEOP_CREATE: + /* Creates are executed immediately, nothing to do */ + break; + + case PENDING_FILEOP_RENAME: + + /* + * Abort-time rename back of a FileOpsRename() at wal_level = + * minimal: path is the new name, newpath the old one. Only + * while the new name exists and the old one is free, as the + * UNDO does at higher wal_level. + */ + Assert(!pending->at_commit); + { + struct stat st; + + if (lstat(pending->path, &st) != 0 || + lstat(pending->newpath, &st) == 0) + break; + } + if (rename(pending->path, pending->newpath) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" back to \"%s\": %m", + pending->path, pending->newpath))); + else if (enableFsync) + { + fileops_fsync_parent(pending->newpath, WARNING); + fileops_fsync_parent(pending->path, WARNING); + } + break; + + case PENDING_FILEOP_DELETE: + + /* + * log-at-commit-execution, but ONLY for the commit path. A + * delete-on-abort registration (at_commit=false) executes + * here during abort; that must NOT be WAL-logged, because a + * standby must never apply an aborted transaction's + * filesystem effect. + */ + if (isCommit && XLogIsNeeded()) + { + xl_fileops_delete xlrec; + int pathlen = (int) strlen(pending->path) + 1; + + xlrec.at_commit = true; + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsDelete); + XLogRegisterData(pending->path, pathlen); + *max_lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_DELETE); + + /* + * WAL-before-mutation durability: flush this record + * BEFORE touching the filesystem. A crash after the + * unlink but before a batched flush would mutate the + * primary's FS without the record ever reaching the + * standby, diverging the two. + */ + XLogFlush(*max_lsn); + } + if (unlink(pending->path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\": %m", + pending->path))); + } + else + { + if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + } + break; + + case PENDING_FILEOP_RMDIR: + + /* + * log-at-commit-execution, commit path only (see the DELETE + * case above for the abort-path rationale). + */ + if (isCommit && XLogIsNeeded()) + { + xl_fileops_rmdir xlrec; + int pathlen = (int) strlen(pending->path) + 1; + + xlrec.at_commit = true; + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRmdir); + XLogRegisterData(pending->path, pathlen); + *max_lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_RMDIR); + + /* WAL-before-mutation durability (see DELETE above). */ + XLogFlush(*max_lsn); + } + if (rmdir(pending->path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + pending->path))); + } + else + { + if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + } + break; + + case PENDING_FILEOP_RMTREE: + + /* + * Abort-time removal of a FileOpsMkdirTree() directory and + * whatever ended up in it. The UNDO record, applied earlier + * in the abort, or the caller's own error cleanup has usually + * removed it already. + */ + { + struct stat st; + + if (lstat(pending->path, &st) != 0 || !S_ISDIR(st.st_mode)) + break; + } + if (!rmtree(pending->path, true)) + ereport(WARNING, + (errmsg("could not remove directory tree \"%s\"", + pending->path))); + else if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + break; + + case PENDING_FILEOP_TRUNCATE: + { + /* + * Restore original file size on abort. The original + * length is stored in pending->length. + */ + int trunc_fd; + + trunc_fd = OpenTransientFile(pending->path, + O_RDWR | PG_BINARY); + if (trunc_fd >= 0) + { + if (ftruncate(trunc_fd, pending->length) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore file \"%s\" to original size %lld: %m", + pending->path, + (long long) pending->length))); + else if (enableFsync && pg_fsync(trunc_fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after restoring size: %m", + pending->path))); + CloseTransientFile(trunc_fd); + } + else + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" to restore size: %m", + pending->path))); + } + } + break; + + case PENDING_FILEOP_CHMOD: + { + /* + * Restore original file mode on abort. The original mode + * is stored in pending->length (cast from mode_t). + */ + mode_t restore_mode = (mode_t) pending->length; + + if (chmod(pending->path, restore_mode) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore file \"%s\" to original mode 0%o: %m", + pending->path, + (unsigned int) restore_mode))); + } + } + break; + + case PENDING_FILEOP_SETXATTR: + { + /* + * Restore an xattr to its original value on abort. path = + * file, newpath = attr name, data = orig value. + */ + if (pending->data && pending->data_len > 0) + { + if (pg_setxattr(pending->path, pending->newpath, + pending->data, pending->data_len) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore extended attribute \"%s\" on \"%s\": %m", + pending->newpath, + pending->path))); + } + } + else + { + /* Zero-length original value */ + if (pg_setxattr(pending->path, pending->newpath, + "", 0) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore extended attribute \"%s\" on \"%s\": %m", + pending->newpath, + pending->path))); + } + } + } + break; + + case PENDING_FILEOP_REMOVEXATTR: + { + /* + * Remove an xattr on abort (attribute was newly created + * in the aborted transaction). + */ + if (pg_removexattr(pending->path, pending->newpath) < 0) + { + /* + * "Attribute already absent" is success for a + * remove-on-abort (idempotent, like our unlink/rmdir + * redo). Use PG_ENOATTR, the portable + * absent-attribute errno: on Linux that is ENODATA, + * on macOS/BSD ENOATTR -- the raw ENODATA/ENOENT + * check missed the macOS case and warned spuriously + * (sec3 portability bug). + */ + if (errno != ENOENT && errno != PG_ENOATTR) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\": %m", + pending->newpath, + pending->path))); + } + } + break; + + default: + break; + } + } +} + +/* + * AtSubCommit_FileOps - Handle subtransaction commit + * + * Reassign all pending ops from the current nesting level to the parent. + */ +void +AtSubCommit_FileOps(void) +{ + int nestLevel = GetCurrentTransactionNestLevel(); + PendingFileOp *pending; + + for (pending = pendingFileOps; pending != NULL; pending = pending->next) + { + if (pending->nestLevel >= nestLevel) + pending->nestLevel = nestLevel - 1; + } +} + +/* + * AtSubAbort_FileOps - Handle subtransaction abort + * + * Execute abort-time actions for the current nesting level immediately. + */ +void +AtSubAbort_FileOps(void) +{ + FileOpsDoPendingOps(false); +} + +/* + * PostPrepare_FileOps - Clean up after PREPARE TRANSACTION + * + * FileOpsAtPrepare() (called from EndPrepare, before the 2PC END sentinel) + * has already serialized every pending file operation into the two-phase + * state file, so this backend can now discard its in-memory list. The ops + * are performed later by the FILEOPS 2PC post-commit/post-abort callbacks + * (fileops_twophase_postcommit/postabort), possibly from a different backend + * after a crash, driven from the durable 2PC state. + */ +void +PostPrepare_FileOps(void) +{ + PendingFileOp *pending; + PendingFileOp *next; + + for (pending = pendingFileOps; pending != NULL; pending = next) + { + next = pending->next; + pendingFileOps = next; + FreePendingFileOp(pending); + } +} + +/* + * Serialized form of a PendingFileOp for the 2PC state file. + * + * A fixed header followed by two variable-length blobs (path, data), in that + * order. Self-describing so a crash-recovered COMMIT/ROLLBACK + * PREPARED -- run by any backend -- can rebuild an equivalent PendingFileOp + * and hand it to FileOpsExecOne(). Only the fields FileOpsExecOne() reads + * are carried; nestLevel is irrelevant once prepared. + */ +typedef struct FileOpsTwoPhaseRecord +{ + int type; /* PendingFileOpType */ + bool at_commit; /* run at COMMIT PREPARED vs ROLLBACK PREPARED */ + off_t length; /* truncate length / write offset / mode */ + uint32 path_len; /* bytes, including NUL, 0 if none */ + uint32 data_len; /* bytes, 0 if none */ + /* followed by path, data */ +} FileOpsTwoPhaseRecord; + +/* + * FileOpsAtPrepare - Serialize deferred commit-time file ops into 2PC state + * + * Called from EndPrepare() between StartPrepare() and the END sentinel. Each + * pending at_commit=true op (DELETE/RMDIR) is written as one + * TWOPHASE_RM_FILEOPS_ID record. This is what makes those deferred + * commit-time ops survive PREPARE for COMMIT PREPARED across a crash between + * PREPARE and the final decision. + * + * The immediate-operation inverse list is serialized separately. ROLLBACK + * PREPARED includes it in its abort record and applies it before releasing + * the prepared transaction's locks. + */ +void +FileOpsAtPrepare(void) +{ + PendingFileOp *pending; + char *data; + uint32 len; + + FileOpsGetAbortData(&data, &len); + if (len) + RegisterTwoPhaseRecord(TWOPHASE_RM_FILEOPS_ID, + FILEOPS_TWOPHASE_INVERSES, data, len); + if (data) + pfree(data); + + + for (pending = pendingFileOps; pending != NULL; pending = pending->next) + { + FileOpsTwoPhaseRecord rec; + char *buf; + uint32 path_len; + uint32 data_len; + uint32 total; + char *p; + + /* Immediate inverses were serialized as one blob above. */ + if (!pending->at_commit) + continue; + + path_len = pending->path ? (uint32) strlen(pending->path) + 1 : 0; + /* No commit-time op has a newpath. */ + Assert(pending->newpath == NULL); + data_len = (pending->data && pending->data_len > 0) ? + (uint32) pending->data_len : 0; + + rec.type = (int) pending->type; + rec.at_commit = pending->at_commit; + rec.length = pending->length; + rec.path_len = path_len; + rec.data_len = data_len; + + total = sizeof(FileOpsTwoPhaseRecord) + path_len + data_len; + buf = palloc(total); + memcpy(buf, &rec, sizeof(FileOpsTwoPhaseRecord)); + p = buf + sizeof(FileOpsTwoPhaseRecord); + if (path_len > 0) + { + memcpy(p, pending->path, path_len); + p += path_len; + } + if (data_len > 0) + memcpy(p, pending->data, data_len); + + RegisterTwoPhaseRecord(TWOPHASE_RM_FILEOPS_ID, 0, buf, total); + pfree(buf); + } +} + +/* + * fileops_twophase_do - Shared body for the FILEOPS 2PC callbacks + * + * Rebuild a PendingFileOp from the serialized 2PC record and, if it matches + * the transaction outcome, perform it exactly as FileOpsDoPendingOps() would. + * A commit-time op emits its own WAL (idempotent redo on a standby); the + * trailing flush makes any accumulated LSN durable. + */ +static void +fileops_twophase_do(bool isCommit, void *recdata, uint32 len) +{ + FileOpsTwoPhaseRecord rec; + PendingFileOp pending; + const char *p; + XLogRecPtr max_lsn = InvalidXLogRecPtr; + + if (len < sizeof(FileOpsTwoPhaseRecord)) + { + elog(WARNING, "FILEOPS 2PC record too short (%u bytes)", len); + return; + } + memcpy(&rec, recdata, sizeof(FileOpsTwoPhaseRecord)); + + /* Only ops matching this outcome run; the rest are simply discarded. */ + if (rec.at_commit != isCommit) + return; + + memset(&pending, 0, sizeof(pending)); + pending.type = (PendingFileOpType) rec.type; + pending.at_commit = rec.at_commit; + pending.length = rec.length; + + p = (const char *) recdata + sizeof(FileOpsTwoPhaseRecord); + if (rec.path_len > 0) + { + pending.path = (char *) p; + p += rec.path_len; + } + if (rec.data_len > 0) + { + pending.data = (void *) p; + pending.data_len = rec.data_len; + } + + FileOpsExecOne(&pending, isCommit, &max_lsn); + + if (!XLogRecPtrIsInvalid(max_lsn)) + XLogFlush(max_lsn); +} + +/* + * fileops_twophase_postcommit - COMMIT PREPARED callback (perform at_commit ops) + */ +void +fileops_twophase_postcommit(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len) +{ + if (info != FILEOPS_TWOPHASE_INVERSES) + fileops_twophase_do(true, recdata, len); +} + +/* + * fileops_twophase_postabort - ROLLBACK PREPARED callback + * + * Immediate inverses are applied with the abort decision, not here. Deferred + * commit-time operations are discarded. + */ +void +fileops_twophase_postabort(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len) +{ + /* Inverses were applied with the durable abort decision, before unlock. */ + if (info != FILEOPS_TWOPHASE_INVERSES) + fileops_twophase_do(false, recdata, len); +} + +/* + * fileops_redo - WAL redo function for FILEOPS records + * + * Replay file operations during crash recovery or standby apply. + * Each operation type has its own redo handler added in separate commits. + */ +void +fileops_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + char *data = XLogRecGetData(record); + + switch (info) + { + case XLOG_FILEOPS_CREATE: + { + xl_fileops_create *xlrec = (xl_fileops_create *) data; + const char *path = data + SizeOfFileOpsCreate; + int fd; + + fd = BasicOpenFilePerm(path, + (xlrec->flags & ~PG_O_DIRECT) | O_CREAT, + xlrec->mode); + if (fd < 0) + { + if (errno == ENOENT) + { + char parentpath[MAXPGPATH]; + char *sep; + + strlcpy(parentpath, path, MAXPGPATH); + sep = strrchr(parentpath, '/'); + if (sep != NULL) + { + *sep = '\0'; + if (MakePGDirectory(parentpath) < 0 && + errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\" during WAL replay: %m", + parentpath))); + } + + fd = BasicOpenFilePerm(path, + (xlrec->flags & ~PG_O_DIRECT) | O_CREAT, + xlrec->mode); + } + + if (fd < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create file \"%s\" during WAL replay: %m", + path))); + } + + if (fd >= 0) + { + if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + } + break; + + case XLOG_FILEOPS_WRITE: + { + xl_fileops_write *xlrec = (xl_fileops_write *) data; + const char *path = data + SizeOfFileOpsWrite; + const char *wdata = path + xlrec->path_len; + int fd; + + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for write during WAL replay: %m", + path))); + } + else + { + if (pg_pwrite(fd, wdata, xlrec->len, xlrec->offset) != + (ssize_t) xlrec->len) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\" during WAL replay: %m", + path))); + else if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + } + } + break; + + case XLOG_FILEOPS_RENAME: + + /* + * Redo of FileOpsRename(), logged before the rename was made. + * Converge on the state the primary reached, whatever part of it + * is on disk already: + * + * - oldpath absent: the rename was made already. + * + * - newpath absent: make it. + * + * - both present: the replay of earlier records (the CREATE, + * MKDIR or LINK of oldpath since the checkpoint) has made oldpath + * again, or a later operation has; newpath, which the primary + * never let the rename replace, is the later state. Remove + * oldpath, a directory only if empty, and keep newpath. Renaming + * over newpath would lose data: after a -> b, b -> c and a new b, + * with a older than the checkpoint, redo of b -> c finds the new + * b and would put it in place of c. A non-empty oldpath + * directory is left, with a WARNING. + * + * Redo replays onto the current, possibly later, filesystem state + * and cannot tell an object made again by redo from one made at + * the same path later (see README). + */ + { + xl_fileops_rename *xlrec = (xl_fileops_rename *) data; + const char *oldpath = data + SizeOfFileOpsRename; + const char *newpath = oldpath + xlrec->oldpath_len; + struct stat st; + struct stat nst; + + if (lstat(oldpath, &st) != 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\" during WAL replay: %m", + oldpath))); + } + else if (lstat(newpath, &nst) != 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\" during WAL replay: %m", + newpath))); + else if (rename(oldpath, newpath) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\" during WAL replay: %m", + oldpath, newpath))); + else if (enableFsync) + { + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + fileops_fsync_parent(oldpath, data_sync_elevel(ERROR)); + } + } + else if ((S_ISDIR(st.st_mode) ? rmdir(oldpath) : + unlink(oldpath)) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove \"%s\", renamed to \"%s\", during WAL replay: %m", + oldpath, newpath))); + else if (enableFsync) + fileops_fsync_parent(oldpath, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_DELETE: + + /* + * idempotent redo of a deferred delete executed at commit on the + * primary. An already-absent file (ENOENT) means the delete + * already took effect -- treat as success. Only commit-path + * deletes are ever logged (see FileOpsDoPendingOps), so redo of + * an aborted transaction's delete-on-abort can never occur here. + */ + { + xl_fileops_delete *xlrec pg_attribute_unused() = + (xl_fileops_delete *) data; + const char *path = data + SizeOfFileOpsDelete; + + if (unlink(path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\" during WAL replay: %m", + path))); + } + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_SYMLINK: + + /* + * sec2: intentional no-op during redo. + * + * FileOpsSymlink() has exactly one caller, + * create_tablespace_directories() in tablespace.c, and it is used + * ONLY for tablespace symlinks under $PGDATA/pg_tblspc. Standby + * (and primary crash-recovery) replay of those symlinks is OWNED + * by RM_TBLSPC: tblspc_redo() -> create_tablespace_directories() + * runs with InRecovery set, first removing any stale symlink + * (remove_tablespace_symlink) and then re-creating it with the + * recovery-time layout logic that honors the standby's own + * tablespace layout / allow_in_place_tablespaces / tablespace_map + * remap. + * + * Replaying the FILEOPS record here would be redundant (RM_TBLSPC + * already creates the link) and, worse, actively wrong on a + * remapped standby: it would unlink the correctly-remapped link + * and re-point it at the primary's LITERAL absolute target, which + * may not exist or may belong to a different location on the + * standby. So FILEOPS defers tablespace-symlink replay to + * RM_TBLSPC and does nothing here. + * + * If FileOpsSymlink ever gains a non-tablespace caller, this must + * be revisited to replay those (and only those) symlinks; that + * would require distinguishing the two cases in the record. + */ + break; + + case XLOG_FILEOPS_LINK: + { + xl_fileops_link *xlrec = (xl_fileops_link *) data; + const char *oldpath = data + SizeOfFileOpsLink; + const char *newpath = oldpath + xlrec->oldpath_len; + + /* Remove existing link first for idempotent redo */ + unlink(newpath); + if (link(oldpath, newpath) < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create hard link \"%s\" during WAL replay: %m", + newpath))); + else if (enableFsync) + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_MKDIR: + { + xl_fileops_mkdir *xlrec pg_attribute_unused() = + (xl_fileops_mkdir *) data; + const char *path = data + SizeOfFileOpsMkdir; + + if (MakePGDirectory(path) < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\" during WAL replay: %m", + path))); + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_RMDIR: + + /* + * idempotent redo of a deferred rmdir executed at commit on the + * primary. An already-absent directory (ENOENT) means the rmdir + * already took effect -- treat as success. + */ + { + xl_fileops_rmdir *xlrec pg_attribute_unused() = + (xl_fileops_rmdir *) data; + const char *path = data + SizeOfFileOpsRmdir; + + if (rmdir(path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\" during WAL replay: %m", + path))); + } + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_CHMOD: + { + xl_fileops_chmod *xlrec = (xl_fileops_chmod *) data; + const char *path = data + SizeOfFileOpsChmod; + + if (chmod(path, xlrec->mode) < 0 && errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not chmod file \"%s\" during WAL replay: %m", + path))); + } + break; + + case XLOG_FILEOPS_CHOWN: + { +#ifndef WIN32 + xl_fileops_chown *xlrec = (xl_fileops_chown *) data; + const char *path = data + SizeOfFileOpsChown; + + if (chown(path, xlrec->uid, xlrec->gid) < 0 && + errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not chown file \"%s\" during WAL replay: %m", + path))); +#endif + } + break; + + case XLOG_FILEOPS_TRUNCATE: + { + xl_fileops_truncate *xlrec = (xl_fileops_truncate *) data; + const char *path = data + SizeOfFileOpsTruncate; + int fd; + + XLogFlush(record->EndRecPtr); + + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for truncation during WAL replay: %m", + path))); + } + else + { + if (ftruncate(fd, xlrec->length) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not truncate file \"%s\" to %lld bytes during WAL replay: %m", + path, (long long) xlrec->length))); + else if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + } + } + break; + + case XLOG_FILEOPS_SETXATTR: + { + xl_fileops_setxattr *xlrec = (xl_fileops_setxattr *) data; + const char *path = data + SizeOfFileOpsSetxattr; + const char *name = path + xlrec->path_len; + const void *value = name + xlrec->name_len; + + if (pg_setxattr(path, name, value, xlrec->value_len) < 0 && + errno != ENOTSUP && errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not set extended attribute \"%s\" on \"%s\" during WAL replay: %m", + name, path))); + } + break; + + case XLOG_FILEOPS_REMOVEXATTR: + { + xl_fileops_removexattr *xlrec = + (xl_fileops_removexattr *) data; + const char *path = data + SizeOfFileOpsRemovexattr; + const char *name = path + xlrec->path_len; + + if (pg_removexattr(path, name) < 0 && + errno != ENOTSUP && errno != ENOENT && + errno != PG_ENOATTR) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\" during WAL replay: %m", + name, path))); + } + break; + + default: + elog(PANIC, "fileops_redo: unknown op code %u", info); + break; + } +} diff --git a/src/backend/storage/file/fileops_undo.c b/src/backend/storage/file/fileops_undo.c new file mode 100644 index 0000000000000..018a6ff52b6d6 --- /dev/null +++ b/src/backend/storage/file/fileops_undo.c @@ -0,0 +1,1065 @@ +/*------------------------------------------------------------------------- + * + * fileops_undo.c + * FILEOPS UNDO resource manager + * + * This module implements the UNDO apply callbacks for transactional file + * operations. The inverse payloads cover structural changes and bounded + * before-images for overwritten or truncated file contents. + * + * On transaction abort, each FILEOPS UNDO record reverses the structural + * change made by the corresponding FileOps* function, if the object still + * shows that change (see fileops_undo_apply): + * - CREATE: unlink the created file + * - WRITE: restore overwritten bytes and original length + * - TRUNCATE: restore removed bytes and original length + * - CHMOD: chmod to original mode + * - CHOWN: chown to original uid/gid + * - MKDIR: rmdir the created directory, or remove its whole tree + * - SYMLINK: unlink the symlink + * - LINK: unlink the hard link + * - RENAME: rename the new name back to the old one + * - SETXATTR: restore/remove the original xattr + * - REMOVEXATTR: restore the original xattr value + * + * DELETE and RMDIR are handled by PendingFileOps (deferred execution) and do + * not need UNDO records since they execute only at commit time. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/file/fileops_undo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include + +#include "access/undo_xlog.h" +#include "access/undormgr.h" +#include "access/xact.h" +#include "access/transam.h" +#include "access/xlog.h" +#include "access/xlogutils.h" +#include "port/pg_xattr.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "storage/md.h" +#include "utils/hsearch.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" + +/* + * UNDO subtype constants and payload structures are defined in fileops.h, + * shared between this file (apply side) and fileops.c (insert side). + */ + +static bool applyingAbort = false; +bool FileOpsSkipUndo = false; + +/* + * Recovery: redo, and the undo phase that follows it. An object an UNDO + * record describes may have been created again by redo, with another inode. + */ +#define FILEOPS_RECOVERING() (InRecovery || UndoRecoveryRollingBack) + +/* Forward declarations */ +static void fileops_note_inverse(uint16 info, const char *data, uint32 len, + int elevel); +static UndoApplyResult fileops_undo_apply(uint8 rmid, uint16 info, + TransactionId xid, Oid reloid, + const char *payload, Size payload_len, + UndoRecPtr urec_ptr); +static void fileops_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len); + +/* + * Report an xattr restore that failed, and return UNDO_APPLY_ERROR so the + * caller counts it as a record that could not be applied. Like the other + * subtypes, a file that no longer exists is not a failure. + */ +static UndoApplyResult +fileops_undo_xattr_failed(const char *op, const char *path, const char *name) +{ + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO %s: could not restore attribute \"%s\" of \"%s\": %m", + op, name, path))); + return UNDO_APPLY_ERROR; +} + +/* The FILEOPS UNDO RM registration entry */ +static const UndoRmgrData fileops_undo_rmgr = { + .rm_name = "fileops", + .rm_undo = fileops_undo_apply, + .rm_desc = fileops_undo_desc, +}; + +/* + * FileopsUndoRmgrInit - Register the FILEOPS UNDO resource manager + */ +void +FileopsUndoRmgrInit(void) +{ + RegisterUndoRmgr(UNDO_RMID_FILEOPS, &fileops_undo_rmgr); + + /* + * Wire FILEOPS's pending-structural-file-operation callbacks into the + * AM-neutral hooks in xact.c (see access/xact.h for why xact.c cannot + * call these by name directly). + */ + PendingPhysOpsDo_hook = FileOpsDoPendingOps; + PendingPhysOpsPostPrepare_hook = PostPrepare_FileOps; + PendingPhysOpsAtSubCommit_hook = AtSubCommit_FileOps; + PendingPhysOpsAtSubAbort_hook = AtSubAbort_FileOps; +} + +/* + * fileops_undo_lstat - lstat() for an UNDO action + * + * Returns 1 if path exists, 0 if it does not (nothing left to undo), and -1 + * after a WARNING for any other failure. + */ +static int +fileops_undo_lstat(const char *op, const char *path, struct stat *st) +{ + if (lstat(path, st) == 0) + return 1; + if (errno == ENOENT) + return 0; + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO %s: could not stat \"%s\": %m", op, path))); + return -1; +} + +/* + * fileops_undo_superseded - Report an object that no longer shows the effect + * being undone: it was already reverted by an earlier walk of this chain, or + * changed since. Either way it is not ours to touch. + */ +static UndoApplyResult +fileops_undo_superseded(const char *op, const char *path) +{ + ereport(DEBUG2, + (errmsg("FILEOPS UNDO %s: \"%s\" was changed since, leaving it", + op, path))); + return UNDO_APPLY_SUCCESS; +} + +/* + * fileops_xattr_is - Does attribute name of path hold exactly value? + * + * Returns 1 if so, 0 if not (or the attribute is absent), and -1 with errno + * set if it cannot be read. + */ +static int +fileops_xattr_is(const char *path, const char *name, const char *value, + uint32 len) +{ + ssize_t cur_len; + char *cur; + int result; + + cur_len = pg_getxattr(path, name, NULL, 0); + if (cur_len < 0) + return (errno == PG_ENOATTR) ? 0 : -1; + if ((size_t) cur_len != len) + return 0; + if (len == 0) + return 1; + + cur = palloc(len); + cur_len = pg_getxattr(path, name, cur, len); + if (cur_len < 0) + result = (errno == PG_ENOATTR || errno == ERANGE) ? 0 : -1; + else + result = ((size_t) cur_len == len && memcmp(cur, value, len) == 0); + pfree(cur); + return result; +} + +/* + * fileops_undo_apply - Apply a single FILEOPS UNDO record + * + * Reverses the structural filesystem operation, but only if the object still + * shows its effect, as recorded in the payload. The same chain can be walked + * again later (see storage/fileops.h), when the object may already have been + * reverted, or changed by someone else; then the record is a no-op. A CREATE + * or MKDIR cannot tell its own file or directory from one created again at + * the same path, since the inode is not known when the UNDO is written, and + * a REMOVEXATTR cannot tell an attribute it removed from one removed again; + * nor can a RENAME tell the object it renamed from one put at the new name + * since. + */ +static UndoApplyResult +fileops_undo_apply(uint8 rmid, uint16 info, TransactionId xid, Oid reloid, + const char *payload, Size payload_len, UndoRecPtr urec_ptr) +{ + UndoApplyResult result = UNDO_APPLY_SUCCESS; + struct stat st; + + Assert(rmid == UNDO_RMID_FILEOPS); + + /* Abort WAL owns completed aborts; generic UNDO is for crash losers. */ + if (!applyingAbort && FileOpsSkipUndo) + return UNDO_APPLY_SUCCESS; + + INJECTION_POINT("fileops-undo-apply-begin", NULL); + + switch (info) + { + case FILEOPS_UNDO_CREATE: + { + FileopsUndoCreate hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoCreate)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoCreate)); + path = payload + sizeof(FileopsUndoCreate); + + switch (fileops_undo_lstat("CREATE", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!S_ISREG(st.st_mode)) + result = fileops_undo_superseded("CREATE", path); + else if (unlink(path) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CREATE: could not unlink \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_WRITE: + { + FileopsUndoWrite hdr; + const char *path; + const char *before; + const char *after; + char *current; + int fd; + int rc; + + if (payload_len < sizeof(hdr)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(hdr)); + if (payload_len != sizeof(hdr) + hdr.path_len + + hdr.before_len + hdr.after_len) + return UNDO_APPLY_ERROR; + path = payload + sizeof(hdr); + before = path + hdr.path_len; + after = before + hdr.before_len; + + /* + * 0 (gone) is nothing to undo, but -1 (a stat error) must + * fail like every other subtype, so a crash loser's + * overwritten bytes are reported as not restored rather than + * silently dropped. + */ + rc = fileops_undo_lstat("WRITE", path, &st); +#ifdef USE_INJECTION_POINTS + /* Lets a test reach the stat-error (-1) arm. */ + if (rc == 1 && + IS_INJECTION_POINT_ATTACHED("fileops-undo-write-lstat-error")) + rc = -1; +#endif + if (rc == -1) + result = UNDO_APPLY_ERROR; + if (rc != 1) + break; + if (!S_ISREG(st.st_mode) || st.st_size != hdr.new_length) + break; + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO WRITE: could not open \"%s\": %m", path))); + return UNDO_APPLY_ERROR; + } + current = palloc(hdr.after_len ? hdr.after_len : 1); + if (pg_pread(fd, current, hdr.after_len, hdr.offset) == hdr.after_len && + memcmp(current, after, hdr.after_len) == 0) + { + if ((hdr.before_len && + pg_pwrite(fd, before, hdr.before_len, hdr.offset) != hdr.before_len) || + ftruncate(fd, hdr.orig_length) != 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO WRITE: could not restore \"%s\": %m", path))); + result = UNDO_APPLY_ERROR; + } + } + pfree(current); + close(fd); + } + break; + + case FILEOPS_UNDO_TRUNCATE: + { + FileopsUndoTruncate hdr; + const char *path; + int fd; + + if (payload_len < sizeof(FileopsUndoTruncate)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoTruncate)); + path = payload + sizeof(FileopsUndoTruncate); + + switch (fileops_undo_lstat("TRUNCATE", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!S_ISREG(st.st_mode) || st.st_size != hdr.new_length) + { + result = fileops_undo_superseded("TRUNCATE", path); + break; + } + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO TRUNCATE: could not open \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + } + else + { + if (hdr.before_len && + pg_pwrite(fd, path + hdr.path_len, hdr.before_len, + hdr.new_length) != hdr.before_len) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore bytes of file \"%s\": %m", path))); + result = UNDO_APPLY_ERROR; + } + if (ftruncate(fd, hdr.orig_length) < 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO TRUNCATE: could not restore length of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + close(fd); + } + break; + } + } + break; + + case FILEOPS_UNDO_CHMOD: + { + FileopsUndoChmod hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoChmod)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoChmod)); + path = payload + sizeof(FileopsUndoChmod); + + switch (fileops_undo_lstat("CHMOD", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if ((st.st_mode & 07777) != hdr.new_mode) + result = fileops_undo_superseded("CHMOD", path); + else if (chmod(path, hdr.orig_mode) < 0 && + errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CHMOD: could not restore mode of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_CHOWN: + { +#ifndef WIN32 + FileopsUndoChown hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoChown)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoChown)); + path = payload + sizeof(FileopsUndoChown); + + switch (fileops_undo_lstat("CHOWN", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (st.st_uid != hdr.new_uid || st.st_gid != hdr.new_gid) + result = fileops_undo_superseded("CHOWN", path); + else if (chown(path, hdr.orig_uid, hdr.orig_gid) < 0 && + errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CHOWN: could not restore ownership of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } +#endif + } + break; + + case FILEOPS_UNDO_MKDIR: + { + FileopsUndoMkdir hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoMkdir)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoMkdir)); + path = payload + sizeof(FileopsUndoMkdir); + + switch (fileops_undo_lstat("MKDIR", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!S_ISDIR(st.st_mode)) + result = fileops_undo_superseded("MKDIR", path); + else if ((hdr.flags & FILEOPS_MKDIR_TREE) && + !FILEOPS_RECOVERING() && + (st.st_dev != hdr.dev || st.st_ino != hdr.ino)) + result = fileops_undo_superseded("MKDIR", path); + else if (hdr.flags & FILEOPS_MKDIR_TREE) + { + if (OidIsValid(hdr.dboid)) + { + DropDatabaseBuffers(hdr.dboid); + ForgetDatabaseSyncRequests(hdr.dboid); + } + if (!rmtree(path, true)) + { + ereport(WARNING, + (errmsg("FILEOPS UNDO MKDIR: could not remove directory tree \"%s\"", + path))); + result = UNDO_APPLY_ERROR; + } + } + else if (rmdir(path) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO MKDIR: could not rmdir \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_SYMLINK: + { + FileopsUndoSymlink hdr; + const char *linkpath; + const char *target; + char cur[MAXPGPATH]; + ssize_t cur_len; + + if (payload_len < sizeof(FileopsUndoSymlink)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoSymlink)); + if (payload_len < sizeof(FileopsUndoSymlink) + + hdr.linkpath_len + hdr.target_len) + return UNDO_APPLY_ERROR; + linkpath = payload + sizeof(FileopsUndoSymlink); + target = linkpath + hdr.linkpath_len; + + cur_len = readlink(linkpath, cur, sizeof(cur)); + if (cur_len < 0) + { + if (errno == ENOENT) + break; + if (errno == EINVAL) /* not a symlink any more */ + { + result = fileops_undo_superseded("SYMLINK", linkpath); + break; + } + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO SYMLINK: could not read symbolic link \"%s\": %m", + linkpath))); + result = UNDO_APPLY_ERROR; + } + else if ((size_t) cur_len + 1 != hdr.target_len || + memcmp(cur, target, cur_len) != 0) + result = fileops_undo_superseded("SYMLINK", linkpath); + else if (unlink(linkpath) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO SYMLINK: could not unlink \"%s\": %m", + linkpath))); + result = UNDO_APPLY_ERROR; + } + } + break; + + case FILEOPS_UNDO_LINK: + { + FileopsUndoLink hdr; + const char *newpath; + + if (payload_len < sizeof(FileopsUndoLink)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoLink)); + newpath = payload + sizeof(FileopsUndoLink); + + switch (fileops_undo_lstat("LINK", newpath, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!FILEOPS_RECOVERING() && + (st.st_dev != hdr.dev || st.st_ino != hdr.ino)) + result = fileops_undo_superseded("LINK", newpath); + else if (unlink(newpath) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO LINK: could not unlink \"%s\": %m", + newpath))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_RENAME: + { + FileopsUndoRename hdr; + const char *oldpath; + const char *newpath; + + if (payload_len < sizeof(FileopsUndoRename)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoRename)); + if (payload_len < sizeof(FileopsUndoRename) + + hdr.oldpath_len + hdr.newpath_len) + return UNDO_APPLY_ERROR; + oldpath = payload + sizeof(FileopsUndoRename); + newpath = oldpath + hdr.oldpath_len; + + /* + * Rename back while newpath exists and oldpath is free (see + * FileopsUndoRename). An old name reused since is not ours + * to replace. + */ + switch (fileops_undo_lstat("RENAME", newpath, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + switch (fileops_undo_lstat("RENAME", oldpath, &st)) + { + case 0: + if (rename(newpath, oldpath) < 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO RENAME: could not rename \"%s\" back to \"%s\": %m", + newpath, oldpath))); + result = UNDO_APPLY_ERROR; + } + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + result = fileops_undo_superseded("RENAME", + oldpath); + break; + } + break; + } + } + break; + + case FILEOPS_UNDO_SETXATTR: + { + FileopsUndoSetxattr hdr; + const char *path; + const char *name; + const char *orig_value; + const char *new_value; + + if (payload_len < sizeof(FileopsUndoSetxattr)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoSetxattr)); + if (payload_len < sizeof(FileopsUndoSetxattr) + hdr.path_len + + hdr.name_len + hdr.orig_value_len + hdr.new_value_len) + return UNDO_APPLY_ERROR; + path = payload + sizeof(FileopsUndoSetxattr); + name = path + hdr.path_len; + orig_value = name + hdr.name_len; + new_value = orig_value + hdr.orig_value_len; + + switch (fileops_undo_lstat("SETXATTR", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + switch (fileops_xattr_is(path, name, new_value, + hdr.new_value_len)) + { + case 0: + result = fileops_undo_superseded("SETXATTR", + path); + break; + case -1: + result = fileops_undo_xattr_failed("SETXATTR", + path, name); + break; + default: + if (hdr.had_value) + { + if (pg_setxattr(path, name, orig_value, + hdr.orig_value_len) < 0 && + errno != ENOENT) + result = fileops_undo_xattr_failed("SETXATTR", + path, name); + } + else if (pg_removexattr(path, name) < 0 && + errno != ENOENT && + errno != PG_ENOATTR) + result = fileops_undo_xattr_failed("SETXATTR", + path, name); + break; + } + break; + } + } + break; + + case FILEOPS_UNDO_REMOVEXATTR: + { + FileopsUndoRemovexattr hdr; + const char *path; + const char *name; + const void *value; + + if (payload_len < sizeof(FileopsUndoRemovexattr)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoRemovexattr)); + path = payload + sizeof(FileopsUndoRemovexattr); + name = path + hdr.path_len; + value = name + hdr.name_len; + + switch (fileops_undo_lstat("REMOVEXATTR", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + /* Restore the removed xattr, unless it is back */ + if (pg_getxattr(path, name, NULL, 0) >= 0) + result = fileops_undo_superseded("REMOVEXATTR", + path); + else if (errno != PG_ENOATTR) + result = fileops_undo_xattr_failed("REMOVEXATTR", + path, name); + else if (pg_setxattr(path, name, value, + hdr.value_len) < 0 && + errno != ENOENT) + result = fileops_undo_xattr_failed("REMOVEXATTR", + path, name); + break; + } + } + break; + + default: + ereport(WARNING, + (errmsg("FILEOPS UNDO: unknown subtype %u", info))); + return UNDO_APPLY_ERROR; + } + + INJECTION_POINT("fileops-undo-apply-end", NULL); + + /* + * Recovery logs a crash loser's rollback once it has applied the records + * (XLOG_UNDO_ROLLBACK), so make the inverse durable before returning. + */ + if (!applyingAbort && result == UNDO_APPLY_SUCCESS) + { + fileops_note_inverse(info, payload, payload_len, + data_sync_elevel(ERROR)); + if (FILEOPS_RECOVERING()) + FileOpsSyncInverses(data_sync_elevel(ERROR)); + } + + return result; +} + +/* + * Apply an abort-WAL inverse with the same identity tests as loser recovery. + * Ordinary syscall failures remain warnings. A failed inverse can therefore + * leave an online effect until recovery replays the abort record; callers must + * not infer successful filesystem rollback merely from transaction abort. + * + * The caller makes the inverses durable with FileOpsSyncInverses(PANIC) once + * it has applied them all. Failing to is a PANIC regardless of + * data_sync_retry. Online, we run after the abort record is flushed and the + * transaction is marked aborted in clog, so an ERROR would re-enter + * AbortTransaction and log a second abort record. The caller holds off + * checkpoints until the inverses are durable, so crash recovery replays the + * abort record and applies the inverses again. + */ +void +FileOpsApplyInverse(uint16 info, const char *data, uint32 len) +{ + applyingAbort = true; + PG_TRY(); + { + (void) fileops_undo_apply(UNDO_RMID_FILEOPS, info, InvalidTransactionId, + InvalidOid, data, len, InvalidUndoRecPtr); + fileops_note_inverse(info, data, len, PANIC); + } + PG_FINALLY(); + { + applyingAbort = false; + } + PG_END_TRY(); +} + +/* + * Paths that applied inverses changed and that are not durable yet, each with + * its parent directory. N inverses on one file need one fsync of the file + * and one of its directory, not N of each: every caller makes a whole + * rollback durable before it records the rollback as complete, by calling + * FileOpsSyncInverses() once. + * + * Online, the inverses are applied inside AbortTransaction, where an ERROR + * would re-enter it. So the table is created when the transaction writes its + * first FILEOPS UNDO (FileOpsReserveInverseSyncs), and an entry that cannot be + * allocated there is not an error: that path is synced at once instead. + */ +typedef struct FileOpsSyncEntry +{ + char path[MAXPGPATH]; /* hash key */ + bool target; /* an inverse's own path, not just a parent */ +} FileOpsSyncEntry; + +static HTAB *fileopsPendingSyncs = NULL; + +/* Create the table now, where an out-of-memory ERROR is still harmless. */ +void +FileOpsReserveInverseSyncs(void) +{ + HASHCTL ctl; + + if (fileopsPendingSyncs != NULL) + return; + ctl.keysize = MAXPGPATH; + ctl.entrysize = sizeof(FileOpsSyncEntry); + ctl.hcxt = TopMemoryContext; + fileopsPendingSyncs = hash_create("FILEOPS inverses to sync", 16, &ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); +} + +/* fsync one path an inverse changed, as it is now; see FileOpsSyncInverses. */ +static void +fileops_sync_path(const char *path, bool target, int elevel) +{ + struct stat st; + +#ifdef USE_INJECTION_POINTS + if (target && IS_INJECTION_POINT_ATTACHED("fileops-inverse-fsync-fail")) + { + errno = EIO; + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", path))); + } +#endif + if (lstat(path, &st) == 0 && !S_ISLNK(st.st_mode)) + { + INJECTION_POINT("fileops-inverse-fsync", unconstify(char *, path)); + fsync_fname_ext(path, S_ISDIR(st.st_mode), true, elevel); + } +} + +static void +fileops_note_sync(const char *path, bool target, int elevel) +{ + FileOpsSyncEntry *entry = NULL; + bool found; + + if (!enableFsync) + return; + + /* + * Inside AbortTransaction, an out-of-memory ERROR would re-enter it, so + * the table was created with the transaction's first FILEOPS UNDO. + * Elsewhere (abort redo, the startup undo phase) the ERROR is harmless. + */ + if (fileopsPendingSyncs == NULL && !(applyingAbort && !InRecovery)) + FileOpsReserveInverseSyncs(); + if (fileopsPendingSyncs != NULL) + entry = hash_search(fileopsPendingSyncs, path, HASH_ENTER_NULL, + &found); + if (entry == NULL) + { + /* No memory to defer it: make this path durable now. */ + fileops_sync_path(path, target, elevel); + return; + } + if (!found) + entry->target = false; + entry->target |= target; +} + +/* Remember the directory of a path an inverse changed, for syncing. */ +static void +fileops_note_parent(const char *path, int elevel) +{ + char parent[MAXPGPATH]; + char *sep; + + strlcpy(parent, path, sizeof(parent)); + sep = strrchr(parent, '/'); + if (sep == NULL) + strlcpy(parent, ".", sizeof(parent)); + else if (sep == parent) + parent[1] = '\0'; + else + *sep = '\0'; + + fileops_note_sync(parent, false, elevel); +} + +/* Remember the paths an inverse changed, and their directories. */ +static void +fileops_note_inverse(uint16 info, const char *data, uint32 len, int elevel) +{ + Size hdrlen = 0; + + switch (info) + { + case FILEOPS_UNDO_CREATE: + hdrlen = sizeof(FileopsUndoCreate); + break; + case FILEOPS_UNDO_WRITE: + hdrlen = sizeof(FileopsUndoWrite); + break; + case FILEOPS_UNDO_TRUNCATE: + hdrlen = sizeof(FileopsUndoTruncate); + break; + case FILEOPS_UNDO_CHMOD: + hdrlen = sizeof(FileopsUndoChmod); + break; + case FILEOPS_UNDO_CHOWN: + hdrlen = sizeof(FileopsUndoChown); + break; + case FILEOPS_UNDO_MKDIR: + hdrlen = sizeof(FileopsUndoMkdir); + break; + case FILEOPS_UNDO_SYMLINK: + hdrlen = sizeof(FileopsUndoSymlink); + break; + case FILEOPS_UNDO_LINK: + hdrlen = sizeof(FileopsUndoLink); + break; + case FILEOPS_UNDO_RENAME: + { + FileopsUndoRename hdr; + + /* + * The old name is back, in its directory, and the new name is + * gone from its own. + */ + if (len < sizeof(hdr)) + break; + memcpy(&hdr, data, sizeof(hdr)); + if (hdr.oldpath_len == 0 || hdr.newpath_len == 0 || + len < sizeof(hdr) + hdr.oldpath_len + hdr.newpath_len) + break; + fileops_note_sync(data + sizeof(hdr), true, elevel); + fileops_note_parent(data + sizeof(hdr), elevel); + fileops_note_parent(data + sizeof(hdr) + hdr.oldpath_len, + elevel); + return; + } + case FILEOPS_UNDO_SETXATTR: + hdrlen = sizeof(FileopsUndoSetxattr); + break; + case FILEOPS_UNDO_REMOVEXATTR: + hdrlen = sizeof(FileopsUndoRemovexattr); + break; + } + if (hdrlen == 0 || len <= hdrlen) + elog(PANIC, "invalid FILEOPS inverse payload"); + fileops_note_sync(data + hdrlen, true, elevel); + fileops_note_parent(data + hdrlen, elevel); +} + +/* + * FileOpsSyncInverses - Make the inverses applied so far durable + * + * fsyncs each path noted since the last call once, as it is now: a file or + * directory that still exists (a symlink is covered by its directory), and + * the directory of each. A path a later inverse removed is skipped; its + * removal is in its directory, which an inverse that removed the directory + * itself has noted in turn. Failures are reported at elevel. After an + * ERROR the paths not yet synced stay noted, and the next call syncs them. + */ +void +FileOpsSyncInverses(int elevel) +{ + HASH_SEQ_STATUS seq; + FileOpsSyncEntry *entry; + FileOpsSyncEntry **entries; + long n; + long i = 0; + + if (fileopsPendingSyncs == NULL || + hash_get_num_entries(fileopsPendingSyncs) == 0) + return; + + /* + * Sync from a copy of the entries: an ERROR must not leave a hash scan + * open, and every entry must stay noted until it is synced. Without + * memory for the copy (we may be inside AbortTransaction), take the + * entries one at a time instead, each with a scan of its own. + */ + n = hash_get_num_entries(fileopsPendingSyncs); + entries = palloc_extended(n * sizeof(FileOpsSyncEntry *), + MCXT_ALLOC_NO_OOM); + if (entries != NULL) + { + hash_seq_init(&seq, fileopsPendingSyncs); + while ((entry = hash_seq_search(&seq)) != NULL) + entries[i++] = entry; + for (i = 0; i < n; i++) + { + fileops_sync_path(entries[i]->path, entries[i]->target, elevel); + (void) hash_search(fileopsPendingSyncs, entries[i]->path, + HASH_REMOVE, NULL); + } + pfree(entries); + } + else + { + for (;;) + { + hash_seq_init(&seq, fileopsPendingSyncs); + entry = hash_seq_search(&seq); + if (entry == NULL) + break; + hash_seq_term(&seq); + fileops_sync_path(entry->path, entry->target, elevel); + (void) hash_search(fileopsPendingSyncs, entry->path, HASH_REMOVE, + NULL); + } + } +} + +/* + * fileops_undo_desc - Describe a FILEOPS UNDO record for debugging + */ +static void +fileops_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len) +{ + const char *opname; + + switch (info) + { + case FILEOPS_UNDO_CREATE: + opname = "CREATE"; + break; + case FILEOPS_UNDO_TRUNCATE: + opname = "TRUNCATE"; + break; + case FILEOPS_UNDO_CHMOD: + opname = "CHMOD"; + break; + case FILEOPS_UNDO_CHOWN: + opname = "CHOWN"; + break; + case FILEOPS_UNDO_MKDIR: + opname = "MKDIR"; + break; + case FILEOPS_UNDO_SYMLINK: + opname = "SYMLINK"; + break; + case FILEOPS_UNDO_LINK: + opname = "LINK"; + break; + case FILEOPS_UNDO_RENAME: + opname = "RENAME"; + break; + case FILEOPS_UNDO_SETXATTR: + opname = "SETXATTR"; + break; + case FILEOPS_UNDO_REMOVEXATTR: + opname = "REMOVEXATTR"; + break; + default: + opname = "UNKNOWN"; + break; + } + + appendStringInfo(buf, "fileops %s", opname); +} diff --git a/src/backend/storage/file/meson.build b/src/backend/storage/file/meson.build index 795402589b0b9..7e487324c5f3e 100644 --- a/src/backend/storage/file/meson.build +++ b/src/backend/storage/file/meson.build @@ -4,6 +4,8 @@ backend_sources += files( 'buffile.c', 'copydir.c', 'fd.c', + 'fileops.c', + 'fileops_undo.c', 'fileset.c', 'reinit.c', 'sharedfileset.c', diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 1512d2df196e6..613984a301ce2 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1187,6 +1187,7 @@ ProcessUtilitySlow(ParseState *pstate, validnsps, true, false); + (void) heap_reloptions(RELKIND_TOASTVALUE, toast_options, true); diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 8b6ea195ecade..7a0522ae859d8 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -21,6 +21,7 @@ #include "access/genam.h" #include "access/heapam.h" +#include "access/undo.h" #include "access/htup_details.h" #include "access/session.h" #include "access/tableam.h" @@ -857,6 +858,9 @@ InitPostgres(const char *in_dbname, Oid dboid, InitCatalogCache(); InitPlanCache(); + /* Initialize per-backend undo subsystem state */ + InitializeUndo(); + /* Initialize portal manager */ EnablePortalManager(); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index a3f38f3b88679..811f6869bb8a9 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -17530,6 +17530,7 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) if (nonemptyReloptions(tbinfo->reloptions)) { addcomma = true; + /* Emits all heap reloptions */ appendReloptionsArrayAH(q, tbinfo->reloptions, "", fout); } if (nonemptyReloptions(tbinfo->toast_reloptions)) diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c index 931ab8b979e23..b16c93f51a31b 100644 --- a/src/bin/pg_waldump/rmgrdesc.c +++ b/src/bin/pg_waldump/rmgrdesc.c @@ -11,6 +11,7 @@ #include "access/brin_xlog.h" #include "access/clog.h" #include "access/commit_ts.h" +#include "access/fileops_xlog.h" #include "access/generic_xlog.h" #include "access/ginxlog.h" #include "access/gistxlog.h" @@ -20,6 +21,7 @@ #include "access/nbtxlog.h" #include "access/rmgr.h" #include "access/spgxlog.h" +#include "access/undo_xlog.h" #include "access/xact.h" #include "access/xlog_internal.h" #include "catalog/storage_xlog.h" diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index 7b33efc62999f..ded5e37feda65 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -96,7 +96,9 @@ ReplicationOrigin Generic LogicalMessage -XLOG2$/, +XLOG2 +Undo +FileOps$/, 'rmgr list'); diff --git a/src/include/access/fileops_xlog.h b/src/include/access/fileops_xlog.h new file mode 100644 index 0000000000000..b3ece9a268691 --- /dev/null +++ b/src/include/access/fileops_xlog.h @@ -0,0 +1,40 @@ +/*------------------------------------------------------------------------- + * + * fileops_xlog.h + * Transactional file operations XLOG resource manager definitions + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/include/access/fileops_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILEOPS_XLOG_H +#define FILEOPS_XLOG_H + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* XLOG stuff - all record types defined upfront for WAL compatibility */ +#define XLOG_FILEOPS_CREATE 0x00 +#define XLOG_FILEOPS_DELETE 0x10 +#define XLOG_FILEOPS_RENAME 0x20 +#define XLOG_FILEOPS_WRITE 0x30 +#define XLOG_FILEOPS_TRUNCATE 0x40 +#define XLOG_FILEOPS_CHMOD 0x50 +#define XLOG_FILEOPS_CHOWN 0x60 +#define XLOG_FILEOPS_MKDIR 0x70 +#define XLOG_FILEOPS_RMDIR 0x80 +#define XLOG_FILEOPS_SYMLINK 0x90 +#define XLOG_FILEOPS_LINK 0xA0 +#define XLOG_FILEOPS_SETXATTR 0xB0 +#define XLOG_FILEOPS_REMOVEXATTR 0xC0 + +/* Resource manager functions */ +extern void fileops_redo(XLogReaderState *record); +extern void fileops_desc(StringInfo buf, XLogReaderState *record); +extern const char *fileops_identify(uint8 info); + +#endif /* FILEOPS_XLOG_H */ diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 99b51296e8018..ea91747ae85ab 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -544,4 +544,5 @@ heap_execute_freeze_tuple(HeapTupleHeader tuple, HeapTupleFreeze *frz) tuple->t_infomask2 = frz->t_infomask2; } + #endif /* HEAPAM_H */ diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index ae32ef16d67b6..6d8f61f724f9d 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -48,3 +48,5 @@ PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask, NULL) PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL, logicalmsg_decode) PG_RMGR(RM_XLOG2_ID, "XLOG2", xlog2_redo, xlog2_desc, xlog2_identify, NULL, NULL, NULL, xlog2_decode) +PG_RMGR(RM_UNDO_ID, "Undo", undo_redo, undo_desc, undo_identify, NULL, NULL, NULL, NULL) +PG_RMGR(RM_FILEOPS_ID, "FileOps", fileops_redo, fileops_desc, fileops_identify, NULL, NULL, NULL, NULL) diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h index 1d2ff42c9b72f..3714ea62b3385 100644 --- a/src/include/access/twophase.h +++ b/src/include/access/twophase.h @@ -48,6 +48,7 @@ extern GlobalTransaction MarkAsPreparing(FullTransactionId fxid, const char *gid extern void StartPrepare(GlobalTransaction gxact); extern void EndPrepare(GlobalTransaction gxact); extern bool StandbyTransactionIdIsPrepared(TransactionId xid); +extern bool RecoveryTransactionIdIsPrepared(TransactionId xid); extern TransactionId PrescanPreparedTransactions(TransactionId **xids_p, int *nxids_p); @@ -72,4 +73,6 @@ extern bool LookupGXactBySubid(Oid subid); extern TransactionId TwoPhaseGetOldestXidInCommit(void); +extern XLogRecPtr TwoPhaseGetOldestUndoBatchLSN(void); + #endif /* TWOPHASE_H */ diff --git a/src/include/access/twophase_rmgr.h b/src/include/access/twophase_rmgr.h index 8927f369c39b5..37be3c7b2900c 100644 --- a/src/include/access/twophase_rmgr.h +++ b/src/include/access/twophase_rmgr.h @@ -28,7 +28,8 @@ typedef uint8 TwoPhaseRmgrId; #define TWOPHASE_RM_PGSTAT_ID 2 #define TWOPHASE_RM_MULTIXACT_ID 3 #define TWOPHASE_RM_PREDICATELOCK_ID 4 -#define TWOPHASE_RM_MAX_ID TWOPHASE_RM_PREDICATELOCK_ID +#define TWOPHASE_RM_FILEOPS_ID 5 +#define TWOPHASE_RM_MAX_ID TWOPHASE_RM_FILEOPS_ID extern PGDLLIMPORT const TwoPhaseCallback twophase_recover_callbacks[]; extern PGDLLIMPORT const TwoPhaseCallback twophase_postcommit_callbacks[]; @@ -39,4 +40,10 @@ extern PGDLLIMPORT const TwoPhaseCallback twophase_standby_recover_callbacks[]; extern void RegisterTwoPhaseRecord(TwoPhaseRmgrId rmid, uint16 info, const void *data, uint32 len); +/* FILEOPS two-phase commit callbacks (defined in storage/file/fileops.c) */ +extern void fileops_twophase_postcommit(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); +extern void fileops_twophase_postabort(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); + #endif /* TWOPHASE_RMGR_H */ diff --git a/src/include/access/undo.h b/src/include/access/undo.h new file mode 100644 index 0000000000000..84007d10e06c0 --- /dev/null +++ b/src/include/access/undo.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * undo.h + * Common undo layer interface + * + * See access/undo/README for the pieces of the UNDO engine. This header + * provides the entry points for shared memory set-up and per-backend + * initialization across them. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undo.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDO_H +#define UNDO_H + +#include "access/undodefs.h" +#include "utils/palloc.h" + +/* + * Shared memory initialization, called through UndoShmemCallbacks (see + * storage/subsystemlist.h). + */ +extern void UndoShmemInit(void); + +/* Per-backend initialization */ +extern void InitializeUndo(void); + +/* Memory context for undo-related allocations */ +extern MemoryContext UndoContext; + +#endif /* UNDO_H */ diff --git a/src/include/access/undo_xlog.h b/src/include/access/undo_xlog.h new file mode 100644 index 0000000000000..aceef55fdbbe6 --- /dev/null +++ b/src/include/access/undo_xlog.h @@ -0,0 +1,169 @@ +/*------------------------------------------------------------------------- + * + * undo_xlog.h + * UNDO resource manager WAL record definitions + * + * This file contains the WAL record format definitions for UNDO log + * operations. These records are logged by the RM_UNDO_ID resource manager. + * + * Record types: + * XLOG_UNDO_BATCH - A batch of a transaction's UNDO records + * XLOG_UNDO_INFLIGHT - UNDO chains in flight at a checkpoint + * XLOG_UNDO_ROLLBACK - A crash loser rolled back by recovery + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undo_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDO_XLOG_H +#define UNDO_XLOG_H + +#include "access/transam.h" +#include "access/xlogdefs.h" +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* + * UndoRecPtr type definition. We use undodefs.h which is lightweight + * and can be included in both frontend and backend code. If undodefs.h + * has already been included (via undolog.h or directly), this is a no-op. + */ +#include "access/undodefs.h" + +/* + * WAL record types for UNDO operations + * + * These are the info codes for UNDO WAL records. The low 4 bits are used + * for operation type, leaving the upper 4 bits for flags. + */ +#define XLOG_UNDO_BATCH 0x00 /* Batched UNDO records in WAL */ +#define XLOG_UNDO_INFLIGHT 0x10 /* UNDO chains in flight at a + * checkpoint */ +#define XLOG_UNDO_ROLLBACK 0x20 /* A crash loser rolled back by + * recovery */ + +/* + * xl_undo_rollback - A crash loser that the end of recovery rolled back + * + * A transaction that a crash leaves in progress has no abort record. At the + * end of crash recovery, once WAL can be written again, the startup process + * applies each such loser's UNDO records and then writes one of these, + * carrying the records that applied, in the order it applied them (newest + * first). Redo applies them again, the same way, so that a standby, and a + * later recovery that replays this record, end up where the primary did, + * without reading the loser's UNDO chain from WAL. Every UNDO resource + * manager's apply function must therefore leave an object alone that no + * longer shows the record's effect. + * + * The records follow the header as a packed stream of UndoRecordHeader and + * payload, each urec_len bytes long, as in an XLOG_UNDO_BATCH record. + */ +typedef struct xl_undo_rollback +{ + TransactionId xid; /* the loser */ + uint32 nrecords; /* UNDO records that follow */ + /* followed by the UNDO records */ +} xl_undo_rollback; + +#define SizeOfUndoRollback sizeof(xl_undo_rollback) + +/* The records must start MAXALIGN'd, as they do in a batch. */ +StaticAssertDecl(SizeOfUndoRollback % MAXIMUM_ALIGNOF == 0, + "xl_undo_rollback must keep the UNDO records after it aligned"); + +/* + * xl_undo_batch - WAL record for batched UNDO data (XLOG_UNDO_BATCH) + * + * All UNDO records for a batch are serialized into a single WAL record. + * The batch payload contains concatenated UndoRecordHeader+payload pairs + * in their exact serialized format. + * + * The chain_prev field links this batch to the previous batch for the + * same transaction. During rollback, the UNDO chain is walked backward + * by reading WAL records at successive chain_prev LSNs. + * + * Coalescing: The existing UndoRecordSet mechanism batches records + * (flush at 256KB or 1000 records). This batch becomes one WAL record. + * A 1000-row INSERT produces ~1 WAL record containing 1000 UNDO records. + */ +typedef struct xl_undo_batch +{ + TransactionId xid; /* Owning transaction */ + XLogRecPtr chain_prev; /* LSN of previous batch for this xact + * (InvalidXLogRecPtr if first batch) */ + uint32 nrecords; /* Number of UNDO records in batch */ + uint32 total_len; /* Total bytes of serialized UNDO data */ + Oid primary_reloid; /* Relation OID (optimization for + * single-relation batches) */ + UndoPersistenceLevel persistence; /* Persistence level of this batch */ + /* Followed by total_len bytes of serialized UndoRecordHeader+payload */ +} xl_undo_batch; + +#define SizeOfUndoBatch (offsetof(xl_undo_batch, persistence) + sizeof(UndoPersistenceLevel)) + +/* + * xl_undo_inflight - UNDO chains of transactions in flight at a checkpoint + * + * Written by every online checkpoint after its redo pointer. Crash recovery + * starts redo there and so replays none of the batches written before it; a + * transaction whose batches all precede the redo pointer is known to + * recovery only through this record. + */ +typedef struct xl_undo_inflight_xact +{ + TransactionId xid; /* top-level transaction */ + XLogRecPtr last_batch_lsn[NUndoPersistenceLevels]; +} xl_undo_inflight_xact; + +typedef struct xl_undo_inflight +{ + uint32 nxacts; + xl_undo_inflight_xact xacts[FLEXIBLE_ARRAY_MEMBER]; +} xl_undo_inflight; + +#define SizeOfUndoInflight offsetof(xl_undo_inflight, xacts) + +/* Function declarations for WAL operations */ +extern void undo_redo(XLogReaderState *record); +extern void undo_desc(StringInfo buf, XLogReaderState *record); +extern const char *undo_identify(uint8 info); + +/* + * UNDO batch reading from WAL for rollback and recovery. + * + * UndoReadBatchFromWAL reads a single XLOG_UNDO_BATCH record at the + * given LSN and returns the header plus a pointer to the payload data. + * The caller must pfree the returned data when done. + */ +typedef struct UndoBatchData +{ + xl_undo_batch header; /* Batch header */ + char *payload; /* Serialized UNDO records (palloc'd) */ + Size payload_len; /* Length of payload */ +} UndoBatchData; + +extern bool UndoValidateBatchLSN(XLogRecPtr batch_lsn); +extern UndoBatchData *UndoReadBatchFromWAL(XLogRecPtr batch_lsn); +extern void UndoFreeBatchData(UndoBatchData *batch); + +/* + * Recovery UNDO phase support. + * + * During WAL redo, XLOG_UNDO_BATCH and XLOG_UNDO_INFLIGHT records are + * tracked, so that at the end of recovery the transactions a crash left in + * progress can be found and rolled back (UndoRecoveryRollBack). + */ +extern void UndoRecoveryTrackBatch(TransactionId xid, XLogRecPtr batch_lsn, + XLogRecPtr chain_prev, + UndoPersistenceLevel persistence); +extern void UndoRecoveryRemoveXid(TransactionId xid); +extern void UndoRecoveryTrackInflight(const xl_undo_inflight_xact *x); +extern void UndoRecoveryRollBack(void); + +/* True while recovery applies UNDO records outside redo; see there. */ +extern PGDLLIMPORT bool UndoRecoveryRollingBack; + +#endif /* UNDO_XLOG_H */ diff --git a/src/include/access/undodefs.h b/src/include/access/undodefs.h new file mode 100644 index 0000000000000..c64192f01fb6d --- /dev/null +++ b/src/include/access/undodefs.h @@ -0,0 +1,47 @@ +/*------------------------------------------------------------------------- + * + * undodefs.h + * + * Basic definitions for PostgreSQL undo layer. These are separated into + * their own header file to avoid including more things than necessary + * into widely-used headers like xact.h. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undodefs.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDODEFS_H +#define UNDODEFS_H + +/* The type used to identify an undo log and position within it. */ +typedef uint64 UndoRecPtr; + +/* Special value for undo record pointer which indicates that it is invalid. */ +#define InvalidUndoRecPtr ((UndoRecPtr) 0) + +/* + * UndoRecPtrIsValid + * True iff undoRecPtr is valid. + */ +#define UndoRecPtrIsValid(undoRecPtr) \ + ((bool) ((UndoRecPtr) (undoRecPtr) != InvalidUndoRecPtr)) + +/* Persistence levels as small integers that can be used as array indexes. */ +typedef enum +{ + UNDOPERSISTENCE_PERMANENT = 0, + UNDOPERSISTENCE_UNLOGGED = 1, + UNDOPERSISTENCE_TEMP = 2 +} UndoPersistenceLevel; + +/* Number of supported persistence levels for undo. */ +#define NUndoPersistenceLevels 3 + +/* Opaque types. */ +struct UndoRecordSet; +typedef struct UndoRecordSet UndoRecordSet; + +#endif diff --git a/src/include/access/undolog.h b/src/include/access/undolog.h new file mode 100644 index 0000000000000..519739acdace1 --- /dev/null +++ b/src/include/access/undolog.h @@ -0,0 +1,41 @@ +/*------------------------------------------------------------------------- + * + * undolog.h + * WAL retention for UNDO records + * + * UNDO records live in the WAL stream as XLOG_UNDO_BATCH records, so the + * WAL holding a transaction's UNDO must not be recycled while something may + * still read it: the transaction's own abort, ROLLBACK PREPARED, or crash + * recovery. Each backend publishes the LSN of its current transaction's + * first UNDO batch in a shared slot, and KeepLogSeg() keeps WAL from the + * oldest such LSN onward. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undolog.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDOLOG_H +#define UNDOLOG_H + +#include "access/xlogdefs.h" +#include "port/atomics.h" + +/* + * The per-backend first-batch LSN slots, MaxBackends of them, indexed by + * MyProcNumber. A slot holds the LSN of the first XLOG_UNDO_BATCH record of + * the backend's current transaction, or InvalidXLogRecPtr. Only the owning + * backend writes its slot. + */ +extern PGDLLIMPORT pg_atomic_uint64 *UndoBatchLSNs; + +extern void UndoLogShmemRequest(void); +extern void UndoLogShmemInit(void); + +extern void UndoRegisterBatchLSN(XLogRecPtr batch_lsn); +extern void UndoClearBatchLSN(void); +extern XLogRecPtr UndoGetOldestBatchLSN(void); + +#endif /* UNDOLOG_H */ diff --git a/src/include/access/undorecord.h b/src/include/access/undorecord.h new file mode 100644 index 0000000000000..19337f9291389 --- /dev/null +++ b/src/include/access/undorecord.h @@ -0,0 +1,166 @@ +/*------------------------------------------------------------------------- + * + * undorecord.h + * UNDO record format and insertion API + * + * This file defines the generic UNDO record format that can be used by + * any access method or subsystem. UNDO records are AM-agnostic: each + * record carries an RM ID (urec_rmid) that identifies the resource + * manager responsible for interpreting and applying the record. + * + * Design principles: + * - Physical: UNDO stores opaque payload data for direct restore + * - Generic: Usable by any AM or subsystem + * - Compact: Variable-length format to minimize space + * - Chained: Batches form backward chains via xl_undo_batch.chain_prev + * - Batch-oriented: API encourages batching for performance + * - AM-agnostic: No AM-specific types in the generic header or API + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undorecord.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDORECORD_H +#define UNDORECORD_H + +#include "access/undodefs.h" +#include "access/undolog.h" +#include "access/xlogdefs.h" +#include "storage/block.h" +#include "storage/itemptr.h" + +/* + * UNDO record info flags + * + * These flags provide additional metadata about the UNDO record. + * The lower byte is reserved for generic flags; the upper byte is + * available for RM-specific use. + */ +#define UNDO_INFO_HAS_PAYLOAD 0x01 /* Record contains opaque payload */ +#define UNDO_INFO_XID_VALID 0x08 /* urec_xid is valid */ + +/* + * UndoRecordHeader - Fixed header for all UNDO records + * + * Every UNDO record starts with this header, followed by an optional + * opaque payload whose interpretation is RM-specific. + * + * The urec_rmid field identifies which resource manager owns this record. + * The urec_info field carries RM-specific subtype/flags (e.g., an in-place + * update AM uses it to distinguish INSERT vs DELETE vs UPDATE). + * + * Size: 20 bytes; each record is padded to MAXALIGN (see undorecord.c). + */ +typedef struct UndoRecordHeader +{ + uint8 urec_rmid; /* UNDO RM ID */ + uint8 urec_flags; /* Generic flags (UNDO_INFO_*) */ + uint16 urec_info; /* RM-specific subtype and flags */ + uint32 urec_len; /* Total length including header + payload */ + + TransactionId urec_xid; /* Transaction that created this */ + Oid urec_reloid; /* Relation OID (InvalidOid if N/A) */ + + /* + * Payload length: size of the RM-specific opaque data that follows the + * header. Interpretation is entirely RM-specific. + */ + uint32 urec_payload_len; + + /* Followed by variable-length RM-specific payload */ +} UndoRecordHeader; + +#define SizeOfUndoRecordHeader (offsetof(UndoRecordHeader, urec_payload_len) + sizeof(uint32)) + +/* + * Access macros for payload data following the header + * + * The payload immediately follows the fixed header in the serialized + * record. Its interpretation is entirely RM-specific. + */ +#define UndoRecGetPayload(header) \ + ((char *)(header) + SizeOfUndoRecordHeader) + +/* + * UndoRecordSet - Batch container for UNDO records + * + * This structure accumulates multiple UNDO records before writing them + * to the UNDO log in a single operation. This improves performance by + * reducing the number of I/O operations and lock acquisitions. + * + * The records are serialized into a contiguous buffer that grows + * dynamically. + */ +typedef struct UndoRecordSet +{ + TransactionId xid; /* Transaction ID for all records */ + UndoPersistenceLevel persistence; /* Persistence level of this set */ + + int nrecords; /* Number of records in set */ + + /* + * Dynamic buffer for serialized records. Grows as needed; no fixed + * maximum. This replaces the old fixed-capacity max_records array. + */ + char *buffer; /* Serialized record buffer */ + Size buffer_size; /* Current buffer size */ + Size buffer_capacity; /* Allocated buffer capacity */ + + /* + * WAL-based UNDO chain tracking. When UNDO records are written to WAL + * via XLOG_UNDO_BATCH, last_batch_lsn tracks the LSN of the most recent + * batch for this record set. This is used as the chain_prev link when + * the next batch is written. + */ + XLogRecPtr last_batch_lsn; /* LSN of last XLOG_UNDO_BATCH record */ + + MemoryContext mctx; /* Memory context for allocations */ +} UndoRecordSet; + +/* + * Public API for UNDO record management + */ + +/* Create/destroy/reset UNDO record sets */ +extern UndoRecordSet *UndoRecordSetCreate(TransactionId xid); +extern void UndoRecordSetFree(UndoRecordSet *uset); +extern void UndoRecordSetReset(UndoRecordSet *uset); +extern void UndoRecordSetResetCache(void); + +/* Add records to a set - generic payload API */ +extern void UndoRecordAddPayload(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *payload, + Size payload_len); + +/* Add records with scatter-gather payload (avoids intermediate buffer) */ +extern void UndoRecordAddPayloadParts(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *part1, + Size part1_len, + const char *part2, + Size part2_len); + +/* Insert the accumulated records into UNDO log */ +extern UndoRecPtr UndoRecordSetInsert(UndoRecordSet *uset); + +/* Utility functions for record manipulation */ +extern Size UndoRecordGetPayloadSize(Size payload_len); + +/* Statistics and debugging */ + +/* Records currently accumulated in the set (0 after a flush/reset). */ + +/* UNDO application during rollback */ +extern bool ApplyUndoChainFromWAL(XLogRecPtr last_batch_lsn); +extern bool ApplyUndoChainFromWALBounded(XLogRecPtr last_batch_lsn, + XLogRecPtr stop_at_lsn); + +#endif /* UNDORECORD_H */ diff --git a/src/include/access/undormgr.h b/src/include/access/undormgr.h new file mode 100644 index 0000000000000..f03f5e0c8dd7b --- /dev/null +++ b/src/include/access/undormgr.h @@ -0,0 +1,117 @@ +/*------------------------------------------------------------------------- + * + * undormgr.h + * UNDO resource manager dispatch definitions + * + * This module provides a dispatch mechanism for UNDO record application, + * analogous to the WAL resource manager (rmgr) system. Each access method + * or subsystem that writes UNDO records registers an UndoRmgrData entry + * with callbacks for applying UNDO records and describing them for debugging. + * + * The generic UNDO infrastructure (undoapply.c) dispatches to the appropriate + * RM callback based on the urec_rmid field in the UNDO record header. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undormgr.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDORMGR_H +#define UNDORMGR_H + +#include "postgres_ext.h" +#include "access/undodefs.h" +#include "access/xlogdefs.h" +#include "lib/stringinfo.h" + +/* + * UNDO Resource Manager IDs + * + * Each AM or subsystem that writes UNDO records is assigned a unique ID. + * This ID is stored in the urec_rmid field of every UNDO record header, + * enabling the generic UNDO infrastructure to dispatch to the correct + * apply callback during rollback. + * + * The core defines only UNDO_RMID_INVALID. Each consumer defines its own + * UNDO_RMID_* constant in its own header (the same commit that registers it + * in access/undormgrlist.h), so this core header names no specific AM or + * subsystem. The ID number space is a shared resource: IDs must be unique + * and stable across releases (they are stamped into WAL-durable UNDO + * records), so a new consumer takes the next free value in + * [1, MAX_UNDO_RMGRS) and never reuses one. + */ +#define UNDO_RMID_INVALID 0 + +#define MAX_UNDO_RMGRS 256 + +/* + * UndoApplyResult - Return value from undo apply callbacks + */ +typedef enum UndoApplyResult +{ + UNDO_APPLY_SUCCESS = 0, /* Successfully applied */ + UNDO_APPLY_SKIPPED, /* Skipped (e.g., relation dropped) */ + UNDO_APPLY_ERROR /* Error during application */ +} UndoApplyResult; + +/* + * UndoRmgrData - Resource manager registration entry + * + * Each UNDO RM provides: + * rm_name: Human-readable name for debugging/logging + * rm_undo: Apply one UNDO record (rollback callback) + * rm_desc: Describe an UNDO record for debugging output + * + * The rm_undo callback receives: + * - rmid: The RM ID (for verification) + * - info: RM-specific subtype/flags from urec_info + * - xid: Transaction being rolled back + * - reloid: Target relation OID (may be InvalidOid for non-relation ops) + * - payload: RM-specific opaque payload data + * - payload_len: Length of payload + * - urec_ptr: Position of this record in UNDO log, if known + * + * The callback is responsible for all of the work, including releasing any + * locks or buffers it takes. It may be called more than once for the same + * record: by an online abort, by the undo phase at the end of crash + * recovery, and by redo of the XLOG_UNDO_ROLLBACK record that logs the undo + * phase's work, on the primary and on standbys. So it must leave alone an + * object that no longer shows the record's effect, and it must not write + * WAL, since redo cannot. + */ +typedef UndoApplyResult (*UndoRmgrApplyFunc) (uint8 rmid, + uint16 info, + TransactionId xid, + Oid reloid, + const char *payload, + Size payload_len, + UndoRecPtr urec_ptr); + +typedef void (*UndoRmgrDescFunc) (StringInfo buf, + uint8 rmid, + uint16 info, + const char *payload, + Size payload_len); + +typedef struct UndoRmgrData +{ + const char *rm_name; /* Human-readable name */ + UndoRmgrApplyFunc rm_undo; /* Apply callback */ + UndoRmgrDescFunc rm_desc; /* Describe callback */ +} UndoRmgrData; + +/* Global registration table */ +extern const UndoRmgrData *UndoRmgrs[MAX_UNDO_RMGRS]; + +/* Registration function (called during _PG_init or startup) */ +extern void RegisterUndoRmgr(uint8 rmid, const UndoRmgrData *rmgr); + +/* Lookup function */ +extern const UndoRmgrData *GetUndoRmgr(uint8 rmid); + +/* Initialization */ +extern void InitUndoRmgrs(void); + +#endif /* UNDORMGR_H */ diff --git a/src/include/access/undormgrlist.h b/src/include/access/undormgrlist.h new file mode 100644 index 0000000000000..a9f8ec5afd730 --- /dev/null +++ b/src/include/access/undormgrlist.h @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------- + * + * undormgrlist.h + * + * List of *UndoRmgrInit() initialization calls for built-in UNDO resource + * managers. Kept in its own source file, mirroring the pattern used by + * storage/subsystemlist.h for ShmemCallbacks registration, so that the + * generic UNDO core (undo.c) can register every built-in resource manager + * without knowing any of their names at compile time: undo.c only expands + * this list through a macro it controls, and the extern declarations are + * generated the same way (see access/undormgrs.h). + * + * Each access method or subsystem that writes UNDO records adds itself here + * when it is compiled in. This file is edited by whichever commit + * introduces a new UNDO-writing consumer; undo.c itself is never touched to + * add a new consumer. + * + * UNDO_RMGR_INIT is defined by the caller depending on how the list is used. + * + * No built-in resource managers exist yet at this point in the UNDO + * subsystem's history; this file starts empty and grows as each consumer + * (an index AM, a table AM, or a subsystem that writes UNDO records) is + * compiled in and adds itself here. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undormgrlist.h + * + *------------------------------------------------------------------------- + */ + +/* there is deliberately not an #ifndef UNDORMGRLIST_H here */ + +/* FILEOPS: transactional filesystem operations UNDO rmgr (UNDO_RMID_FILEOPS=2) */ +UNDO_RMGR_INIT(FileopsUndoRmgrInit) diff --git a/src/include/access/undormgrs.h b/src/include/access/undormgrs.h new file mode 100644 index 0000000000000..2a399f78dfab2 --- /dev/null +++ b/src/include/access/undormgrs.h @@ -0,0 +1,28 @@ +/*------------------------------------------------------------------------- + * + * undormgrs.h + * Provide extern declarations for all the built-in UNDO resource + * manager *Init() functions. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undormgrs.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDORMGRS_H +#define UNDORMGRS_H + +/* + * Extern declarations of all the built-in *UndoRmgrInit() functions. + * + * The actual list is in undormgrlist.h, so that the same list can be used + * for other purposes (e.g. undo.c's RegisterUndoRmgrs()). + */ +#define UNDO_RMGR_INIT(initfunc) \ + extern void initfunc(void); +#include "access/undormgrlist.h" +#undef UNDO_RMGR_INIT + +#endif /* UNDORMGRS_H */ diff --git a/src/include/access/xact.h b/src/include/access/xact.h index a8cbdf247c866..799ddcd6896f7 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -138,6 +138,24 @@ typedef enum typedef void (*XactCallback) (XactEvent event, void *arg); +/* + * PendingPhysOps{Do,PostPrepare,AtSubCommit,AtSubAbort}_hook: mirror the + * matching core smgr{DoPendingDeletes,PostPrepare,AtSubCommit,AtSubAbort} + * calls in xact.c. A subsystem that manages pending structural filesystem + * operations needs the SAME commit/abort-sequencing position + * as smgr's own pending-deletes housekeeping -- e.g. + * PendingPhysOpsDo_hook(true) is placed immediately after + * smgrDoPendingDeletes(true), which core's own comment there documents as + * "best done after releasing relcache and buffer pins... this ordering is + * definitely critical during abort" -- a physical-file-cleanup ordering + * requirement no ordinary XactCallback/SubXactCallback event exposes. NULL + * is a valid no-op for a build with no such subsystem compiled in. + */ +extern void (*PendingPhysOpsDo_hook) (bool isCommit); +extern void (*PendingPhysOpsPostPrepare_hook) (void); +extern void (*PendingPhysOpsAtSubCommit_hook) (void); +extern void (*PendingPhysOpsAtSubAbort_hook) (void); + typedef enum { SUBXACT_EVENT_START_SUB, @@ -195,6 +213,7 @@ typedef struct SavedTransactionCharacteristics #define XACT_XINFO_HAS_AE_LOCKS (1U << 6) #define XACT_XINFO_HAS_GID (1U << 7) #define XACT_XINFO_HAS_DROPPED_STATS (1U << 8) +#define XACT_XINFO_HAS_FILEOPS (1U << 9) /* * Also stored in xinfo, these indicating a variety of additional actions that @@ -368,8 +387,24 @@ typedef struct xl_xact_prepare uint16 gidlen; /* length of the GID - GID follows the header */ XLogRecPtr origin_lsn; /* lsn of this record at origin node */ TimestampTz origin_timestamp; /* time of prepare at origin node */ + + /* + * UNDO chain head LSN per persistence level (3 == NUndoPersistenceLevels + * from undodefs.h; hardcoded here to keep xact.h free of UNDO headers). + * If NUndoPersistenceLevels changes, this array must be updated and both + * XLOG_PAGE_MAGIC (xlog_internal.h) and TWOPHASE_MAGIC (twophase.c) must + * be bumped. See StaticAssertDecl in xactundo.c for compile-time guard. + */ + XLogRecPtr last_batch_lsn[3]; } xl_xact_prepare; +/* + * xl_xact_prepare is written into WAL as part of XLOG_XACT_PREPARE records, + * and into 2PC state files via TwoPhaseFileHeader. Any layout change + * requires bumping both XLOG_PAGE_MAGIC (xlog_internal.h) and TWOPHASE_MAGIC + * (twophase.c). + */ + /* * Commit/Abort records in the above form are a bit verbose to parse, so * there's a deconstructed versions generated by ParseCommit/AbortRecord() for @@ -430,6 +465,10 @@ typedef struct xl_xact_parsed_abort XLogRecPtr origin_lsn; TimestampTz origin_timestamp; + + /* FILEOPS inverses to apply on redo (XACT_XINFO_HAS_FILEOPS) */ + uint32 fileops_len; + const char *fileops; } xl_xact_parsed_abort; @@ -439,6 +478,8 @@ typedef struct xl_xact_parsed_abort */ extern bool IsTransactionState(void); extern bool IsAbortedTransactionBlockState(void); +extern int EnterInlineUndoApplyState(void); +extern void LeaveInlineUndoApplyState(int saved); extern TransactionId GetTopTransactionId(void); extern TransactionId GetTopTransactionIdIfAny(void); extern TransactionId GetCurrentTransactionId(void); @@ -519,7 +560,8 @@ extern XLogRecPtr XactLogAbortRecord(TimestampTz abort_time, int ndroppedstats, xl_xact_stats_item *droppedstats, int xactflags, TransactionId twophase_xid, - const char *twophase_gid); + const char *twophase_gid, + const char *fileops, uint32 fileops_len); extern void xact_redo(XLogReaderState *record); /* xactdesc.c */ diff --git a/src/include/access/xactundo.h b/src/include/access/xactundo.h new file mode 100644 index 0000000000000..d624e63af5785 --- /dev/null +++ b/src/include/access/xactundo.h @@ -0,0 +1,88 @@ +/*------------------------------------------------------------------------- + * + * xactundo.h + * Transaction-level undo management + * + * This module manages per-transaction undo record sets. It maintains + * up to NUndoPersistenceLevels (3) record sets per transaction -- one + * for each persistence level (permanent, unlogged, temporary), so that + * records for different persistence levels are kept separate. + * + * Code that wants to write transactional undo should interface with + * these functions rather than manipulating UndoRecordSet directly. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/xactundo.h + * + *------------------------------------------------------------------------- + */ +#ifndef XACTUNDO_H +#define XACTUNDO_H + +#include "access/undodefs.h" +#include "access/undorecord.h" +#include "access/xlogdefs.h" + +/* + * XactUndoContext - Context for a single undo insertion within a transaction. + * + * Created by PrepareXactUndoData(), consumed by InsertXactUndoData() + * and cleaned up by CleanupXactUndoInsertion(). The plevel tracks which + * persistence-level record set this insertion belongs to. + */ +typedef struct XactUndoContext +{ + UndoPersistenceLevel plevel; + UndoRecordSet *uset; /* borrowed reference, do not free */ +} XactUndoContext; + +/* Shared memory initialization */ +extern Size XactUndoShmemSize(void); +extern void XactUndoShmemRequest(void); +extern void XactUndoShmemInit(void); + +/* Per-backend initialization */ +extern void InitializeXactUndo(void); + +/* + * Undo insertion API for any AM or subsystem. + * + * PrepareXactUndoData: Find or create the appropriate per-persistence-level + * UndoRecordSet for the current transaction and prepare it for a new + * record. Returns the UndoRecPtr where the record will be written. + * + * Parameters are AM-agnostic: the caller provides an RM ID, RM-specific + * info flags, a relation OID, and an opaque payload. + * + * InsertXactUndoData: Actually write the record data into the undo log. + * + * CleanupXactUndoInsertion: Release any resources held by the context. + */ +extern UndoRecPtr PrepareXactUndoData(XactUndoContext *ctx, + char persistence, + uint8 rmid, + uint16 info, + Oid reloid, + const char *payload, + Size payload_len); +extern void InsertXactUndoData(XactUndoContext *ctx); +extern void CleanupXactUndoInsertion(XactUndoContext *ctx); + +/* Transaction lifecycle hooks */ +extern void AtCommit_XactUndo(void); +extern void AtAbort_XactUndo(void); +extern void AtPrepare_XactUndo(void); +extern void AtSubCommit_XactUndo(int level); +extern void AtSubAbort_XactUndo(int level); +extern void AtProcExit_XactUndo(void); + + +/* Checkpoint: WAL-log the UNDO chains of in-flight transactions */ +extern void LogXactUndoInflight(void); + +/* Undo chain traversal for rollback */ +extern XLogRecPtr GetCurrentXactLastBatchLSN(UndoPersistenceLevel plevel); + +#endif /* XACTUNDO_H */ diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index bf609c3c703d9..949bfd496c81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -32,7 +32,7 @@ /* * Each page of XLOG file has a header like this: */ -#define XLOG_PAGE_MAGIC 0xD122 /* can be used as WAL version indicator */ +#define XLOG_PAGE_MAGIC 0xD12E /* can be used as WAL version indicator */ typedef struct XLogPageHeaderData { diff --git a/src/include/port/pg_xattr.h b/src/include/port/pg_xattr.h new file mode 100644 index 0000000000000..7159fc27a26a1 --- /dev/null +++ b/src/include/port/pg_xattr.h @@ -0,0 +1,40 @@ +/*------------------------------------------------------------------------- + * + * pg_xattr.h + * Cross-platform extended attribute abstraction + * + * Provides pg_setxattr() and pg_removexattr() that work across: + * - Linux: setxattr/removexattr + * - macOS: setxattr/removexattr (extra options param) + * - FreeBSD: extattr_set_file/extattr_delete_file + * - Windows: NTFS Alternate Data Streams + * - Fallback: returns ENOTSUP with WARNING + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * src/include/port/pg_xattr.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_XATTR_H +#define PG_XATTR_H + +/* + * ENODATA is Linux-specific. FreeBSD/macOS use ENOATTR for "attribute not + * found". Provide a portable PG_ENOATTR so callers don't need #ifdefs. + */ +#if defined(ENOATTR) +#define PG_ENOATTR ENOATTR +#elif defined(ENODATA) +#define PG_ENOATTR ENODATA +#else +#define PG_ENOATTR ENOENT /* last-resort fallback */ +#endif + +extern int pg_setxattr(const char *path, const char *name, + const void *value, size_t size); +extern ssize_t pg_getxattr(const char *path, const char *name, + void *value, size_t size); +extern int pg_removexattr(const char *path, const char *name); + +#endif /* PG_XATTR_H */ diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index b8e9b7f109312..b40c825013c37 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -322,6 +322,7 @@ extern bool BufferBeginSetHintBits(Buffer buffer); extern void BufferFinishSetHintBits(Buffer buffer, bool mark_dirty, bool buffer_std); extern void UnlockBuffers(void); +extern void BufferLockReleaseAll(void); extern void UnlockBuffer(Buffer buffer); extern void LockBufferInternal(Buffer buffer, BufferLockMode mode); diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 40849c2ef9947..b979ee4258b7d 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -22,7 +22,19 @@ typedef enum FileCopyMethod /* GUC parameters */ extern PGDLLIMPORT int file_copy_method; -extern void copydir(const char *fromdir, const char *todir, bool recurse); +/* + * copydir(): copy a directory tree. + * + * If register_for_abort_cleanup is true, the destination is created with + * FileOpsMkdirTree(). Forward-path callers in transactional commands + * (CREATE DATABASE STRATEGY=FILE_COPY, ALTER DATABASE SET TABLESPACE) should + * pass true so that a transaction abort, or a crash before commit, removes + * the partial copy. Recovery callers (XLOG_DBASE_CREATE_FILE_COPY redo) must + * pass false because there is no surrounding transaction context and the + * redo handler is itself the recovery action. + */ +extern void copydir(const char *fromdir, const char *todir, bool recurse, + bool register_for_abort_cleanup); extern void copy_file(const char *fromfile, const char *tofile); #endif /* COPYDIR_H */ diff --git a/src/include/storage/fileops.h b/src/include/storage/fileops.h new file mode 100644 index 0000000000000..523f1240f38c6 --- /dev/null +++ b/src/include/storage/fileops.h @@ -0,0 +1,519 @@ +/*------------------------------------------------------------------------- + * + * fileops.h + * Transactional file operations API + * + * This module provides transactional filesystem operations that are + * WAL-logged and integrated with PostgreSQL's transaction management. + * Constructive operations and renames take effect at once and are reversed + * if the transaction aborts; deletions are deferred to transaction end (see + * src/backend/storage/file/README). + * + * The RM_FILEOPS_ID resource manager handles WAL replay for these + * operations, ensuring correct behavior during crash recovery and + * standby replay. + * + * The operation set follows the Berkeley DB fileops.src model: each + * filesystem operation is a composable unit with its own WAL record + * type, redo handler, and descriptor. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/fileops.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILEOPS_H +#define FILEOPS_H + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* + * WAL record types for FILEOPS operations. + * + * The high 4 bits of the info byte are used for record type, + * leaving the low bits for flags (following PostgreSQL convention). + * + * Following the Berkeley DB fileops.src model, each filesystem + * operation has its own WAL record type for independent redo. + */ +#define XLOG_FILEOPS_CREATE 0x00 +#define XLOG_FILEOPS_DELETE 0x10 +#define XLOG_FILEOPS_RENAME 0x20 +#define XLOG_FILEOPS_WRITE 0x30 +#define XLOG_FILEOPS_TRUNCATE 0x40 +#define XLOG_FILEOPS_CHMOD 0x50 +#define XLOG_FILEOPS_CHOWN 0x60 +#define XLOG_FILEOPS_MKDIR 0x70 +#define XLOG_FILEOPS_RMDIR 0x80 +#define XLOG_FILEOPS_SYMLINK 0x90 +#define XLOG_FILEOPS_LINK 0xA0 +#define XLOG_FILEOPS_SETXATTR 0xB0 +#define XLOG_FILEOPS_REMOVEXATTR 0xC0 + +/* + * PendingFileOp - Deferred file operation entry + * + * File operations are collected in a linked list during a transaction + * and executed at commit or abort time. This follows the same pattern + * used by PendingRelDelete in catalog/storage.c. + */ +typedef enum PendingFileOpType +{ + PENDING_FILEOP_CREATE, + PENDING_FILEOP_DELETE, + PENDING_FILEOP_RENAME, + PENDING_FILEOP_WRITE, + PENDING_FILEOP_TRUNCATE, + PENDING_FILEOP_CHMOD, + PENDING_FILEOP_CHOWN, + PENDING_FILEOP_MKDIR, + PENDING_FILEOP_RMDIR, + PENDING_FILEOP_RMTREE, /* abort: remove a FileOpsMkdirTree() tree */ + PENDING_FILEOP_SYMLINK, + PENDING_FILEOP_LINK, + PENDING_FILEOP_SETXATTR, + PENDING_FILEOP_REMOVEXATTR, + PENDING_FILEOP_INVERSE +} PendingFileOpType; + +typedef struct PendingFileOp +{ + PendingFileOpType type; /* operation type */ + char *path; /* primary file path */ + char *newpath; /* old path to restore (RENAME), or xattr name */ + off_t length; /* truncation length, write offset, or mode */ + void *data; /* generic data (e.g., original xattr value) */ + size_t data_len; /* length of data */ + bool at_commit; /* execute at commit (true) or abort (false) */ + int nestLevel; /* transaction nesting level */ + struct PendingFileOp *next; /* linked list link */ +} PendingFileOp; + +/* + * Public API for transactional file operations + * + * These functions handle platform-specific differences automatically: + * - O_DIRECT: PG_O_DIRECT (Linux/FreeBSD native, macOS F_NOCACHE, + * Windows FILE_FLAG_NO_BUFFERING) + * - fsync: pg_fsync() (Linux fdatasync, macOS F_FULLFSYNC, + * BSD fsync, Windows FlushFileBuffers) + * - Directory sync: fsync_parent_path() (Unix only, no-op on Windows) + * + * Operation-specific API functions are declared below their WAL + * record structures in subsequent sections. + */ + +/* Abort WAL carries a length-prefixed sequence of inverse payloads. */ +#define FILEOPS_MAX_ABORT_BYTES (16 * 1024 * 1024) +#define FILEOPS_MAX_DATA_BYTES (1024 * 1024) +#define FILEOPS_TWOPHASE_INVERSES 1 +extern void FileOpsGetAbortData(char **data, uint32 *len); +extern void FileOpsApplyAbort(const char *data, uint32 len); + +/* Scoped to a generic walk whose FILEOPS inverses belong to abort WAL. */ +extern PGDLLIMPORT bool FileOpsSkipUndo; +extern void FileOpsApplyInverse(uint16 info, const char *data, uint32 len); +extern void FileOpsReserveInverseSyncs(void); +extern void FileOpsSyncInverses(int elevel); +extern void FileOpsBeginAbort(void); +extern void FileOpsEndAbort(void); + +/* Transaction lifecycle hooks */ +extern void FileOpsDoPendingOps(bool isCommit); +extern void AtSubCommit_FileOps(void); +extern void AtSubAbort_FileOps(void); +extern void PostPrepare_FileOps(void); +extern void FileOpsAtPrepare(void); + +/* + * xl_fileops_create - WAL record for file creation + * + * Records that a file was created within a transaction. If the transaction + * aborts, the file will be deleted. The path is stored as variable-length + * data following the fixed header. + */ +typedef struct xl_fileops_create +{ + int flags; /* open flags used for creation */ + mode_t mode; /* file permission mode */ + bool register_delete; /* register for delete-on-abort */ + /* variable-length path follows */ +} xl_fileops_create; + +#define SizeOfFileOpsCreate (offsetof(xl_fileops_create, register_delete) + sizeof(bool)) + +/* File creation API */ +extern int FileOpsCreate(const char *path, int flags, mode_t mode, + bool register_delete); + +/* + * xl_fileops_delete - WAL record for file deletion + * + * Records that a file deletion was requested. The at_commit flag indicates + * whether the deletion should happen at commit (true) or was registered + * as a delete-on-abort from a prior create (false). + */ +typedef struct xl_fileops_delete +{ + bool at_commit; /* true = delete at commit, false = at abort */ + /* variable-length path follows */ +} xl_fileops_delete; + +#define SizeOfFileOpsDelete (offsetof(xl_fileops_delete, at_commit) + sizeof(bool)) + +/* File deletion API */ +extern void FileOpsDelete(const char *path, bool at_commit); + +/* + * xl_fileops_rename - WAL record for file rename + * + * Records that a file or directory is being renamed; it is written before + * the rename is made. Both old and new paths are stored as variable-length + * data: oldpath_len bytes of old path, then the new path follows. + */ +typedef struct xl_fileops_rename +{ + uint16 oldpath_len; /* length of old path (including NUL) */ + /* variable-length old path follows, then new path */ +} xl_fileops_rename; + +#define SizeOfFileOpsRename (offsetof(xl_fileops_rename, oldpath_len) + sizeof(uint16)) + +/* + * File rename API: immediate, refuses an existing newpath, renamed back if + * the transaction aborts. + */ +extern int FileOpsRename(const char *oldpath, const char *newpath); + +/* + * xl_fileops_write - WAL record for file write at offset + * + * Records that data was written to a file at a specific offset. + * The path and data are stored as variable-length data following + * the fixed header. + */ +typedef struct xl_fileops_write +{ + off_t offset; /* write offset in file */ + uint32 len; /* data length */ + uint16 path_len; /* length of path (including NUL) */ + /* variable-length path follows, then data */ +} xl_fileops_write; + +#define SizeOfFileOpsWrite (offsetof(xl_fileops_write, path_len) + sizeof(uint16)) + +/* File write API */ +extern int FileOpsWrite(const char *path, off_t offset, + const void *data, uint32 len); + +/* + * xl_fileops_truncate - WAL record for file truncation + */ +typedef struct xl_fileops_truncate +{ + off_t length; /* new file length */ + /* variable-length path follows */ +} xl_fileops_truncate; + +#define SizeOfFileOpsTruncate (offsetof(xl_fileops_truncate, length) + sizeof(off_t)) + +/* File truncation API */ +extern void FileOpsTruncate(const char *path, off_t length); + +/* + * xl_fileops_chmod - WAL record for file permission change + */ +typedef struct xl_fileops_chmod +{ + mode_t mode; /* new permission mode */ + mode_t orig_mode; /* pre-chmod mode (crash-revert before-state) */ + /* variable-length path follows */ +} xl_fileops_chmod; + +#define SizeOfFileOpsChmod (offsetof(xl_fileops_chmod, orig_mode) + sizeof(mode_t)) + +/* + * xl_fileops_chown - WAL record for file ownership change + */ +typedef struct xl_fileops_chown +{ + uid_t uid; /* new owner user id */ + gid_t gid; /* new owner group id */ + uid_t orig_uid; /* pre-chown uid (crash-revert before-state) */ + gid_t orig_gid; /* pre-chown gid (crash-revert before-state) */ + /* variable-length path follows */ +} xl_fileops_chown; + +#define SizeOfFileOpsChown (offsetof(xl_fileops_chown, orig_gid) + sizeof(gid_t)) + +/* File metadata API */ +extern int FileOpsChmod(const char *path, mode_t mode); +extern int FileOpsChown(const char *path, uid_t uid, gid_t gid); + +/* + * xl_fileops_mkdir - WAL record for directory creation + */ +typedef struct xl_fileops_mkdir +{ + mode_t mode; /* directory permission mode */ + uint16 flags; /* FILEOPS_MKDIR_* */ + /* variable-length path follows */ +} xl_fileops_mkdir; + +#define SizeOfFileOpsMkdir (offsetof(xl_fileops_mkdir, flags) + sizeof(uint16)) + +/* + * The transaction fills the directory itself (with files FILEOPS does not + * track, e.g. through smgr), so undoing the mkdir removes the whole tree. + */ +#define FILEOPS_MKDIR_TREE 0x0001 + +/* + * xl_fileops_rmdir - WAL record for directory removal + */ +typedef struct xl_fileops_rmdir +{ + bool at_commit; /* true = rmdir at commit, false = at abort */ + /* variable-length path follows */ +} xl_fileops_rmdir; + +#define SizeOfFileOpsRmdir (offsetof(xl_fileops_rmdir, at_commit) + sizeof(bool)) + +/* Directory lifecycle API */ +extern int FileOpsMkdir(const char *path, mode_t mode); +extern int FileOpsMkdirTree(const char *path, mode_t mode, Oid dboid); +extern void FileOpsRmdir(const char *path, bool at_commit); + +/* + * xl_fileops_symlink - WAL record for symbolic link creation + */ +typedef struct xl_fileops_symlink +{ + uint16 target_len; /* length of target (including NUL) */ + /* variable-length target follows, then linkpath */ +} xl_fileops_symlink; + +#define SizeOfFileOpsSymlink (offsetof(xl_fileops_symlink, target_len) + sizeof(uint16)) + +/* + * xl_fileops_link - WAL record for hard link creation + */ +typedef struct xl_fileops_link +{ + uint16 oldpath_len; /* length of old path (including NUL) */ + /* variable-length old path follows, then new path */ +} xl_fileops_link; + +#define SizeOfFileOpsLink (offsetof(xl_fileops_link, oldpath_len) + sizeof(uint16)) + +/* Link operations API */ +extern int FileOpsSymlink(const char *target, const char *linkpath); +extern int FileOpsLink(const char *oldpath, const char *newpath); + +/* + * xl_fileops_setxattr - WAL record for setting an extended attribute + */ +typedef struct xl_fileops_setxattr +{ + uint16 name_len; /* attribute name length (including NUL) */ + uint32 value_len; /* attribute value length */ + uint16 path_len; /* file path length (including NUL) */ + uint32 orig_value_len; /* pre-set value length (crash-revert) */ + bool had_value; /* true if xattr existed before this set */ + /* variable-length: path, name, value, orig_value (if had_value) */ +} xl_fileops_setxattr; + +#define SizeOfFileOpsSetxattr (offsetof(xl_fileops_setxattr, had_value) + sizeof(bool)) + +/* + * xl_fileops_removexattr - WAL record for removing an extended attribute + */ +typedef struct xl_fileops_removexattr +{ + uint16 name_len; /* attribute name length (including NUL) */ + uint16 path_len; /* file path length (including NUL) */ + uint32 orig_value_len; /* removed value length (crash-revert); 0 = + * none */ + bool had_value; /* true if xattr existed before this remove */ + /* variable-length: path, name, orig_value (if had_value) */ +} xl_fileops_removexattr; + +#define SizeOfFileOpsRemovexattr (offsetof(xl_fileops_removexattr, had_value) + sizeof(bool)) + +/* Extended attribute API */ +extern int FileOpsSetXattr(const char *path, const char *name, + const void *value, size_t len); +extern int FileOpsRemoveXattr(const char *path, const char *name); + +/* WAL redo and descriptor functions */ +extern void fileops_redo(XLogReaderState *record); +extern void fileops_desc(StringInfo buf, XLogReaderState *record); +extern const char *fileops_identify(uint8 info); + +/* FILEOPS UNDO RM (fileops_undo.c) */ +extern void FileopsUndoRmgrInit(void); + +/* + * UNDO resource-manager ID for FILEOPS (stored in urec_rmid). Defined here, + * with FILEOPS, rather than in the generic access/undormgr.h, so the UNDO + * core names no specific consumer. See access/undormgr.h for the shared, + * WAL-durable ID number space. + */ +#define UNDO_RMID_FILEOPS 2 + +/* FILEOPS UNDO subtypes (stored in urec_info) */ +#define FILEOPS_UNDO_WRITE 0x0002 +#define FILEOPS_UNDO_CREATE 0x0001 +#define FILEOPS_UNDO_TRUNCATE 0x0003 +#define FILEOPS_UNDO_CHMOD 0x0004 +#define FILEOPS_UNDO_CHOWN 0x0005 +#define FILEOPS_UNDO_MKDIR 0x0006 +#define FILEOPS_UNDO_SYMLINK 0x0007 +#define FILEOPS_UNDO_LINK 0x0008 +#define FILEOPS_UNDO_SETXATTR 0x0009 +#define FILEOPS_UNDO_REMOVEXATTR 0x000A +#define FILEOPS_UNDO_RENAME 0x000B + +/* + * FILEOPS UNDO payload structures + * + * These structs define the on-disk layout of UNDO record payloads for each + * FILEOPS operation. Variable-length paths and data follow the fixed header. + * + * An inverse can be applied more than once (online at ROLLBACK, and again by + * redo of the abort record after a crash), and the object may have changed in + * between. + * So each payload also records what the operation left behind (its post-state), + * and fileops_undo_apply() reverts only an object still in that state. + */ + +/* CREATE undo payload: the path to unlink, if it is still a regular file */ +typedef struct FileopsUndoCreate +{ + uint16 path_len; /* including NUL */ + /* followed by path */ +} FileopsUndoCreate; + +/* WRITE undo: path, before-image, then after-image. */ +typedef struct FileopsUndoWrite +{ + off_t orig_length; + off_t new_length; + off_t offset; + uint32 before_len; + uint32 after_len; + uint16 path_len; +} FileopsUndoWrite; + +/* TRUNCATE undo payload: path, followed by the bytes removed by a shrink. */ +typedef struct FileopsUndoTruncate +{ + off_t orig_length; + off_t new_length; + uint32 before_len; + uint16 path_len; + /* followed by path */ +} FileopsUndoTruncate; + +/* CHMOD undo payload: path + original and new mode */ +typedef struct FileopsUndoChmod +{ + mode_t orig_mode; + mode_t new_mode; + uint16 path_len; + /* followed by path */ +} FileopsUndoChmod; + +/* CHOWN undo payload: path + original and new uid/gid */ +typedef struct FileopsUndoChown +{ + uid_t orig_uid; + gid_t orig_gid; + uid_t new_uid; + gid_t new_gid; + uint16 path_len; + /* followed by path */ +} FileopsUndoChown; + +/* + * MKDIR undo payload: the path to rmdir, if it is still a directory. + * + * With FILEOPS_MKDIR_TREE the whole tree is removed instead, after dropping + * the shared buffers and pending fsync requests of database dboid (if valid), + * as a database directory's files are written through shared buffers. Since + * that destroys whatever is in the tree, outside recovery it is done only + * while the path is still the directory created (dev, ino): a later walk of + * the chain must not remove a directory created again at the same path, e.g. + * by a retried ALTER DATABASE SET TABLESPACE. Crash recovery cannot rely on + * the inode (redo on a standby or a restored backup creates the directory + * anew), and needs not: until the transaction's locks are gone no other + * transaction can have made that path. Even so, the inode is evidence that + * the directory is ours, not proof. + */ +typedef struct FileopsUndoMkdir +{ + dev_t dev; /* the directory created */ + ino_t ino; + Oid dboid; /* database filling the tree, or InvalidOid */ + uint16 flags; /* FILEOPS_MKDIR_* */ + uint16 path_len; + /* followed by path */ +} FileopsUndoMkdir; + +/* SYMLINK undo payload: the linkpath to unlink, if it still points to target */ +typedef struct FileopsUndoSymlink +{ + uint16 linkpath_len; + uint16 target_len; + /* followed by linkpath, then target */ +} FileopsUndoSymlink; + +/* LINK undo payload: the newpath to unlink, if it is still the linked inode */ +typedef struct FileopsUndoLink +{ + dev_t dev; /* device and inode of the link source */ + ino_t ino; + uint16 newpath_len; + /* followed by newpath */ +} FileopsUndoLink; + +/* + * RENAME undo payload: rename newpath back to oldpath, while newpath exists + * and oldpath does not. Like CREATE and MKDIR, the test is the state of the + * two paths alone, not the device and inode of the object renamed: those are + * not stable across a restore, a promoted standby or redo that makes the + * object again, so ROLLBACK PREPARED on a promoted standby, for one, would + * take its own copy of the object for a different one and leave it renamed. + * The paths are evidence that the object is the one renamed, not proof. + */ +typedef struct FileopsUndoRename +{ + uint16 oldpath_len; /* including NUL */ + uint16 newpath_len; /* including NUL */ + /* followed by oldpath, then newpath */ +} FileopsUndoRename; + +/* SETXATTR undo payload: path + name + original value (if any) + new value */ +typedef struct FileopsUndoSetxattr +{ + uint16 path_len; + uint16 name_len; + uint32 orig_value_len; /* 0 if xattr didn't exist before */ + uint32 new_value_len; + bool had_value; /* true if xattr existed before setxattr */ + /* followed by path, name, original value (if had_value), new value */ +} FileopsUndoSetxattr; + +/* REMOVEXATTR undo payload: path + name + removed value */ +typedef struct FileopsUndoRemovexattr +{ + uint16 path_len; + uint16 name_len; + uint32 value_len; + /* followed by path, name, value */ +} FileopsUndoRemovexattr; + +#endif /* FILEOPS_H */ diff --git a/src/include/storage/subsystemlist.h b/src/include/storage/subsystemlist.h index 9ad619080be22..9b995f9bad799 100644 --- a/src/include/storage/subsystemlist.h +++ b/src/include/storage/subsystemlist.h @@ -88,3 +88,6 @@ PG_SHMEM_SUBSYSTEM(DataChecksumsShmemCallbacks) /* AIO subsystem. This delegates to the method-specific callbacks */ PG_SHMEM_SUBSYSTEM(AioShmemCallbacks) + +/* UNDO subsystem */ +PG_SHMEM_SUBSYSTEM(UndoShmemCallbacks) diff --git a/src/port/Makefile b/src/port/Makefile index 7e9b58776529a..39e1388227adc 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -51,6 +51,7 @@ OBJS = \ pg_popcount_aarch64.o \ pg_popcount_x86.o \ pg_strong_random.o \ + pg_xattr.o \ pgcheckdir.o \ pgmkdirp.o \ pgsleep.o \ diff --git a/src/port/meson.build b/src/port/meson.build index 922b3f646768d..20e1ca0043110 100644 --- a/src/port/meson.build +++ b/src/port/meson.build @@ -14,6 +14,7 @@ pgport_sources = [ 'pg_popcount_aarch64.c', 'pg_popcount_x86.c', 'pg_strong_random.c', + 'pg_xattr.c', 'pgcheckdir.c', 'pgmkdirp.c', 'pgsleep.c', diff --git a/src/port/pg_xattr.c b/src/port/pg_xattr.c new file mode 100644 index 0000000000000..58a3d422817f7 --- /dev/null +++ b/src/port/pg_xattr.c @@ -0,0 +1,217 @@ +/*------------------------------------------------------------------------- + * + * pg_xattr.c + * Cross-platform extended attribute abstraction + * + * Platform detection uses compiler-defined macros rather than + * configure-time checks, avoiding meson.build/configure.ac changes. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/port/pg_xattr.c + * + *------------------------------------------------------------------------- + */ +#include "c.h" + +#include +#include "port/pg_xattr.h" + +/* + * Platform detection via compiler macros. + * Linux and macOS both provide but with different APIs. + * FreeBSD uses . Windows uses NTFS Alternate Data Streams. + */ +#if defined(__linux__) || defined(__APPLE__) +#include +#define PG_HAVE_XATTR 1 +#elif defined(__FreeBSD__) +#include +#include +#define PG_HAVE_EXTATTR 1 +#elif defined(WIN32) +#define PG_HAVE_ADS 1 +#endif + +#if defined(PG_HAVE_ADS) +/* + * Set errno for a failed operation on the stream "path:name". A missing + * stream fails with ERROR_FILE_NOT_FOUND, like a missing file; report it as + * PG_ENOATTR when the file itself exists, as the other platforms do. + */ +static void +pg_ads_set_errno(const char *path, DWORD err) +{ + _dosmaperr(err); + if (errno == ENOENT && GetFileAttributesA(path) != INVALID_FILE_ATTRIBUTES) + errno = PG_ENOATTR; +} +#endif + +/* + * pg_setxattr - Set an extended attribute on a file + * + * Returns 0 on success, -1 on failure (errno set). + */ +int +pg_setxattr(const char *path, const char *name, + const void *value, size_t size) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return setxattr(path, name, value, size, 0, 0); +#else + return setxattr(path, name, value, size, 0); +#endif + +#elif defined(PG_HAVE_EXTATTR) + ssize_t ret; + + ret = extattr_set_file(path, EXTATTR_NAMESPACE_USER, + name, value, size); + return (ret >= 0) ? 0 : -1; + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + HANDLE hFile; + DWORD written; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + + hFile = CreateFileA(ads_path, GENERIC_WRITE, 0, NULL, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) + { + _dosmaperr(GetLastError()); + return -1; + } + + if (!WriteFile(hFile, value, (DWORD) size, &written, NULL) || + written != (DWORD) size) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + CloseHandle(hFile); + return 0; + +#else + /* Unsupported platform: succeed in WAL but no-op locally */ + (void) path; + (void) name; + (void) value; + (void) size; + errno = ENOTSUP; + return -1; +#endif +} + +/* + * pg_getxattr - Get an extended attribute value from a file + * + * Returns the number of bytes placed in value on success, + * or -1 on failure (errno set). If value is NULL or size is 0, + * returns the size of the attribute value without reading it. + */ +ssize_t +pg_getxattr(const char *path, const char *name, + void *value, size_t size) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return getxattr(path, name, value, size, 0, 0); +#else + return getxattr(path, name, value, size); +#endif + +#elif defined(PG_HAVE_EXTATTR) + return extattr_get_file(path, EXTATTR_NAMESPACE_USER, + name, value, size); + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + HANDLE hFile; + DWORD bytesRead; + LARGE_INTEGER fileSize; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + + hFile = CreateFileA(ads_path, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) + { + pg_ads_set_errno(path, GetLastError()); + return -1; + } + + if (!GetFileSizeEx(hFile, &fileSize)) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + if (value == NULL || size == 0) + { + CloseHandle(hFile); + return (ssize_t) fileSize.QuadPart; + } + + if (!ReadFile(hFile, value, (DWORD) size, &bytesRead, NULL)) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + CloseHandle(hFile); + return (ssize_t) bytesRead; + +#else + (void) path; + (void) name; + (void) value; + (void) size; + errno = ENOTSUP; + return -1; +#endif +} + +/* + * pg_removexattr - Remove an extended attribute from a file + * + * Returns 0 on success, -1 on failure (errno set). + */ +int +pg_removexattr(const char *path, const char *name) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return removexattr(path, name, 0); +#else + return removexattr(path, name); +#endif + +#elif defined(PG_HAVE_EXTATTR) + return extattr_delete_file(path, EXTATTR_NAMESPACE_USER, name); + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + if (DeleteFileA(ads_path)) + return 0; + + pg_ads_set_errno(path, GetLastError()); + return -1; + +#else + (void) path; + (void) name; + errno = ENOTSUP; + return -1; +#endif +} diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 6c1504ac9323a..f16d803fa2c85 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -32,6 +32,7 @@ SUBDIRS = \ test_escape \ test_extensible \ test_extensions \ + test_fileops \ test_ginpostinglist \ test_int128 \ test_integerset \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index ef3cdb8faf9c9..49cc09d041b6f 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -34,6 +34,7 @@ subdir('test_dsm_registry') subdir('test_escape') subdir('test_extensible') subdir('test_extensions') +subdir('test_fileops') subdir('test_ginpostinglist') subdir('test_int128') subdir('test_integerset') diff --git a/src/test/modules/test_fileops/Makefile b/src/test/modules/test_fileops/Makefile new file mode 100644 index 0000000000000..69100c7527288 --- /dev/null +++ b/src/test/modules/test_fileops/Makefile @@ -0,0 +1,19 @@ +MODULE_big = test_fileops +OBJS = test_fileops.o +EXTENSION = test_fileops +DATA = test_fileops--1.0.sql +PGFILEDESC = "test_fileops - test transactional file operations" +REGRESS = test_fileops +TAP_TESTS = 1 +EXTRA_INSTALL = src/test/modules/injection_points +export enable_injection_points +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_fileops +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_fileops/expected/test_fileops.out b/src/test/modules/test_fileops/expected/test_fileops.out new file mode 100644 index 0000000000000..b529cfe3bb29f --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops.out @@ -0,0 +1,413 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif +CREATE EXTENSION test_fileops; +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; + initial_size +-------------- + 1024 +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + initial_mode +-------------- + 384 +(1 row) + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; + size_during_txn +----------------- + 256 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + size_after_rollback +--------------------- + 1024 +(1 row) + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); + test_fileops_truncate +----------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + size_after_commit +------------------- + 512 +(1 row) + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + mode_baseline +--------------- + 420 +(1 row) + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; + mode_during_txn +----------------- + 384 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + mode_after_rollback +--------------------- + 420 +(1 row) + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal + test_fileops_chmod +-------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; + link_during_txn +----------------- + t +(1 row) + +ROLLBACK; +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + link_after_rollback +--------------------- + f +(1 row) + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + link_after_commit +------------------- + t +(1 row) + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + size_before_test7 +------------------- + 1024 +(1 row) + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + test_fileops_truncate +----------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; + size_in_savepoint +------------------- + 100 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; + size_after_rollback_to +------------------------ + 800 +(1 row) + +COMMIT; +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + size_after_commit +------------------- + 800 +(1 row) + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + test_fileops_chmod +-------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; + mode_in_savepoint +------------------- + 256 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; + mode_after_rollback_to +------------------------ + 448 +(1 row) + +COMMIT; +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; + link2_in_savepoint +-------------------- + t +(1 row) + +ROLLBACK TO sp1; +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; + link2_after_rollback_to +------------------------- + f +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + link2_after_commit +-------------------- + f +(1 row) + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset +\if :xattr_supported +-- Test 10a: SetXattr UNDO - rollback removes newly-set xattr +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.rollback_test', 'will_vanish'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_during_txn; + xattr_during_txn +------------------ + will_vanish +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_after_rollback; + xattr_after_rollback +---------------------- + +(1 row) + +-- Test 10b: SetXattr overwrite UNDO - rollback restores old value +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'overwritten'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_overwritten; + xattr_overwritten +------------------- + overwritten +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_restored; + xattr_restored +---------------- + initial_value +(1 row) + +-- Test 10c: RemoveXattr UNDO - rollback restores removed xattr +BEGIN; +SELECT test_fileops_removexattr(:'filepath_a', 'user.test_key'); + test_fileops_removexattr +-------------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove; + xattr_after_remove +-------------------- + +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove_rollback; + xattr_after_remove_rollback +----------------------------- + initial_value +(1 row) + +-- Test 10d: Xattr in subtransaction +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_setxattr(:'filepath_a', 'user.sub_key', 'sub_value'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_in_savepoint; + xattr_in_savepoint +-------------------- + sub_value +(1 row) + +ROLLBACK TO sp1; +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_after_sp_rollback; + xattr_after_sp_rollback +------------------------- + +(1 row) + +COMMIT; +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; +\endif +-- ================================================================ +-- Cleanup +-- ================================================================ +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/expected/test_fileops_1.out b/src/test/modules/test_fileops/expected/test_fileops_1.out new file mode 100644 index 0000000000000..41193304cd627 --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops_1.out @@ -0,0 +1,332 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif +CREATE EXTENSION test_fileops; +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; + initial_size +-------------- + 1024 +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + initial_mode +-------------- + 384 +(1 row) + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; + size_during_txn +----------------- + 256 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + size_after_rollback +--------------------- + 1024 +(1 row) + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); + test_fileops_truncate +----------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + size_after_commit +------------------- + 512 +(1 row) + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + mode_baseline +--------------- + 420 +(1 row) + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; + mode_during_txn +----------------- + 384 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + mode_after_rollback +--------------------- + 420 +(1 row) + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal + test_fileops_chmod +-------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; + link_during_txn +----------------- + t +(1 row) + +ROLLBACK; +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + link_after_rollback +--------------------- + f +(1 row) + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + link_after_commit +------------------- + t +(1 row) + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + size_before_test7 +------------------- + 1024 +(1 row) + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + test_fileops_truncate +----------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; + size_in_savepoint +------------------- + 100 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; + size_after_rollback_to +------------------------ + 800 +(1 row) + +COMMIT; +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + size_after_commit +------------------- + 800 +(1 row) + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + test_fileops_chmod +-------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; + mode_in_savepoint +------------------- + 256 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; + mode_after_rollback_to +------------------------ + 448 +(1 row) + +COMMIT; +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; + link2_in_savepoint +-------------------- + t +(1 row) + +ROLLBACK TO sp1; +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; + link2_after_rollback_to +------------------------- + f +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + link2_after_commit +-------------------- + f +(1 row) + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset +\if :xattr_supported +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; + notice +----------------------------------------------------------------- + xattr tests skipped (not supported on this platform/filesystem) +(1 row) + +\endif +-- ================================================================ +-- Cleanup +-- ================================================================ +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/expected/test_fileops_2.out b/src/test/modules/test_fileops/expected/test_fileops_2.out new file mode 100644 index 0000000000000..d25aa81d54bab --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops_2.out @@ -0,0 +1,18 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit diff --git a/src/test/modules/test_fileops/meson.build b/src/test/modules/test_fileops/meson.build new file mode 100644 index 0000000000000..e8f3210277e3b --- /dev/null +++ b/src/test/modules/test_fileops/meson.build @@ -0,0 +1,55 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +test_fileops_sources = files( + 'test_fileops.c', +) + +if host_system == 'windows' + test_fileops_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_fileops', + '--FILEDESC', 'test_fileops - regression testing of FILEOPS subsystem',]) +endif + +test_fileops = shared_module('test_fileops', + test_fileops_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_fileops + +test_install_data += files( + 'test_fileops.control', + 'test_fileops--1.0.sql', +) + +tests += { + 'name': 'test_fileops', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_fileops', + ], + }, + 'tap': { + 'env': { + 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', + }, + 'tests': [ + 't/001_fileops_apply_failure.pl', + 't/004_fileops_fs_errors.pl', + 't/005_fileops_checkpoint_loser.pl', + 't/007_fileops_commit_crash.pl', + 't/008_fileops_standby_abort.pl', + 't/010_fileops_abort_redo.pl', + 't/018_fileops_rename.pl', + 't/023_fileops_inline_abort_ctx.pl', + 't/024_fileops_recovery_apply_failure.pl', + 't/025_fileops_create_existing.pl', + 't/026_fileops_abort_checkpoint_order.pl', + 't/027_fileops_crash_loser_standby.pl', + 't/028_fileops_insert_holdoff.pl', + 't/029_fileops_crash_loser_recycled_wal.pl', + 't/030_fileops_rollback_record.pl', + ], + }, +} diff --git a/src/test/modules/test_fileops/sql/test_fileops.sql b/src/test/modules/test_fileops/sql/test_fileops.sql new file mode 100644 index 0000000000000..ef1a10523d10f --- /dev/null +++ b/src/test/modules/test_fileops/sql/test_fileops.sql @@ -0,0 +1,229 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif + +CREATE EXTENSION test_fileops; + +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ + +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset + +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ + +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; +ROLLBACK; + +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); +COMMIT; + +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ + +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; +ROLLBACK; + +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal +COMMIT; + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ + +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset + +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; +ROLLBACK; + +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); +COMMIT; + +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ + +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; +ROLLBACK TO sp1; + +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; +COMMIT; + +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ + +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; +ROLLBACK TO sp1; + +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; +COMMIT; + +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ + +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset + +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; +ROLLBACK TO sp1; + +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; +COMMIT; + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ + +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset + +\if :xattr_supported + +-- Test 10a: SetXattr UNDO - rollback removes newly-set xattr +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.rollback_test', 'will_vanish'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_during_txn; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_after_rollback; + +-- Test 10b: SetXattr overwrite UNDO - rollback restores old value +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'overwritten'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_overwritten; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_restored; + +-- Test 10c: RemoveXattr UNDO - rollback restores removed xattr +BEGIN; +SELECT test_fileops_removexattr(:'filepath_a', 'user.test_key'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove_rollback; + +-- Test 10d: Xattr in subtransaction +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_setxattr(:'filepath_a', 'user.sub_key', 'sub_value'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_in_savepoint; +ROLLBACK TO sp1; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_after_sp_rollback; +COMMIT; + +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; +\endif + +-- ================================================================ +-- Cleanup +-- ================================================================ + +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl b/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl new file mode 100644 index 0000000000000..77a131292640a --- /dev/null +++ b/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl @@ -0,0 +1,38 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +# A FILEOPS inverse that cannot be applied at ROLLBACK is reported as a +# failure. The inverse rides in the abort record, so crash replay retries it. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_apply_failure'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf( + 'postgresql.conf', q{ +wal_level = replica +}); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $path = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('fileops_fail.dat')"); + +my $bg = $node->background_psql('postgres'); +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_truncate('$path', 10)"); +chmod(0444, $path) or die "chmod $path: $!"; +$bg->query('ROLLBACK'); +$bg->quit; + +like( + slurp_file($node->logfile), + qr/FILEOPS UNDO TRUNCATE: could not open/, + 'the failure was reported'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/004_fileops_fs_errors.pl b/src/test/modules/test_fileops/t/004_fileops_fs_errors.pl new file mode 100644 index 0000000000000..7d947c62b9fb6 --- /dev/null +++ b/src/test/modules/test_fileops/t/004_fileops_fs_errors.pl @@ -0,0 +1,107 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +# An ordinary filesystem error in an immediate FileOps operation (permission +# denied, no space, ...) must be a plain ERROR, exactly as without FILEOPS: +# the server must not PANIC, and the failed operation must leave nothing in +# the WAL for a standby to replay. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($windows_os) +{ + plan skip_all => 'POSIX permission semantics required'; +} + +my $node = PostgreSQL::Test::Cluster->new('fileops_fs_errors'); +$node->init; +$node->append_conf('postgresql.conf', "wal_level = replica\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; +my $ro = "$datadir/ro_dir"; +mkdir($ro) or die "mkdir $ro: $!"; +my $file = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('fserr.dat')"); +my $rofile = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('fserr_ro.dat')"); +my $has_xattr = $node->safe_psql('postgres', + "SELECT test_fileops_setxattr('$rofile', 'user.pgtest', 'v')"); +chmod(0500, $ro) or die "chmod $ro: $!"; +chmod(0444, $rofile) or die "chmod $rofile: $!"; + +my $start_time = + $node->safe_psql('postgres', 'SELECT pg_postmaster_start_time()'); +my $start_lsn = + $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); + +my %cases = ( + create => "SELECT test_fileops_create('$ro/new.dat', 384)", + write => "SELECT test_fileops_write('/dev/full', 0, 'x')", + truncate => "SELECT test_fileops_truncate('$rofile', 10)", + chmod => "SELECT test_fileops_chmod('/', 448)", + chown => "SELECT test_fileops_chown('$file', 0, 0)", + mkdir => "SELECT test_fileops_mkdir('$ro/sub', 448)", + symlink => "SELECT test_fileops_symlink('$file', '$ro/lnk')", + link => "SELECT test_fileops_link('$file', '$ro/hard')",); + +if ($has_xattr eq 't') +{ + $cases{setxattr} = + "SELECT test_fileops_setxattr('$rofile', 'user.pgtest2', 'v')"; + $cases{removexattr} = + "SELECT test_fileops_removexattr('$rofile', 'user.pgtest')"; +} + +foreach my $op (sort keys %cases) +{ + my ($ret, $stdout, $stderr) = $node->psql('postgres', $cases{$op}); + + # The xattr wrappers report a refused operation as false. + ok( ($stderr =~ /ERROR:/ && $stderr !~ /PANIC|FATAL/) + || ($stdout eq 'f' && $stderr eq ''), + "$op: a filesystem error is an ERROR, not a PANIC" + ) or diag("stdout: $stdout\nstderr: $stderr"); + is($node->safe_psql('postgres', 'SELECT pg_postmaster_start_time()'), + $start_time, "$op: the server did not restart"); +} + +unlike( + slurp_file($node->logfile), + qr/PANIC|terminated by signal/, + 'no PANIC or backend crash in the server log'); + +# CREATE TABLESPACE on a directory we do not own fails as it does without +# FILEOPS. "/" is owned by root, and the server never runs as root. +my ($ret, $stdout, $stderr) = + $node->psql('postgres', "CREATE TABLESPACE t LOCATION '/'"); +like( + $stderr, + qr/ERROR: could not set permissions on directory "\/": (?:Operation not permitted|Read-only file system)/, + 'CREATE TABLESPACE on a root-owned directory gives the usual error'); +is($node->safe_psql('postgres', 'SELECT pg_postmaster_start_time()'), + $start_time, 'the server survived CREATE TABLESPACE'); + +# Nothing the failed operations attempted reached the WAL. +my $end_lsn = + $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +my ($walout, $walerr); +IPC::Run::run [ + 'pg_waldump', + '--path' => "$datadir/pg_wal", + '--start' => $start_lsn, + '--end' => $end_lsn, + '--rmgr' => 'FileOps' + ], + '>' => \$walout, + '2>' => \$walerr; +is($walout, '', 'no FILEOPS WAL record was written for a failed operation'); + +chmod(0700, $ro); +chmod(0644, $rofile); +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/005_fileops_checkpoint_loser.pl b/src/test/modules/test_fileops/t/005_fileops_checkpoint_loser.pl new file mode 100644 index 0000000000000..034ac1ccc2a9f --- /dev/null +++ b/src/test/modules/test_fileops/t/005_fileops_checkpoint_loser.pl @@ -0,0 +1,38 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A transaction that the crash leaves in progress must be rolled back even +# when all of its UNDO precedes the last checkpoint's redo pointer. Redo +# starts there, so it sees the chain only through the checkpoint's +# XLOG_UNDO_INFLIGHT record. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('checkpoint_loser'); +$node->init; +$node->append_conf('postgresql.conf', + "wal_level = replica\nautovacuum = off\ncheckpoint_timeout = '1h'\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $pd = $node->data_dir; +foreach my $ckpt (0, 1) +{ + my $f = "loser_$ckpt.dat"; + my $bg = $node->background_psql('postgres'); + $bg->query_until(qr/ready/, + "BEGIN;\nSELECT test_fileops_create('$f', 384);\n\\echo ready\n"); + $node->safe_psql('postgres', 'CHECKPOINT') if $ckpt; + + # Crash before closing the client, so the backend cannot roll back first. + $node->stop('immediate'); + $bg->{run}->kill_kill; + $node->start; + ok(!-e "$pd/$f", + "checkpoint=$ckpt: the loser's file is removed after a crash"); +} + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl new file mode 100644 index 0000000000000..f40dcd271b05b --- /dev/null +++ b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl @@ -0,0 +1,95 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +# A deferred commit-time file operation must survive a crash right after +# its transaction's commit record: the commit is durable, so the deletion, +# rename or rmdir it implies must happen too, on restart if not before. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('fileops_commit_crash'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +shared_preload_libraries = 'injection_points' +wal_level = replica +}); +$node->start; +if (!$node->check_extension('injection_points')) +{ + plan skip_all => 'Extension injection_points not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $datadir = $node->data_dir; + +# Commit $sql, crash right after the commit record, restart. +sub commit_and_crash +{ + my ($sql) = @_; + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-commit-before-pending-ops', 'wait')" + ); + my $bg = $node->background_psql('postgres', on_error_stop => 0); + $bg->query_until(qr/start/, "\\echo start\n$sql\n"); + $node->wait_for_event('client backend', + 'fileops-commit-before-pending-ops'); + $node->stop('immediate'); + $bg->{run}->finish; + $node->start; +} + +$node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('del.dat')"); +$node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('ren.dat')"); +$node->safe_psql('postgres', "SELECT test_fileops_mkdir('rmd', 448)"); +commit_and_crash( + q{BEGIN; +SELECT test_fileops_delete('del.dat'); +SELECT test_fileops_rename('ren.dat', 'ren2.dat'); +SELECT test_fileops_rmdir('rmd'); +COMMIT;}); +TODO: +{ + # The deferred ops are WAL-logged only when executed, after the commit + # record, so this crash loses them. Their intent must be durable before + # the commit record and executed by commit redo. + local $TODO = + 'commit-time FILEOPS ops are logged after the commit record'; + ok(!-e "$datadir/del.dat", + 'committed FileOpsDelete happened despite the crash'); + ok( !-e "$datadir/ren.dat" && -e "$datadir/ren2.dat", + 'committed FileOpsRename happened despite the crash'); + ok(!-e "$datadir/rmd", + 'committed FileOpsRmdir happened despite the crash'); +} + +# DROP TABLESPACE is safe: its XLOG_TBLSPC_DROP precedes the commit record +# and its redo removes the directories. (In-place, so they are under +# pg_tblspc.) +$node->safe_psql('postgres', + 'SET allow_in_place_tablespaces = on; CREATE TABLESPACE ts LOCATION \'\'' +); +my $tsoid = $node->safe_psql('postgres', + "SELECT oid FROM pg_tablespace WHERE spcname = 'ts'"); +ok(-d "$datadir/pg_tblspc/$tsoid", 'tablespace directory exists'); +commit_and_crash('DROP TABLESPACE ts;'); +is( $node->safe_psql( + 'postgres', "SELECT count(*) FROM pg_tablespace WHERE spcname = 'ts'" + ), + '0', + 'DROP TABLESPACE committed'); +ok(!-e "$datadir/pg_tblspc/$tsoid", + 'committed DROP TABLESPACE removed its directory despite the crash'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/008_fileops_standby_abort.pl b/src/test/modules/test_fileops/t/008_fileops_standby_abort.pl new file mode 100644 index 0000000000000..aee36a32fc919 --- /dev/null +++ b/src/test/modules/test_fileops/t/008_fileops_standby_abort.pl @@ -0,0 +1,89 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', "max_prepared_transactions = 5\n"); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->safe_psql('postgres', 'CREATE EXTENSION injection_points') + if $ENV{enable_injection_points} eq 'yes'; +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; +$primary->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_create('base.dat', 384); +SELECT test_fileops_write('base.dat', 0, 'committed-base'); +COMMIT; +BEGIN; +SELECT test_fileops_create('abort.dat', 384); +SELECT test_fileops_write('abort.dat', 0, 'aborted'); +ROLLBACK; +BEGIN; SELECT test_fileops_mkdir('abortdir', 448); ROLLBACK; +BEGIN; SELECT test_fileops_chmod('base.dat', 448); ROLLBACK; +BEGIN; SELECT test_fileops_write('base.dat', 0, 'OVERWRITTEN!!!'); ROLLBACK; +BEGIN; SELECT test_fileops_truncate('base.dat', 3); ROLLBACK; +}); +$primary->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_write('base.dat', 0, 'prepared'); +SELECT test_fileops_truncate('base.dat', 2); +PREPARE TRANSACTION 'p'; +ROLLBACK PREPARED 'p'; +BEGIN; +SAVEPOINT a; +SELECT test_fileops_chmod('base.dat', 448); +ROLLBACK TO a; +COMMIT; +}); +$primary->wait_for_catchup($standby); +if ($ENV{enable_injection_points} eq 'yes') +{ + $standby->safe_psql('postgres', + "SELECT injection_points_attach('fileops-abort-after-flush', 'wait')" + ); + $primary->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_create('standby-window.dat', 384); +ROLLBACK; +}); + $standby->wait_for_event('startup', 'fileops-abort-after-flush'); + ok(-e $standby->data_dir . '/standby-window.dat', + 'standby paused before inverse'); + $standby->stop('immediate'); + $standby->start; + $primary->wait_for_catchup($standby); + ok(!-e $standby->data_dir . '/standby-window.dat', + 'standby crash replays inverse'); +} +for my $stage ('streaming', 'promoted') +{ + $standby->promote if $stage eq 'promoted'; + for my $node ($primary, $standby) + { + my $dir = $node->data_dir; + my $name = $node->name; + ok(!-e "$dir/abort.dat", "$stage $name: aborted create absent"); + ok(!-e "$dir/abortdir", "$stage $name: aborted mkdir absent"); + SKIP: + { + skip 'POSIX permission semantics required', 1 if $windows_os; + is((stat("$dir/base.dat"))[2] & 07777, + 0600, "$stage $name: original mode"); + } + is(slurp_file("$dir/base.dat"), + 'committed-base', "$stage $name: original bytes"); + } +} +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/010_fileops_abort_redo.pl b/src/test/modules/test_fileops/t/010_fileops_abort_redo.pl new file mode 100644 index 0000000000000..11a33c94c589b --- /dev/null +++ b/src/test/modules/test_fileops/t/010_fileops_abort_redo.pl @@ -0,0 +1,202 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('abort_redo'); +$node->init; +$node->append_conf('postgresql.conf', + "wal_level = replica\nautovacuum = off\ncheckpoint_timeout = '1h'\nmax_prepared_transactions = 5\n" +); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $dir = $node->data_dir; +$node->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_create('base.dat', 384); +SELECT test_fileops_write('base.dat', 0, 'committed-base'); +COMMIT; +CHECKPOINT; +BEGIN; +SELECT test_fileops_create('abort.dat', 384); +SELECT test_fileops_write('abort.dat', 0, 'aborted'); +ROLLBACK; +BEGIN; SELECT test_fileops_mkdir('abortdir', 448); ROLLBACK; +BEGIN; SELECT test_fileops_chmod('base.dat', 448); ROLLBACK; +BEGIN; SELECT test_fileops_write('base.dat', 0, 'OVERWRITTEN!!!'); ROLLBACK; +BEGIN; SELECT test_fileops_truncate('base.dat', 3); ROLLBACK; +BEGIN; +SAVEPOINT s; +SELECT test_fileops_create('sub.dat', 384); +ROLLBACK TO s; +SELECT test_fileops_write('base.dat', 0, 'parent'); +ROLLBACK; +BEGIN; SELECT test_fileops_symlink('base.dat', 'abort.sym'); ROLLBACK; +BEGIN; SELECT test_fileops_link('base.dat', 'abort.link'); ROLLBACK; + +}); +my $xattrs = $node->safe_psql('postgres', + "SELECT test_fileops_setxattr('base.dat', 'user.abort_test', 'original')") + eq 't'; +if ($xattrs) +{ + $node->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_setxattr('base.dat', 'user.abort_test', 'changed'); +ROLLBACK; +BEGIN; +SELECT test_fileops_removexattr('base.dat', 'user.abort_test'); +ROLLBACK; +}); +} +my ($uid, $gid) = (stat("$dir/base.dat"))[ 4, 5 ]; +# An unprivileged test can chown to its own identity; a different uid needs +# a privileged harness. This still exercises CHOWN inverse serialization. +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_chown('base.dat', $uid, $gid); ROLLBACK;"); +for my $restart (0 .. 2) +{ + if ($restart) + { + $node->stop('immediate'); + $node->start; + } + ok(!-e "$dir/sub.dat", "$restart: subabort then parent abort"); + ok(!-l "$dir/abort.sym", "$restart: symlink inverse"); + ok(!-e "$dir/abort.link", "$restart: link inverse"); + ok(!-e "$dir/abort.dat", "$restart: aborted create absent"); + ok(!-e "$dir/abortdir", "$restart: aborted mkdir absent"); + SKIP: + { + skip 'POSIX permission semantics required', 1 if $windows_os; + is((stat("$dir/base.dat"))[2] & 07777, + 0600, "$restart: original mode"); + } + is((stat("$dir/base.dat"))[4], + $uid, "$restart: owner after chown inverse"); + is((stat("$dir/base.dat"))[5], + $gid, "$restart: group after chown inverse"); + SKIP: + { + skip 'extended attributes not supported', 1 unless $xattrs; + is( $node->safe_psql( + 'postgres', + "SELECT test_fileops_getxattr('base.dat', 'user.abort_test')" + ), + 'original', + "$restart: xattr inverses restore value"); + } + is(-s "$dir/base.dat", 14, "$restart: original size"); + is(slurp_file("$dir/base.dat"), + 'committed-base', "$restart: original bytes"); +} + +# Stop with a durable abort decision but no inverses applied yet. A +# concurrently requested checkpoint must wait, not omit the abort from redo. +if ($ENV{enable_injection_points} eq 'yes') +{ + $node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + for my $checkpoint (0, 1, 2) + { + my $finish = + $checkpoint == 2 + ? "PREPARE TRANSACTION 'window'; ROLLBACK PREPARED 'window';" + : 'ROLLBACK;'; + $node->safe_psql('postgres', 'CHECKPOINT'); + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-abort-after-flush', 'wait')" + ); + my $bg = $node->background_psql('postgres', on_error_stop => 0); + $bg->query_until( + qr/start/, qq(\\echo start +BEGIN; +SELECT test_fileops_create('mid.dat', 384); +SELECT test_fileops_write('mid.dat', 0, 'uncommitted'); +SELECT test_fileops_truncate('base.dat', 3); +$finish +)); + $node->wait_for_event('client backend', 'fileops-abort-after-flush'); + my $ckpt; + if ($checkpoint == 1) + { + $ckpt = $node->background_psql('postgres', on_error_stop => 0); + $ckpt->query_until(qr/checkpoint/, + "\\echo checkpoint\nCHECKPOINT;\n"); + $node->wait_for_event('checkpointer', 'CheckpointDelayStart'); + } + $node->stop('immediate'); + $bg->{run}->finish; + $ckpt->{run}->finish if $ckpt; + $node->start; + ok(!-e "$dir/mid.dat", + "$checkpoint: interrupted abort removed create"); + is(slurp_file("$dir/base.dat"), + 'committed-base', + "$checkpoint: interrupted abort restored shrink bytes"); + $node->stop('immediate'); + $node->start; + ok(!-e "$dir/mid.dat", + "$checkpoint: repeated recovery keeps create absent"); + } +} + +if ($ENV{enable_injection_points} eq 'yes') +{ + $node->safe_psql('postgres', 'CHECKPOINT'); + $node->safe_psql( + 'postgres', q{ +SELECT injection_points_attach('createdb-wal-log-after-copy-relation', 'error'); +SELECT injection_points_attach('fileops-abort-after-flush', 'wait'); +}); + my $db = $node->background_psql('postgres', on_error_stop => 0); + $db->query_until(qr/start/, + "\\echo start\nCREATE DATABASE aborted_tree STRATEGY WAL_LOG OID 90000;\n" + ); + $node->wait_for_event('client backend', 'fileops-abort-after-flush'); + # createdb's error cleanup removes the online directory already. Redo + # recreates it from forward WAL, so the embedded tree inverse must remove + # it again even though the original inode no longer exists. + ok(!-e "$dir/base/90000", 'createdb error cleanup removed online tree'); + $node->stop('immediate'); + $db->{run}->finish; + $node->start; + ok(!-e "$dir/base/90000", 'abort redo removes database tree'); +} + +# A WARNING is not a durable retry obligation: fixing permissions does not +# retry this inverse online. A crash before the next checkpoint replays it. +$node->safe_psql('postgres', 'CHECKPOINT'); +my $failed = $node->background_psql('postgres'); +$failed->query_safe('BEGIN'); +$failed->query_safe("SELECT test_fileops_truncate('base.dat', 3)"); +chmod(0400, "$dir/base.dat") or die "chmod: $!"; +my $log_offset = -s $node->logfile; +$failed->query('ROLLBACK'); +$failed->quit; +like( + slurp_file($node->logfile, $log_offset), + qr/WARNING:.*FILEOPS UNDO TRUNCATE: could not open/, + 'failed inverse reports a warning'); +chmod(0600, "$dir/base.dat") or die "chmod: $!"; +is(slurp_file("$dir/base.dat"), + 'com', 'failed inverse remains unapplied online'); +$node->stop('immediate'); +$node->start; +is(slurp_file("$dir/base.dat"), + 'committed-base', 'abort redo retries failed inverse after crash'); + +# Refuse excessive before-images before touching the file. +my ($ret, $out, $err) = $node->psql('postgres', + "SELECT test_fileops_write('base.dat', 0, repeat('x', 1048577))"); +like( + $err, + qr/ERROR:.*excessive transactional file write range/, + 'oversized write is an ordinary error'); +is(slurp_file("$dir/base.dat"), + 'committed-base', 'refused write changes nothing'); +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/018_fileops_rename.pl b/src/test/modules/test_fileops/t/018_fileops_rename.pl new file mode 100644 index 0000000000000..1967f84427b25 --- /dev/null +++ b/src/test/modules/test_fileops/t/018_fileops_rename.pl @@ -0,0 +1,347 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# FileOpsRename() renames at once, so the new name is usable inside the +# transaction, and the rename is reversed like every other immediate FILEOPS +# operation: on ROLLBACK, ROLLBACK TO SAVEPOINT and ROLLBACK PREPARED, by the +# startup undo phase after a crash, and on a standby that replays the abort. +# At wal_level = minimal an abort-time action renames it back. A rename onto +# an existing name, or of a missing file, is refused and changes nothing. +# Crash redo of committed renames converges on the state before the crash. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $injection = $ENV{enable_injection_points} eq 'yes'; + +my $node = PostgreSQL::Test::Cluster->new('fileops_rename'); +$node->init(allows_streaming => 1); +$node->append_conf( + 'postgresql.conf', q{ +autovacuum = off +max_prepared_transactions = 5 +}); +$node->append_conf('postgresql.conf', + "shared_preload_libraries = 'injection_points'\n") + if $injection; +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $datadir = $node->data_dir; + +# Commit file $name holding $content on $n. FILEOPS paths are relative to +# the data directory, and these are WAL-logged, so a standby gets them. +sub mkfile +{ + my ($n, $name, $content) = @_; + $n->safe_psql( + 'postgres', qq{ +BEGIN; +SELECT test_fileops_create('$name', 384); +SELECT test_fileops_write('$name', 0, '$content'); +COMMIT; +}); +} + +# What file $name in $dir holds, or ''. +sub content +{ + my ($dir, $name) = @_; + return -f "$dir/$name" ? slurp_file("$dir/$name") : ''; +} + +# Rename $from to $to in $bg's open transaction; it must show at once. +sub rename_in +{ + my ($bg, $dir, $from, $to, $tag) = @_; + $bg->query_safe("SELECT test_fileops_rename('$from', '$to')"); + ok( !-e "$dir/$from" && -e "$dir/$to", + "$tag: renamed inside the transaction"); +} + +my $bg = $node->background_psql('postgres', on_error_stop => 0); + +# a. COMMIT keeps the rename +mkfile($node, 'a.dat', 'content-a'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'a.dat', 'a2.dat', 'a'); +is(content($datadir, 'a2.dat'), + 'content-a', 'a: the new name has the content inside the transaction'); +$bg->query_safe('COMMIT'); +is(content($datadir, 'a2.dat'), 'content-a', 'a: COMMIT keeps the new name'); +ok(!-e "$datadir/a.dat", 'a: COMMIT keeps the old name gone'); + +# b. ROLLBACK restores the old name +mkfile($node, 'b.dat', 'content-b'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'b.dat', 'b2.dat', 'b'); +$bg->query_safe('ROLLBACK'); +is(content($datadir, 'b.dat'), + 'content-b', 'b: ROLLBACK restores the old name with its content'); +ok(!-e "$datadir/b2.dat", 'b: ROLLBACK removes the new name'); + +# c. ROLLBACK TO SAVEPOINT restores it, and COMMIT does not redo it +mkfile($node, 'c.dat', 'content-c'); +$bg->query_safe('BEGIN'); +$bg->query_safe('SAVEPOINT s'); +rename_in($bg, $datadir, 'c.dat', 'c2.dat', 'c'); +$bg->query_safe('ROLLBACK TO SAVEPOINT s'); +is(content($datadir, 'c.dat'), + 'content-c', 'c: ROLLBACK TO SAVEPOINT restores the old name'); +ok(!-e "$datadir/c2.dat", 'c: ROLLBACK TO SAVEPOINT removes the new name'); +$bg->query_safe('COMMIT'); +is(content($datadir, 'c.dat'), 'content-c', 'c: still restored after COMMIT'); +ok(!-e "$datadir/c2.dat", 'c: new name still absent after COMMIT'); + +# d. a later operation on the new name is undone first (LIFO) +mkfile($node, 'd.dat', 'content-d'); +my ($ret, $stdout, $stderr) = $node->psql( + 'postgres', q{BEGIN; +SELECT test_fileops_rename('d.dat', 'd2.dat'); +SELECT test_fileops_write('d2.dat', 0, 'CHANGED-AND-LONGER'); +SELECT pg_read_file('d2.dat'); +ROLLBACK;}); +like($stdout, qr/^CHANGED-AND-LONGER$/m, 'd: the write went to the new name'); +is(content($datadir, 'd.dat'), + 'content-d', 'd: ROLLBACK undid the write, then the rename'); +ok(!-e "$datadir/d2.dat", 'd: new name absent after ROLLBACK'); + +# e. refused: an existing destination, a missing source +mkfile($node, 'e.dat', 'content-e'); +mkfile($node, 'e2.dat', 'content-e2'); +($ret, $stdout, $stderr) = + $node->psql('postgres', "SELECT test_fileops_rename('e.dat', 'e2.dat')"); +isnt($ret, 0, 'e: renaming onto an existing name fails'); +like($stderr, qr/already exists/, 'e: ... saying the name exists'); +is(content($datadir, 'e.dat'), 'content-e', 'e: source unchanged'); +is(content($datadir, 'e2.dat'), 'content-e2', 'e: destination unchanged'); +($ret, $stdout, $stderr) = $node->psql('postgres', + "SELECT test_fileops_rename('e-missing.dat', 'e3.dat')"); +isnt($ret, 0, 'e: renaming a missing file fails'); +like($stderr, qr/could not stat file/, 'e: ... saying it is missing'); +ok(!-e "$datadir/e3.dat", 'e: ... and creates nothing'); + +# f. a directory, with a file in it +$node->safe_psql('postgres', "SELECT test_fileops_mkdir('fdir', 448)"); +mkfile($node, 'fdir/inner.dat', 'content-f'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'fdir', 'fdir2', 'f'); +is(content($datadir, 'fdir2/inner.dat'), + 'content-f', 'f: the directory contents moved with it'); +$bg->query_safe('ROLLBACK'); +is(content($datadir, 'fdir/inner.dat'), + 'content-f', 'f: ROLLBACK restores the directory and its contents'); +ok(!-e "$datadir/fdir2", 'f: new directory name absent after ROLLBACK'); + +# l. the rollback leaves an old name that was reused in the meantime alone +mkfile($node, 'l.dat', 'content-l'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'l.dat', 'l2.dat', 'l: old name reused'); +append_to_file("$datadir/l.dat", 'reused'); +$bg->query_safe('ROLLBACK'); +is(content($datadir, 'l.dat'), 'reused', 'l: the reused old name is kept'); +is(content($datadir, 'l2.dat'), + 'content-l', 'l: the renamed file stays under the new name'); +$bg->quit; + +# g. crash recovery reverses the rename of an in-flight transaction +mkfile($node, 'g.dat', 'content-g'); +my $bgc = $node->background_psql('postgres', on_error_stop => 0); +$bgc->query_safe('BEGIN'); +rename_in($bgc, $datadir, 'g.dat', 'g2.dat', 'g'); +$node->stop('immediate'); +$bgc->{run}->finish; +$node->start; +is(content($datadir, 'g.dat'), + 'content-g', 'g: crash recovery restores the old name'); +ok(!-e "$datadir/g2.dat", 'g: crash recovery removes the new name'); + +# g. a committed rename of a file created since the last checkpoint: redo +# re-creates the old name from the earlier records, and the rename's redo must +# not leave it behind +mkfile($node, 'g3.dat', 'content-g3'); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('g3.dat', 'g4.dat')"); +$node->stop('immediate'); +$node->start; +is(content($datadir, 'g4.dat'), + 'content-g3', 'g: a committed rename survives a crash'); +ok(!-e "$datadir/g3.dat", 'g: ... and redo leaves no old name behind'); + +# g. committed history a -> b, b -> c, then b made again, where a predates the +# checkpoint. Redo of b -> c finds the later b and c both present; renaming +# over c would replace the renamed file with the later b. +mkfile($node, 'ga.dat', 'content-ga'); +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('ga.dat', 'gb.dat')"); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('gb.dat', 'gc.dat')"); +mkfile($node, 'gb.dat', 'new-gb'); +$node->stop('immediate'); +$node->start; +ok(!-e "$datadir/ga.dat", 'g: a -> b -> c, b again: no a after the crash'); +is(content($datadir, 'gb.dat'), 'new-gb', 'g: ... b is the later file'); +is(content($datadir, 'gc.dat'), + 'content-ga', 'g: ... c is still the renamed file'); + +# g. a renamed directory that redo re-creates and fills under its old name is +# not removed: both names are left, with a WARNING, and nothing is lost +$node->safe_psql('postgres', "SELECT test_fileops_mkdir('gdir', 448)"); +mkfile($node, 'gdir/x.dat', 'content-gx'); +$node->safe_psql('postgres', "SELECT test_fileops_rename('gdir', 'gdir2')"); +my $offset = -s $node->logfile; +$node->stop('immediate'); +$node->start; +is(content($datadir, 'gdir2/x.dat'), + 'content-gx', 'g: a renamed directory keeps its contents after a crash'); +like( + slurp_file($node->logfile, $offset), + qr/WARNING: .*"gdir"/, + 'g: redo warns about the re-created old directory it leaves'); + +# h. two-phase commit +mkfile($node, 'h1.dat', 'content-h1'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('h1.dat', 'h1b.dat'); PREPARE TRANSACTION 'h1';" +); +ok(!-e "$datadir/h1.dat" && -e "$datadir/h1b.dat", + 'h: renamed while prepared'); +$node->safe_psql('postgres', "ROLLBACK PREPARED 'h1'"); +is(content($datadir, 'h1.dat'), + 'content-h1', 'h: ROLLBACK PREPARED restores the old name'); +ok(!-e "$datadir/h1b.dat", 'h: ROLLBACK PREPARED removes the new name'); + +mkfile($node, 'h2.dat', 'content-h2'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('h2.dat', 'h2b.dat'); PREPARE TRANSACTION 'h2';" +); +$node->stop('immediate'); +$node->start; +ok( !-e "$datadir/h2.dat" && -e "$datadir/h2b.dat", + 'h: still renamed while prepared after a crash'); +$node->safe_psql('postgres', "ROLLBACK PREPARED 'h2'"); +is(content($datadir, 'h2.dat'), + 'content-h2', 'h: ROLLBACK PREPARED after a crash restores the old name'); +ok(!-e "$datadir/h2b.dat", + 'h: ROLLBACK PREPARED after a crash removes the new name'); + +mkfile($node, 'h3.dat', 'content-h3'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('h3.dat', 'h3b.dat'); PREPARE TRANSACTION 'h3';" +); +$node->safe_psql('postgres', "COMMIT PREPARED 'h3'"); +is(content($datadir, 'h3b.dat'), + 'content-h3', 'h: COMMIT PREPARED keeps the new name'); +ok(!-e "$datadir/h3.dat", 'h: COMMIT PREPARED keeps the old name gone'); + +# i. a streaming standby converges, for COMMIT and for ROLLBACK +$node->backup('bk'); +my $standby = PostgreSQL::Test::Cluster->new('fileops_rename_standby'); +$standby->init_from_backup($node, 'bk', has_streaming => 1); +$standby->start; +my $sdir = $standby->data_dir; +mkfile($node, 'i1.dat', 'content-i1'); +mkfile($node, 'i2.dat', 'content-i2'); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('i1.dat', 'i1b.dat')"); +my $bgi = $node->background_psql('postgres', on_error_stop => 0); +$bgi->query_safe('BEGIN'); +$bgi->query_safe("SELECT test_fileops_rename('i2.dat', 'i2b.dat')"); +$node->wait_for_catchup($standby); +is(content($sdir, 'i1b.dat'), + 'content-i1', 'i: standby has the committed new name'); +ok(!-e "$sdir/i1.dat", 'i: standby lost the committed old name'); +ok( !-e "$sdir/i2.dat" && -e "$sdir/i2b.dat", + 'i: standby shows a rename in flight'); +$bgi->query_safe('ROLLBACK'); +$bgi->quit; +$node->wait_for_catchup($standby); +is(content($sdir, 'i2.dat'), + 'content-i2', 'i: standby restores the old name of a rolled-back rename'); +ok(!-e "$sdir/i2b.dat", 'i: standby has no new name of a rolled-back rename'); + +# i. ROLLBACK PREPARED on a promoted standby renames back, though the +# standby's file is not the primary's inode +mkfile($node, 'i3.dat', 'content-i3'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('i3.dat', 'i3b.dat'); PREPARE TRANSACTION 'i3';" +); +$node->wait_for_catchup($standby); +ok( !-e "$sdir/i3.dat" && -e "$sdir/i3b.dat", + 'i: standby shows the prepared rename'); +$standby->promote; +$standby->safe_psql('postgres', "ROLLBACK PREPARED 'i3'"); +is(content($sdir, 'i3.dat'), + 'content-i3', + 'i: ROLLBACK PREPARED on the promoted standby restores the old name'); +ok(!-e "$sdir/i3b.dat", + 'i: ROLLBACK PREPARED on the promoted standby removes the new name'); +$node->safe_psql('postgres', "ROLLBACK PREPARED 'i3'"); +$standby->stop; + +# j. wal_level = minimal: no WAL, an abort-time action renames back +my $min = PostgreSQL::Test::Cluster->new('fileops_rename_minimal'); +$min->init; +$min->append_conf('postgresql.conf', + "wal_level = minimal\nmax_wal_senders = 0\n"); +$min->start; +$min->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +is($min->safe_psql('postgres', 'SHOW wal_level'), + 'minimal', 'j: wal_level is minimal'); +my $mdir = $min->data_dir; +mkfile($min, 'j.dat', 'content-j'); +my $bgm = $min->background_psql('postgres', on_error_stop => 0); +$bgm->query_safe('BEGIN'); +rename_in($bgm, $mdir, 'j.dat', 'j2.dat', 'j'); +$bgm->query_safe('ROLLBACK'); +is(content($mdir, 'j.dat'), 'content-j', 'j: ROLLBACK restores the old name'); +ok(!-e "$mdir/j2.dat", 'j: ROLLBACK removes the new name'); +$bgm->query_safe('BEGIN'); +$bgm->query_safe('SAVEPOINT s'); +rename_in($bgm, $mdir, 'j.dat', 'j2.dat', 'j: subtransaction'); +$bgm->query_safe('ROLLBACK TO SAVEPOINT s'); +is(content($mdir, 'j.dat'), + 'content-j', 'j: ROLLBACK TO SAVEPOINT restores the old name'); +rename_in($bgm, $mdir, 'j.dat', 'j3.dat', 'j: then'); +$bgm->query_safe('COMMIT'); +is(content($mdir, 'j3.dat'), 'content-j', 'j: COMMIT keeps the new name'); +ok(!-e "$mdir/j.dat" && !-e "$mdir/j2.dat", 'j: and no other name'); +$bgm->quit; +$min->stop; + +# k. a crash after the WAL is flushed but before rename(): redo performs the +# rename, and the loser's UNDO reverses it +SKIP: +{ + skip 'Injection points not supported by this build', 4 unless $injection; + $node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + mkfile($node, 'k.dat', 'content-k'); + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-rename-after-wal-flush', 'wait')" + ); + my $bgk = $node->background_psql('postgres', on_error_stop => 0); + $bgk->query_until( + qr/start/, q(\echo start +BEGIN; +SELECT test_fileops_rename('k.dat', 'k2.dat'); +)); + $node->wait_for_event('client backend', 'fileops-rename-after-wal-flush'); + ok(-e "$datadir/k.dat" && !-e "$datadir/k2.dat", + 'k: logged, not yet renamed'); + $offset = -s $node->logfile; + $node->stop('immediate'); + $bgk->{run}->finish; + $node->start; + like( + slurp_file($node->logfile, $offset), + qr/UNDO recovery: rolling back transaction/, + 'k: the transaction was rolled back as a loser'); + is(content($datadir, 'k.dat'), 'content-k', 'k: the old name is kept'); + ok(!-e "$datadir/k2.dat", 'k: the new name is absent'); +} + +$node->stop; + +done_testing(); diff --git a/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl b/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl new file mode 100644 index 0000000000000..edce86837732d --- /dev/null +++ b/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl @@ -0,0 +1,60 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# AtAbort_XactUndo()'s inline-abort path must restore the memory context and +# free its working context on every exit, including the early exit taken when +# the batch LSN does not point at a valid UNDO batch. +# +# That exit jumped over the MemoryContextSwitchTo(old_ctx) / +# MemoryContextDelete(undo_ctx) pair, so the "Inline UNDO Apply" context (a +# child of TopMemoryContext) leaked on every such abort, and the rest of the +# abort ran in, and returned into, that context. One leaked context is +# enough to show it: it survives into the next statement of the same +# backend, where pg_backend_memory_contexts lists it. +# +# The invalid-batch exit is forced with the "inline-undo-invalid-batch" +# injection point. A FILEOPS chmod puts one UNDO record in the +# transaction's chain so the inline path runs at all. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('fileops_inline_abort_ctx'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf('postgresql.conf', 'wal_level = replica'); +$node->start; +$node->safe_psql('postgres', + 'CREATE EXTENSION test_fileops; CREATE EXTENSION injection_points;'); + +my $path = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('inline_abort_ctx.dat')"); + +# One backend: the leaked context has to be looked for in the backend that +# aborted, after its transaction has ended. +my $bg = $node->background_psql('postgres'); +$bg->query_safe('SELECT injection_points_set_local()'); +$bg->query_safe( + "SELECT injection_points_attach('inline-undo-invalid-batch', 'error')"); + +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_chmod('$path', 384)"); +$bg->query('ROLLBACK'); + +# The ROLLBACK warns "inline UNDO failed"; query() leaves that on stderr. +my $leaked = $bg->query( + "SELECT count(*) FROM pg_backend_memory_contexts " + . "WHERE name = 'Inline UNDO Apply'"); +chomp $leaked; +is($leaked, '0', + 'the inline-abort working context is freed on the invalid-batch exit'); + +$bg->quit; +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl b/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl new file mode 100644 index 0000000000000..40eb16e6026ff --- /dev/null +++ b/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl @@ -0,0 +1,64 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# The startup undo phase must not report a crash loser as rolled back when one +# of its UNDO records could not be applied. +# +# A crash loser changed two files' modes through FILEOPS. Before the restart +# the files' directory loses its search permission, so both inverse chmod()s +# fail with a real EACCES. Nothing retries them, so the modes stay changed; +# the summary at the end of the undo phase has to say so, and count both. +# +# Must not run as root (root ignores the directory permission). +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +plan skip_all => 'running as root: EACCES cannot be produced' if $> == 0; +plan skip_all => 'directory search permission is POSIX-only' + if $PostgreSQL::Test::Utils::windows_os; + +my $node = PostgreSQL::Test::Cluster->new('fileops_recovery_apply_failure'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf('postgresql.conf', "wal_level = replica\nautovacuum = off\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $dir = PostgreSQL::Test::Utils::tempdir('eacces'); +my @paths = ("$dir/m.dat", "$dir/n.dat"); +foreach my $path (@paths) +{ + open(my $fh, '>', $path) or die "open $path: $!"; + close($fh); + chmod(0644, $path) or die "chmod $path: $!"; +} + +# Two chmods in flight when the server crashes: a crash loser. +my $bg = $node->background_psql('postgres', on_error_stop => 0); +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_chmod('$_', 384)") # 0600 + foreach @paths; +$node->safe_psql('postgres', 'SELECT pg_switch_wal()'); +my $offset = -s $node->logfile; +$node->stop('immediate'); +$bg->{run}->finish; + +chmod(0600, $dir) or die "chmod $dir: $!"; +$node->start; +chmod(0700, $dir) or die "chmod $dir: $!"; + +my $log = slurp_file($node->logfile, $offset); +like($log, qr/FILEOPS UNDO CHMOD: could not (?:stat|restore mode of) .*: Permission denied/, + 'the inverse failed with a real EACCES'); +like($log, qr/UNDO recovery complete: 1 transactions processed, 0 records applied, 2 records could not be applied/, + 'the undo phase reports the records it could not apply'); +unlike($log, qr/UNDO recovery complete: \d+ transactions rolled back/, + 'the loser is not reported as rolled back'); +is(sprintf('%04o', (stat($_))[2] & 07777), '0600', + "the mode change of $_ was not rolled back") + foreach @paths; + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/025_fileops_create_existing.pl b/src/test/modules/test_fileops/t/025_fileops_create_existing.pl new file mode 100644 index 0000000000000..a0db2dc29f0e6 --- /dev/null +++ b/src/test/modules/test_fileops/t/025_fileops_create_existing.pl @@ -0,0 +1,56 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# FileOpsCreate() on a path that already exists, then ROLLBACK. The UNDO +# record of a create unlinks the path, so if the call opened an existing file +# a rollback would remove a committed one. The committed file must survive +# on the primary, on a standby replaying the abort record, and on the primary +# after crash recovery replays it. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; + +$primary->safe_psql('postgres', q{ +BEGIN; +SELECT test_fileops_create('existing.dat', 384); +SELECT test_fileops_write('existing.dat', 0, 'committed'); +COMMIT; +}); + +my ($ret, $out, $err) = $primary->psql('postgres', q{ +BEGIN; +SELECT test_fileops_create('existing.dat', 384); +ROLLBACK; +}); +like($err, qr/could not create file "existing.dat": File exists/, + 'creating an existing path is an ERROR'); + +$primary->wait_for_catchup($standby); +for my $stage ('online', 'after crash') +{ + if ($stage eq 'after crash') + { + $primary->stop('immediate'); + $primary->start; + } + my $f = $primary->data_dir . '/existing.dat'; + is(-e $f ? slurp_file($f) : '(missing)', + 'committed', "$stage: primary keeps the committed file"); +} +my $sf = $standby->data_dir . '/existing.dat'; +is(-e $sf ? slurp_file($sf) : '(missing)', + 'committed', 'standby keeps the committed file'); + +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/026_fileops_abort_checkpoint_order.pl b/src/test/modules/test_fileops/t/026_fileops_abort_checkpoint_order.pl new file mode 100644 index 0000000000000..3bb9dfe6eda3d --- /dev/null +++ b/src/test/modules/test_fileops/t/026_fileops_abort_checkpoint_order.pl @@ -0,0 +1,77 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A checkpoint must not move the redo pointer past an abort record whose +# FILEOPS inverses are not yet durable. Pause the aborting backend after the +# abort record is flushed and before it applies the inverses, request a +# CHECKPOINT, then crash. Recovery must replay the abort record and remove +# the file, for ROLLBACK and for ROLLBACK PREPARED. Without the hold-off the +# checkpoint completes during the pause, redo starts after the abort record, +# and the file survives. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +plan skip_all => 'injection points not supported by this build' + unless $ENV{enable_injection_points} eq 'yes'; + +my $node = PostgreSQL::Test::Cluster->new('n'); +$node->init(allows_streaming => 1); +$node->append_conf('postgresql.conf', + "max_prepared_transactions = 5\ncheckpoint_timeout = 1h\n"); +$node->start; +$node->safe_psql('postgres', + 'CREATE EXTENSION test_fileops; CREATE EXTENSION injection_points'); + +for my $case ('rollback', 'rollback prepared') +{ + (my $f = "ck-$case.dat") =~ s/ /-/g; + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-abort-after-flush', 'wait')"); + my $before = $node->safe_psql('postgres', + 'SELECT checkpoint_lsn FROM pg_control_checkpoint()'); + + my $a = $node->background_psql('postgres'); + if ($case eq 'rollback') + { + $a->query_until(qr/go/, qq{\\echo go +BEGIN; SELECT test_fileops_create('$f', 384); ROLLBACK; +}); + } + else + { + $node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_create('$f', 384); PREPARE TRANSACTION 'p';"); + $a->query_until(qr/go/, qq{\\echo go +ROLLBACK PREPARED 'p'; +}); + } + $node->wait_for_event('client backend', 'fileops-abort-after-flush'); + ok(-e $node->data_dir . "/$f", "$case: file still there at the pause"); + + my $c = $node->background_psql('postgres'); + $c->query_until(qr/go/, "\\echo go\nCHECKPOINT;\n"); + + # Give the checkpoint time to finish if nothing holds it back. + my $after = $before; + for (1 .. 50) + { + $after = $node->safe_psql('postgres', + 'SELECT checkpoint_lsn FROM pg_control_checkpoint()'); + last if $after ne $before; + usleep(100_000); + } + note "$case: checkpoint completed during the pause: " + . ($after ne $before ? 'yes' : 'no'); + + $node->stop('immediate'); + eval { $a->quit; }; + eval { $c->quit; }; + $node->start; + ok(!-e $node->data_dir . "/$f", "$case: crash recovery removed the file"); +} + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/027_fileops_crash_loser_standby.pl b/src/test/modules/test_fileops/t/027_fileops_crash_loser_standby.pl new file mode 100644 index 0000000000000..9e36c7b7ee14a --- /dev/null +++ b/src/test/modules/test_fileops/t/027_fileops_crash_loser_standby.pl @@ -0,0 +1,64 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A transaction is in flight with a FILEOPS create and mkdir when the primary +# crashes, so it writes no abort record. The primary's crash recovery rolls +# it back and logs the rollback (XLOG_UNDO_ROLLBACK), and a streaming standby +# must end with the primary's files when it replays that record. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', "restart_after_crash = on\n"); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->backup('b'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'b', has_streaming => 1); +$standby->start; + +for my $how ('immediate stop', 'backend killed') +{ + (my $f = "loser-$how.dat") =~ s/ /-/g; + my $a = $primary->background_psql('postgres'); + $a->query_safe("BEGIN; SELECT test_fileops_create('$f', 384);" + . " SELECT test_fileops_mkdir('$f.d', 448);"); + # Make it durable and replicated. + $primary->safe_psql('postgres', + 'CREATE TABLE IF NOT EXISTS flush_me (a int)'); + $primary->wait_for_catchup($standby); + ok(-e $standby->data_dir . "/$f", "$how: standby has the in-flight file"); + + if ($how eq 'immediate stop') + { + $primary->stop('immediate'); + eval { $a->quit; }; + $primary->start; + } + else + { + my $pid = $a->query_safe('SELECT pg_backend_pid()'); + chomp $pid; + PostgreSQL::Test::Utils::system_or_bail('pg_ctl', 'kill', 'KILL', $pid); + eval { $a->quit; }; + $primary->poll_query_until('postgres', 'SELECT 1', '1'); + } + ok(!-e $primary->data_dir . "/$f", "$how: primary reversed the loser"); + $primary->safe_psql('postgres', 'CHECKPOINT'); + $primary->wait_for_catchup($standby); + ok(!-e $standby->data_dir . "/$f", "$how: standby reversed the loser file"); + ok(!-e $standby->data_dir . "/$f.d", "$how: standby reversed the loser dir"); +} + +$standby->promote; +for my $how ('immediate-stop', 'backend-killed') +{ + ok(!-e $standby->data_dir . "/loser-$how.dat", + "promoted standby: loser-$how.dat absent"); +} +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/028_fileops_insert_holdoff.pl b/src/test/modules/test_fileops/t/028_fileops_insert_holdoff.pl new file mode 100644 index 0000000000000..bfa48073a345d --- /dev/null +++ b/src/test/modules/test_fileops/t/028_fileops_insert_holdoff.pl @@ -0,0 +1,67 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# InsertXactUndoData() holds off a checkpoint's redo-pointer phase between +# inserting an UNDO batch and publishing the chain head that the checkpoint's +# XLOG_UNDO_INFLIGHT record lists. Stop a backend in that interval, request +# a CHECKPOINT, and crash. Without the hold-off the checkpoint completes, its +# redo pointer follows the batch, and its record omits the chain, so nothing +# reverses the create. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +plan skip_all => 'injection points not supported by this build' + unless $ENV{enable_injection_points} eq 'yes'; + +my $node = PostgreSQL::Test::Cluster->new('n'); +$node->init(allows_streaming => 1); +$node->append_conf('postgresql.conf', "checkpoint_timeout = 1h\n"); +$node->start; +$node->safe_psql('postgres', + 'CREATE EXTENSION test_fileops; CREATE EXTENSION injection_points'); + +# The point is in a critical section, so the waiting backend must attach to +# the injection points' shared state beforehand. +my $a = $node->background_psql('postgres'); +$a->query_safe('SELECT injection_points_set_local()'); +$a->query_safe( + "SELECT injection_points_attach('undo-batch-after-wal-insert', 'wait')"); +$a->query_until(qr/go/, + "\\echo go\nBEGIN; SELECT test_fileops_create('holdoff.dat', 384);\n"); +$node->wait_for_event('client backend', 'undo-batch-after-wal-insert'); + +# Make the create and its UNDO batch durable, so that the crash below falls +# outside the documented window between a system call and its WAL record: +# another transaction's commit flushes everything inserted before it. +my $ins = $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +$node->safe_psql('postgres', 'CREATE TABLE flush_me (a int)'); +is( $node->safe_psql( + 'postgres', "SELECT pg_current_wal_flush_lsn() >= '$ins'::pg_lsn"), + 't', + 'the create is durable before the checkpoint'); + +# The checkpoint either waits for the backend or, without the hold-off, +# completes. +my $before = $node->safe_psql('postgres', + 'SELECT checkpoint_lsn FROM pg_control_checkpoint()'); +my $c = $node->background_psql('postgres'); +$c->query_until(qr/go/, "\\echo go\nCHECKPOINT;\n"); +$node->poll_query_until( + 'postgres', qq{ +SELECT EXISTS (SELECT 1 FROM pg_stat_activity + WHERE backend_type = 'checkpointer' + AND wait_event = 'CheckpointDelayStart') + OR (SELECT checkpoint_lsn <> '$before' FROM pg_control_checkpoint())}) + or die 'checkpoint neither waited nor completed'; + +$node->stop('immediate'); +eval { $a->quit; }; +eval { $c->quit; }; +$node->start; +ok(!-e $node->data_dir . '/holdoff.dat', + 'crash recovery removed the loser file'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/029_fileops_crash_loser_recycled_wal.pl b/src/test/modules/test_fileops/t/029_fileops_crash_loser_recycled_wal.pl new file mode 100644 index 0000000000000..54837e160da19 --- /dev/null +++ b/src/test/modules/test_fileops/t/029_fileops_crash_loser_recycled_wal.pl @@ -0,0 +1,53 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A crash loser's UNDO batches can lie in WAL that a standby has already +# removed at a restartpoint by the time it replays the end of the primary's +# crash recovery. The primary logs the rollback with the UNDO records it +# applied (XLOG_UNDO_ROLLBACK), so the standby does not need the batches. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', + "min_wal_size = 32MB\nmax_wal_size = 32MB\ncheckpoint_timeout = 1h\n"); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->backup('b'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'b', has_streaming => 1); +$standby->start; + +my $bg = $primary->background_psql('postgres'); +$bg->query_safe("BEGIN; SELECT test_fileops_create('old-loser.dat', 384);"); +my $seg = $primary->safe_psql('postgres', + 'SELECT pg_walfile_name(pg_current_wal_insert_lsn())'); + +# Move the standby's restartpoints past the loser's batch, so that it can +# remove the segment. The primary keeps it for the open transaction. +$primary->safe_psql('postgres', 'CREATE TABLE t (a int)'); +for (1 .. 6) +{ + $primary->safe_psql('postgres', + 'INSERT INTO t SELECT generate_series(1, 1000);' + . ' SELECT pg_switch_wal(); CHECKPOINT;'); + $primary->wait_for_catchup($standby); + $standby->safe_psql('postgres', 'CHECKPOINT'); +} +my $sdir = $standby->data_dir; +ok(!-e "$sdir/pg_wal/$seg", "standby removed $seg"); +ok(-e $standby->data_dir . '/old-loser.dat', 'standby has the loser file'); + +$primary->stop('immediate'); +eval { $bg->quit; }; +$primary->start; +$primary->wait_for_catchup($standby); +ok(!-e $primary->data_dir . '/old-loser.dat', 'primary rolled back the loser'); +ok(!-e "$sdir/old-loser.dat", 'standby rolled back the loser'); + +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/030_fileops_rollback_record.pl b/src/test/modules/test_fileops/t/030_fileops_rollback_record.pl new file mode 100644 index 0000000000000..c41661fc99ab6 --- /dev/null +++ b/src/test/modules/test_fileops/t/030_fileops_rollback_record.pl @@ -0,0 +1,74 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# The undo phase at the end of crash recovery logs each loser's rollback in +# an XLOG_UNDO_ROLLBACK record, carrying the records that applied. A standby +# applies them when it replays that record. So the standby must keep a +# change the primary could not roll back, and a second crash of the primary +# must not apply the loser's records again over a committed reuse of its +# name. +# +# Must not run as root (root ignores directory permissions). +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +plan skip_all => 'running as root: EACCES cannot be produced' if $> == 0; +plan skip_all => 'directory search permission is POSIX-only' + if $PostgreSQL::Test::Utils::windows_os; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->backup('b'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'b', has_streaming => 1); +$standby->start; +my $pd = $primary->data_dir; +my $sd = $standby->data_dir; + +# A loser creates two files, one inside a directory that loses its search +# permission before the restart, so the primary cannot remove that one. +mkdir("$pd/locked") or die "mkdir: $!"; +mkdir("$sd/locked") or die "mkdir: $!"; +my $a = $primary->background_psql('postgres'); +$a->query_safe("BEGIN; SELECT test_fileops_create('loser.dat', 384);" + . " SELECT test_fileops_create('locked/stuck.dat', 384);"); +$primary->safe_psql('postgres', 'CREATE TABLE flush_me (a int)'); +$primary->wait_for_catchup($standby); +ok(-e "$sd/loser.dat", 'standby has the loser file'); + +$primary->stop('immediate'); +eval { $a->quit; }; +chmod(0600, "$pd/locked") or die "chmod: $!"; +$primary->start; +chmod(0700, "$pd/locked") or die "chmod: $!"; +ok(!-e "$pd/loser.dat", 'primary removed the loser file'); +ok(-e "$pd/locked/stuck.dat", 'primary could not remove the locked file'); + +# The standby replays the rollback record and does what the primary did. +$primary->wait_for_catchup($standby); +ok(!-e "$sd/loser.dat", 'standby removed the loser file'); +ok(-e "$sd/locked/stuck.dat", + 'standby keeps the file the primary could not remove'); + +# The name is reused and committed. A second crash replays the rollback +# record, which must leave the committed file alone. +$primary->safe_psql( + 'postgres', q{ +BEGIN; SELECT test_fileops_create('loser.dat', 384); +SELECT test_fileops_write('loser.dat', 0, 'committed'); COMMIT; +}); +$primary->stop('immediate'); +$primary->start; +is(slurp_file("$pd/loser.dat"), 'committed', + 'a second crash leaves the committed reuse'); +$primary->wait_for_catchup($standby); +is(slurp_file("$sd/loser.dat"), 'committed', + 'the standby has the committed reuse'); + +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/test_fileops--1.0.sql b/src/test/modules/test_fileops/test_fileops--1.0.sql new file mode 100644 index 0000000000000..80545574727a7 --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops--1.0.sql @@ -0,0 +1,99 @@ +/* src/test/modules/test_fileops/test_fileops--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_fileops" to load this file. \quit + +-- Create a temporary file and return its path +CREATE FUNCTION test_fileops_create_tempfile(filename text) + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a file via FileOpsCreate (exercises the WAL-before-syscall path) +CREATE FUNCTION test_fileops_create(filepath text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Truncate a file to the given length (uses FileOpsTruncate) +CREATE FUNCTION test_fileops_truncate(filepath text, length bigint) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get file size +CREATE FUNCTION test_fileops_file_size(filepath text) + RETURNS bigint + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Chmod a file (uses FileOpsChmod) +CREATE FUNCTION test_fileops_chmod(filepath text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get file permissions (mode) +CREATE FUNCTION test_fileops_get_mode(filepath text) + RETURNS int + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a hard link (uses FileOpsLink) +CREATE FUNCTION test_fileops_link(oldpath text, newpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Check if a file exists +CREATE FUNCTION test_fileops_file_exists(filepath text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Set extended attribute (uses FileOpsSetXattr) +CREATE FUNCTION test_fileops_setxattr(filepath text, attrname text, attrvalue text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get extended attribute value +CREATE FUNCTION test_fileops_getxattr(filepath text, attrname text) + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Remove extended attribute (uses FileOpsRemoveXattr) +CREATE FUNCTION test_fileops_removexattr(filepath text, attrname text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get the data directory path (for constructing absolute paths) +CREATE FUNCTION test_fileops_data_dir() + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred delete at commit (uses FileOpsDelete, at_commit=true) +CREATE FUNCTION test_fileops_delete(filepath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred rename at commit (uses FileOpsRename) +CREATE FUNCTION test_fileops_rename(oldpath text, newpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a directory (uses FileOpsMkdir) +CREATE FUNCTION test_fileops_mkdir(path text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred rmdir at commit (uses FileOpsRmdir, at_commit=true) +CREATE FUNCTION test_fileops_rmdir(path text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Write a string at an offset (uses FileOpsWrite) +CREATE FUNCTION test_fileops_write(filepath text, "offset" bigint, data text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Change ownership (uses FileOpsChown) +CREATE FUNCTION test_fileops_chown(filepath text, uid int, gid int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a symbolic link (uses FileOpsSymlink) +CREATE FUNCTION test_fileops_symlink(target text, linkpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; diff --git a/src/test/modules/test_fileops/test_fileops.c b/src/test/modules/test_fileops/test_fileops.c new file mode 100644 index 0000000000000..c871d974427e9 --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops.c @@ -0,0 +1,471 @@ +/*------------------------------------------------------------------------- + * + * test_fileops.c + * Test module exposing FileOps C API to SQL for regression testing. + * + * This module provides SQL-callable wrappers around the FileOps functions + * that have no DDL-level callers, enabling direct testing of: + * - FileOpsTruncate (with UNDO rollback of file size) + * - FileOpsChmod (with UNDO rollback of permissions) + * - FileOpsLink (with UNDO rollback of hard links) + * - FileOpsSetXattr / FileOpsRemoveXattr (with UNDO rollback) + * + * Copyright (c) 2024-2026, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "port/pg_xattr.h" +#include "access/xlogdefs.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "utils/builtins.h" +#include "utils/pg_lsn.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(test_fileops_create_tempfile); +PG_FUNCTION_INFO_V1(test_fileops_create); +PG_FUNCTION_INFO_V1(test_fileops_truncate); +PG_FUNCTION_INFO_V1(test_fileops_file_size); +PG_FUNCTION_INFO_V1(test_fileops_chmod); +PG_FUNCTION_INFO_V1(test_fileops_get_mode); +PG_FUNCTION_INFO_V1(test_fileops_link); +PG_FUNCTION_INFO_V1(test_fileops_file_exists); +PG_FUNCTION_INFO_V1(test_fileops_setxattr); +PG_FUNCTION_INFO_V1(test_fileops_getxattr); +PG_FUNCTION_INFO_V1(test_fileops_removexattr); +PG_FUNCTION_INFO_V1(test_fileops_data_dir); +PG_FUNCTION_INFO_V1(test_fileops_write); +PG_FUNCTION_INFO_V1(test_fileops_chown); +PG_FUNCTION_INFO_V1(test_fileops_symlink); + +PG_FUNCTION_INFO_V1(test_fileops_delete); +PG_FUNCTION_INFO_V1(test_fileops_rename); +PG_FUNCTION_INFO_V1(test_fileops_mkdir); +PG_FUNCTION_INFO_V1(test_fileops_rmdir); + +/* + * test_fileops_create_tempfile - Create a test file in data directory. + * + * Creates a file with some content using FileOpsCreate + write, returning + * the absolute path. The file is created with 1024 bytes of content. + */ +Datum +test_fileops_create_tempfile(PG_FUNCTION_ARGS) +{ + text *filename = PG_GETARG_TEXT_PP(0); + char *fname = text_to_cstring(filename); + char filepath[MAXPGPATH]; + int fd; + char buf[1024]; + + snprintf(filepath, MAXPGPATH, "%s/%s", DataDir, fname); + + /* Use raw create + write so we have a file to manipulate */ + fd = open(filepath, O_CREAT | O_WRONLY | O_TRUNC | PG_BINARY, 0644); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", filepath))); + + /* Fill with 1024 bytes */ + memset(buf, 'X', sizeof(buf)); + if (write(fd, buf, sizeof(buf)) != sizeof(buf)) + { + int save_errno = errno; + + close(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", filepath))); + } + close(fd); + + PG_RETURN_TEXT_P(cstring_to_text(filepath)); +} + +/* + * test_fileops_create - Create a file using FileOpsCreate. + * + * Unlike test_fileops_create_tempfile (which uses raw open()/write() to set + * up fixture files outside any transactional path), this calls FileOpsCreate + * directly so tests can exercise its WAL-before-syscall crash-recovery + * behavior. + */ +Datum +test_fileops_create(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int mode = PG_GETARG_INT32(1); + char *filepath = text_to_cstring(filepath_text); + int fd; + + fd = FileOpsCreate(filepath, O_WRONLY | PG_BINARY, (mode_t) mode, true); + if (fd >= 0) + CloseTransientFile(fd); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_truncate - Truncate a file using FileOpsTruncate. + * + * This exercises the UNDO path: on rollback, the original file size + * should be restored. + */ +Datum +test_fileops_truncate(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int64 length = PG_GETARG_INT64(1); + char *filepath = text_to_cstring(filepath_text); + + FileOpsTruncate(filepath, (off_t) length); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_file_size - Get the current size of a file. + */ +Datum +test_fileops_file_size(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + + if (stat(filepath, &st) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_INT64((int64) st.st_size); +} + +/* + * test_fileops_chmod - Change file permissions using FileOpsChmod. + * + * This exercises the UNDO path: on rollback, the original permissions + * should be restored. + */ +Datum +test_fileops_chmod(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int mode = PG_GETARG_INT32(1); + char *filepath = text_to_cstring(filepath_text); + + if (FileOpsChmod(filepath, (mode_t) mode) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not chmod file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_get_mode - Get file permission bits. + */ +Datum +test_fileops_get_mode(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + + if (stat(filepath, &st) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_INT32((int32) (st.st_mode & 0777)); +} + +/* + * test_fileops_link - Create a hard link using FileOpsLink. + * + * This exercises the UNDO path: on rollback, the link should be removed. + */ +Datum +test_fileops_link(PG_FUNCTION_ARGS) +{ + text *oldpath_text = PG_GETARG_TEXT_PP(0); + text *newpath_text = PG_GETARG_TEXT_PP(1); + char *oldpath = text_to_cstring(oldpath_text); + char *newpath = text_to_cstring(newpath_text); + + if (FileOpsLink(oldpath, newpath) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not link \"%s\" to \"%s\": %m", + oldpath, newpath))); + + pfree(oldpath); + pfree(newpath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_file_exists - Check whether a file exists. + */ +Datum +test_fileops_file_exists(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + bool exists; + + exists = (stat(filepath, &st) == 0); + + pfree(filepath); + PG_RETURN_BOOL(exists); +} + +/* + * test_fileops_setxattr - Set an extended attribute using FileOpsSetXattr. + * + * Returns true if the platform supports xattrs, false if ENOTSUP. + * This exercises the UNDO path: on rollback, the original xattr value + * (or absence) should be restored. + */ +Datum +test_fileops_setxattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + text *value_text = PG_GETARG_TEXT_PP(2); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + char *value = text_to_cstring(value_text); + int ret; + + ret = FileOpsSetXattr(filepath, name, value, strlen(value)); + if (ret != 0) + { + if (errno == ENOTSUP || errno == EOPNOTSUPP || + errno == EPERM || errno == EACCES) + { + pfree(filepath); + pfree(name); + pfree(value); + PG_RETURN_BOOL(false); + } + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set xattr \"%s\" on \"%s\": %m", + name, filepath))); + } + + pfree(filepath); + pfree(name); + pfree(value); + PG_RETURN_BOOL(true); +} + +/* + * test_fileops_getxattr - Get an extended attribute value. + * + * Returns NULL if the attribute doesn't exist or xattrs are unsupported. + */ +Datum +test_fileops_getxattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + char buf[1024]; + ssize_t len; + + len = pg_getxattr(filepath, name, buf, sizeof(buf) - 1); + if (len < 0) + { + pfree(filepath); + pfree(name); + PG_RETURN_NULL(); + } + + buf[len] = '\0'; + + pfree(filepath); + pfree(name); + PG_RETURN_TEXT_P(cstring_to_text(buf)); +} + +/* + * test_fileops_removexattr - Remove an extended attribute using + * FileOpsRemoveXattr. + * + * Returns true if successful, false if xattrs are unsupported. + * This exercises the UNDO path: on rollback, the removed xattr should + * be restored. + */ +Datum +test_fileops_removexattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + int ret; + + ret = FileOpsRemoveXattr(filepath, name); + if (ret != 0) + { + if (errno == ENOTSUP || errno == EOPNOTSUPP) + { + pfree(filepath); + pfree(name); + PG_RETURN_BOOL(false); + } + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not remove xattr \"%s\" from \"%s\": %m", + name, filepath))); + } + + pfree(filepath); + pfree(name); + PG_RETURN_BOOL(true); +} + +/* + * test_fileops_data_dir - Return the data directory path. + */ +Datum +test_fileops_data_dir(PG_FUNCTION_ARGS) +{ + PG_RETURN_TEXT_P(cstring_to_text(DataDir)); +} + +/* + * test_fileops_delete - Deferred delete via FileOpsDelete(at_commit=true). + * + * Exercises the log-at-commit-execution path: the delete happens (and is + * WAL-logged) in FileOpsDoPendingOps() at commit, so a physical standby + * reproduces it by replay. + */ +Datum +test_fileops_delete(PG_FUNCTION_ARGS) +{ + char *filepath = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + FileOpsDelete(filepath, true); + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_rename - Deferred rename via FileOpsRename (always at_commit). + */ +Datum +test_fileops_rename(PG_FUNCTION_ARGS) +{ + char *oldpath = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *newpath = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + if (FileOpsRename(oldpath, newpath) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not rename \"%s\" to \"%s\": %m", + oldpath, newpath))); + pfree(oldpath); + pfree(newpath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_mkdir - Create a directory via FileOpsMkdir. + * + * FileOpsMkdir creates the directory immediately and registers an + * rmdir-on-abort; on commit the directory persists. Used to set up a + * directory the test can then remove with test_fileops_rmdir. + */ +Datum +test_fileops_mkdir(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + int mode = PG_GETARG_INT32(1); + + if (FileOpsMkdir(path, (mode_t) mode) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", path))); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_rmdir - Deferred rmdir via FileOpsRmdir(at_commit=true). + */ +Datum +test_fileops_rmdir(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + FileOpsRmdir(path, true); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_write - Write a string at an offset via FileOpsWrite. + */ +Datum +test_fileops_write(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + int64 offset = PG_GETARG_INT64(1); + char *data = text_to_cstring(PG_GETARG_TEXT_PP(2)); + + (void) FileOpsWrite(path, (off_t) offset, data, (uint32) strlen(data)); + pfree(path); + pfree(data); + PG_RETURN_VOID(); +} + +/* + * test_fileops_chown - Change ownership via FileOpsChown. + */ +Datum +test_fileops_chown(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + (void) FileOpsChown(path, (uid_t) PG_GETARG_INT32(1), + (gid_t) PG_GETARG_INT32(2)); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_symlink - Create a symbolic link via FileOpsSymlink. + */ +Datum +test_fileops_symlink(PG_FUNCTION_ARGS) +{ + char *target = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *linkpath = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + (void) FileOpsSymlink(target, linkpath); + pfree(target); + pfree(linkpath); + PG_RETURN_VOID(); +} diff --git a/src/test/modules/test_fileops/test_fileops.control b/src/test/modules/test_fileops/test_fileops.control new file mode 100644 index 0000000000000..1d39ab2a9166b --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops.control @@ -0,0 +1,4 @@ +comment = 'Test module for transactional file operations (FILEOPS)' +default_version = '1.0' +module_pathname = '$libdir/test_fileops' +relocatable = true diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile index e90e0b1ad4c61..30181a5fd9a48 100644 --- a/src/test/recovery/Makefile +++ b/src/test/recovery/Makefile @@ -14,6 +14,7 @@ EXTRA_INSTALL=contrib/pg_prewarm \ contrib/pg_visibility \ contrib/test_decoding \ src/test/modules/injection_points \ + src/test/modules/test_fileops \ src/test/modules/test_wait_lsn subdir = src/test/recovery diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index cb3df6ed89502..0c8444e39e329 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -70,6 +70,12 @@ tests += { 't/059_remote_apply_status_interval.pl', 't/060_multixact_crashed_updater.pl', 't/061_recovery_target_prefetch.pl', + 't/062_fileops_2pc_deferred.pl', + 't/063_fileops_recovery.pl', + 't/064_fileops_2pc.pl', + 't/065_fileops_standby_divergence.pl', + 't/066_fileops_crash_undo.pl', + 't/067_fileops_createdb_crash.pl', ], }, } diff --git a/src/test/recovery/t/062_fileops_2pc_deferred.pl b/src/test/recovery/t/062_fileops_2pc_deferred.pl new file mode 100644 index 0000000000000..fb6728cab79f7 --- /dev/null +++ b/src/test/recovery/t/062_fileops_2pc_deferred.pl @@ -0,0 +1,172 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test that DEFERRED commit-time FILEOPS (delete) and abort-time filesystem +# undo (of a create and a rename) survive PREPARE TRANSACTION across a crash. +# +# The bug: PostPrepare_FileOps() used to free the pending-op list without +# recording it anywhere, so a transaction PREPARE'd with a deferred delete +# (at_commit=true) lost that op entirely -- COMMIT PREPARED performed +# nothing -- and an immediate create's abort-time undo was lost for ROLLBACK +# PREPARED. The fix serializes the pending ops into the 2PC state file at +# PREPARE (TWOPHASE_RM_FILEOPS_ID records) and performs them in the FILEOPS +# post-commit / post-abort callbacks, so they survive a crash between PREPARE +# and the final decision, driven from the durable 2PC state. +# +# Unlike 064_fileops_2pc.pl (which only exercises chmod, a constructive op +# reversed via the cluster-wide UNDO chain), this test drives the deferred +# commit-time delete path -- the one that was silently dropped -- next to an +# immediate rename, which ROLLBACK PREPARED must rename back. +# +# NB: FILEOPS writes WAL/UNDO only when XLogIsNeeded() (wal_level >= replica), +# so 2PC durability requires wal_level = replica; this test runs there. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_2pc_deferred'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +max_prepared_transactions = 10 +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; + +sub file_exists +{ + my ($path) = @_; + return $node->safe_psql('postgres', + qq{SELECT test_fileops_file_exists('$path')}); +} + +# Poll until the file's existence matches $want ('t'/'f'). +sub wait_for_exists +{ + my ($path, $want, $desc) = @_; + my $ok = $node->poll_query_until('postgres', + qq{SELECT test_fileops_file_exists('$path') = '$want'}); + is($ok, '1', $desc); +} + +# ================================================================ +# Scenario A: COMMIT PREPARED performs the deferred commit-time ops +# - deferred delete of an existing file +# - immediate rename of an existing file +# - immediate create of a new file (register_delete) +# PREPARE, crash, restart, then COMMIT PREPARED. All three must take effect. +# ================================================================ +my $del_a = "$datadir/fo2pc_del_a.dat"; +my $ren_a = "$datadir/fo2pc_ren_a.dat"; +my $ren_a2 = "$datadir/fo2pc_ren_a.renamed"; +my $new_a = "$datadir/fo2pc_new_a.dat"; + +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_del_a.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_ren_a.dat')}); + +is(file_exists($del_a), 't', 'A: delete target exists before txn'); +is(file_exists($ren_a), 't', 'A: rename source exists before txn'); + +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('$del_a'); +SELECT test_fileops_rename('$ren_a', '$ren_a2'); +SELECT test_fileops_create('$new_a', 384); +PREPARE TRANSACTION 'fo2pc_commit'; +)); + +# While prepared, the deferred delete has NOT run yet; the immediate rename +# and create have. +is(file_exists($del_a), 't', + 'A: delete deferred -- file still present while prepared'); +is(file_exists($ren_a), 'f', + 'A: rename immediate -- source gone while prepared'); +is(file_exists($ren_a2), 't', + 'A: rename immediate -- dest present while prepared'); +is(file_exists($new_a), 't', 'A: immediate create present while prepared'); + +# Crash between PREPARE and COMMIT PREPARED. +$node->stop('immediate'); +$node->start; + +is($node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}), + 'fo2pc_commit', 'A: prepared xact survives crash'); + +$node->safe_psql('postgres', q{COMMIT PREPARED 'fo2pc_commit'}); + +is(file_exists($del_a), 'f', + 'A: COMMIT PREPARED performed the deferred delete'); +is(file_exists($ren_a), 'f', + 'A: COMMIT PREPARED kept the rename (source gone)'); +is(file_exists($ren_a2), 't', + 'A: COMMIT PREPARED kept the rename (dest present)'); +is(file_exists($new_a), 't', + 'A: created file persists after COMMIT PREPARED'); + +# ================================================================ +# Scenario B: ROLLBACK PREPARED does NOT perform the deferred commit-time +# delete, and DOES undo the immediate rename and create. +# PREPARE, crash, restart, then ROLLBACK PREPARED. +# ================================================================ +my $del_b = "$datadir/fo2pc_del_b.dat"; +my $ren_b = "$datadir/fo2pc_ren_b.dat"; +my $ren_b2 = "$datadir/fo2pc_ren_b.renamed"; +my $new_b = "$datadir/fo2pc_new_b.dat"; + +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_del_b.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_ren_b.dat')}); + +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('$del_b'); +SELECT test_fileops_rename('$ren_b', '$ren_b2'); +SELECT test_fileops_create('$new_b', 384); +PREPARE TRANSACTION 'fo2pc_rollback'; +)); + +is(file_exists($ren_b2), 't', 'B: immediate rename made while prepared'); +is(file_exists($new_b), 't', 'B: immediate create present while prepared'); + +# Crash between PREPARE and ROLLBACK PREPARED. +$node->stop('immediate'); +$node->start; + +is($node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}), + 'fo2pc_rollback', 'B: prepared xact survives crash'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo2pc_rollback'}); + +is(file_exists($del_b), 't', + 'B: ROLLBACK PREPARED did NOT delete (file still present)'); +is(file_exists($ren_b), 't', + 'B: ROLLBACK PREPARED renamed back (source present)'); +is(file_exists($ren_b2), 'f', + 'B: ROLLBACK PREPARED renamed back (dest absent)'); +# The immediate create is undone by the constructive op's cluster-wide UNDO +# record on ROLLBACK PREPARED. +wait_for_exists($new_b, 'f', + 'B: ROLLBACK PREPARED undid the immediate create'); + +# Server healthy at the end. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after all deferred-2PC scenarios'); + +$node->stop; +done_testing(); diff --git a/src/test/recovery/t/063_fileops_recovery.pl b/src/test/recovery/t/063_fileops_recovery.pl new file mode 100644 index 0000000000000..fb8f23b63ae72 --- /dev/null +++ b/src/test/recovery/t/063_fileops_recovery.pl @@ -0,0 +1,423 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test crash recovery for transactional file operations (FILEOPS). +# +# These tests verify that FILEOPS WAL replay correctly handles: +# - Crash during file creation (with delete-on-abort) +# - Crash during deferred file deletion +# - Crash during file operations on standby +# - Multiple sequential crashes + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_recovery'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +log_min_messages = debug2 +)); +$node->start; + +# ================================================================ +# Test 1: CREATE TABLE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE fileops_test (id int, data text); +INSERT INTO fileops_test VALUES (1, 'created_table'); +)); + +$node->stop('immediate'); +$node->start; + +my $result = + $node->safe_psql("postgres", "SELECT data FROM fileops_test WHERE id = 1"); +is($result, 'created_table', 'CREATE TABLE survives crash'); + +# ================================================================ +# Test 2: DROP TABLE is properly handled after crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE drop_me (id int); +INSERT INTO drop_me VALUES (1); +)); + +# Get the relfilenode before dropping +my $relpath = + $node->safe_psql("postgres", "SELECT pg_relation_filepath('drop_me')"); + +$node->safe_psql("postgres", "DROP TABLE drop_me"); + +$node->stop('immediate'); +$node->start; + +# Table should be gone +my ($ret, $stdout, $stderr) = + $node->psql("postgres", "SELECT * FROM drop_me"); +isnt($ret, 0, 'dropped table is gone after crash recovery'); + +# ================================================================ +# Test 3: Crash during transaction with CREATE TABLE (uncommitted) +# ================================================================ + +# This table is committed +$node->safe_psql( + "postgres", qq( +CREATE TABLE committed_table (id int); +INSERT INTO committed_table VALUES (42); +)); + +# Crash the server +$node->stop('immediate'); +$node->start; + +# Committed table should exist +$result = $node->safe_psql("postgres", "SELECT id FROM committed_table"); +is($result, '42', 'committed CREATE TABLE survives crash'); + +# ================================================================ +# Test 4: Multiple CREATE and DROP operations then crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE t1 (id int); +CREATE TABLE t2 (id int); +CREATE TABLE t3 (id int); +INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (2); +INSERT INTO t3 VALUES (3); +DROP TABLE t2; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT id FROM t1"); +is($result, '1', 't1 survives crash'); + +($ret, $stdout, $stderr) = $node->psql("postgres", "SELECT * FROM t2"); +isnt($ret, 0, 't2 (dropped) is gone after crash'); + +$result = $node->safe_psql("postgres", "SELECT id FROM t3"); +is($result, '3', 't3 survives crash'); + +# ================================================================ +# Test 5: Crash after checkpoint with file operations +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t3; +CREATE TABLE checkpoint_test (id int); +INSERT INTO checkpoint_test VALUES (1); +CHECKPOINT; +INSERT INTO checkpoint_test VALUES (2); +)); + +$node->stop('immediate'); +$node->start; + +$result = + $node->safe_psql("postgres", "SELECT count(*) FROM checkpoint_test"); +is($result, '2', 'data after checkpoint survives crash'); + +# ================================================================ +# Test 6: Multiple crashes in sequence with file operations +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS checkpoint_test; +CREATE TABLE multi_crash (id int); +INSERT INTO multi_crash VALUES (1); +)); + +$node->stop('immediate'); +$node->start; + +$node->safe_psql( + "postgres", qq( +INSERT INTO multi_crash VALUES (2); +CREATE TABLE multi_crash_2 (id int); +INSERT INTO multi_crash_2 VALUES (10); +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT count(*) FROM multi_crash"); +is($result, '2', 'multi_crash table correct after double crash'); + +$result = $node->safe_psql("postgres", "SELECT id FROM multi_crash_2"); +is($result, '10', 'multi_crash_2 table correct after double crash'); + +# ================================================================ +# Test 7: CREATE TABLESPACE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS multi_crash; +DROP TABLE IF EXISTS multi_crash_2; +SET allow_in_place_tablespaces = on; +CREATE TABLESPACE fileops_crash_ts LOCATION ''; +CREATE TABLE ts_crash_table (id int) TABLESPACE fileops_crash_ts; +INSERT INTO ts_crash_table VALUES (99); +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT id FROM ts_crash_table"); +is($result, '99', 'table in tablespace survives crash'); + +$result = $node->safe_psql("postgres", + "SELECT spcname FROM pg_tablespace WHERE spcname = 'fileops_crash_ts'"); +is($result, 'fileops_crash_ts', 'tablespace survives crash'); + +# ================================================================ +# Test 8: DROP TABLESPACE completes after crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE ts_crash_table; +DROP TABLESPACE fileops_crash_ts; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", + "SELECT count(*) FROM pg_tablespace WHERE spcname = 'fileops_crash_ts'"); +is($result, '0', 'dropped tablespace is gone after crash'); + +# ================================================================ +# Test 9: CREATE DATABASE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE DATABASE fileops_crash_db; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", + "SELECT datname FROM pg_database WHERE datname = 'fileops_crash_db'"); +is($result, 'fileops_crash_db', 'CREATE DATABASE survives crash'); + +$node->safe_psql("postgres", "DROP DATABASE fileops_crash_db"); + +# ================================================================ +# Test 10: Standby crash during FILEOPS replay +# ================================================================ + +# Set up primary + standby +my $primary = PostgreSQL::Test::Cluster->new('fileops_primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + "postgresql.conf", qq( +autovacuum = off +)); +$primary->start; +$primary->backup('backup'); + +my $standby = PostgreSQL::Test::Cluster->new('fileops_standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; + +# Create table on primary and wait for standby to catch up +$primary->safe_psql( + "postgres", qq( +CREATE TABLE standby_test (id int); +INSERT INTO standby_test VALUES (1); +)); + +$primary->wait_for_catchup($standby); + +# Verify on standby +$result = $standby->safe_psql("postgres", "SELECT id FROM standby_test"); +is($result, '1', 'CREATE TABLE replicated to standby'); + +# Crash the standby +$standby->stop('immediate'); +$standby->start; + +# Add more data on primary +$primary->safe_psql( + "postgres", qq( +INSERT INTO standby_test VALUES (2); +)); + +$primary->wait_for_catchup($standby); + +$result = + $standby->safe_psql("postgres", "SELECT count(*) FROM standby_test"); +is($result, '2', 'standby recovers and catches up after crash'); + +# Clean up primary/standby +$standby->stop; +$primary->stop; + +# Clean up original node +$node->stop; + +# ================================================================ +# Test 11/12: Crash right after an immediate op's WAL is flushed, before +# its transaction commits (FileOpsCreate, FileOpsChmod). +# +# An immediate op makes its filesystem change first and then flushes its +# WAL and UNDO records. The injection points pause the backend just after +# that flush, inside the still-open implicit transaction, and the postmaster +# is then hard-killed. Redo replays the change, and crash recovery applies +# the UNDO of the never-committed transaction, so the end state must be as +# if the transaction never ran: no orphaned file, no leftover mode change. +# ================================================================ + +if ($ENV{enable_injection_points} eq 'yes') +{ + my $inode = PostgreSQL::Test::Cluster->new('fileops_wal_ordering'); + $inode->init; + $inode->append_conf( + 'postgresql.conf', qq( +shared_preload_libraries = 'injection_points' +autovacuum = off +log_min_messages = debug2 +wal_level = replica +)); + $inode->start; + + if ( $inode->check_extension('injection_points') + && $inode->check_extension('test_fileops')) + { + $inode->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + $inode->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + + my $idatadir = $inode->data_dir; + + # --- Test 11: FileOpsCreate ----------------------------------- + $inode->safe_psql('postgres', + q{SELECT injection_points_attach('fileops-create-after-wal-flush', 'wait')} + ); + + my $create_path = "$idatadir/wal_ordering_create.dat"; + my $bgpsql = $inode->background_psql('postgres'); + $bgpsql->query_until( + qr/starting_create/, + qq(\\echo starting_create +SELECT test_fileops_create('$create_path', 384); +\\q +)); + + $inode->wait_for_event('client backend', + 'fileops-create-after-wal-flush'); + + # The file exists and its WAL is durable; the transaction is open. + ok(-e $create_path, + 'Test 11: file exists at the injection point (WAL durable)'); + + # Hard-kill the postmaster from exactly this window. The wrapping + # implicit transaction is still in progress (blocked on the + # injection point), so it never committed. + $inode->stop('immediate'); + $inode->start; + + # Recovery redoes the physical CREATE from the durable WAL record + # unconditionally, then determines the transaction never committed + # and applies its UNDO record to reverse the create. Net effect: + # the file must NOT exist -- exactly as if the transaction had + # never run, with no orphaned state left behind by the crash. + ok( !-e $create_path, + 'Test 11: FileOpsCreate of a not-committed transaction is ' + . 'reversed after a crash'); + + # Server is healthy after the reconciliation. + my $result11 = $inode->safe_psql('postgres', 'SELECT 1'); + is($result11, '1', 'Test 11: server operational after recovery'); + + $bgpsql->{run}->finish; + + # --- Test 12: FileOpsChmod ------------------------------------- + $inode->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('wal_ordering_chmod.dat')} + ); + my $chmod_path = "$idatadir/wal_ordering_chmod.dat"; + $inode->safe_psql('postgres', + qq{SELECT test_fileops_chmod('$chmod_path', 420)}); # 0644 + my $mode_before = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + # Windows has no POSIX file modes: chmod only toggles read-only. + my $posix_modes = !$windows_os; + is($mode_before, '420', 'Test 12: baseline mode is 0644') + if $posix_modes; + + $inode->safe_psql('postgres', + q{SELECT injection_points_attach('fileops-chmod-after-wal-flush', 'wait')} + ); + + $bgpsql = $inode->background_psql('postgres'); + $bgpsql->query_until( + qr/starting_chmod/, + qq(\\echo starting_chmod +SELECT test_fileops_chmod('$chmod_path', 448); -- 0700 +\\q +)); + + $inode->wait_for_event('client backend', + 'fileops-chmod-after-wal-flush'); + + # chmod() has run and its WAL is durable; the transaction is open. + my $mode_at_injection = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + is($mode_at_injection, '448', + 'Test 12: mode changed on disk at the injection point (WAL durable)' + ) if $posix_modes; + + # Hard-kill, same as Test 11. The chmod's wrapping implicit + # transaction never committed. + $inode->stop('immediate'); + $inode->start; + + # Recovery redoes the physical CHMOD unconditionally, then reverses + # it via UNDO because the transaction never committed. Net effect: + # mode must be back to the pre-transaction baseline (0644), not the + # attempted 0700, and not some undefined intermediate state. + my $mode_after = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + is($mode_after, '420', + 'Test 12: FileOpsChmod of a not-committed transaction is ' + . 'reversed after a crash') + if $posix_modes; + + # Server is healthy after the reconciliation. + my $result12 = $inode->safe_psql('postgres', 'SELECT 1'); + is($result12, '1', 'Test 12: server operational after recovery'); + + $bgpsql->{run}->finish; + } + else + { + note + 'skipping Test 11/12 (injection_points or test_fileops extension not installed)'; + } + + $inode->stop; +} +else +{ + note 'skipping Test 11/12 (injection points not supported by this build)'; +} + +done_testing(); diff --git a/src/test/recovery/t/064_fileops_2pc.pl b/src/test/recovery/t/064_fileops_2pc.pl new file mode 100644 index 0000000000000..b88c8dd59103b --- /dev/null +++ b/src/test/recovery/t/064_fileops_2pc.pl @@ -0,0 +1,149 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test two-phase commit (PREPARE TRANSACTION / COMMIT PREPARED / ROLLBACK +# PREPARED) for transactions that generated cluster-wide FILEOPS UNDO. +# +# FILEOPS uses the undo-in-common-WAL mechanism. At PREPARE the permanent +# UNDO chain-head LSN is durably saved in the 2PC state (xl_xact_prepare / +# gxact->undo_batch_lsn), and its WAL is retained while the xact stays +# prepared. ROLLBACK PREPARED (even from a different backend, post-crash) +# applies that chain, which reverses the filesystem change. COMMIT PREPARED +# keeps it. +# +# Vehicle: the test_fileops extension's SQL wrappers for FileOpsChmod, which +# are legal inside a transaction block that can then be PREPARE'd. +# +# NB: FILEOPS only writes UNDO WAL when XLogIsNeeded() (wal_level >= replica). +# At wal_level = minimal it relies on the backend-private pending-op list, +# which is discarded at PREPARE and cannot survive 2PC. This test therefore +# runs at wal_level = replica, which is what any FILEOPS + 2PC deployment +# requires. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Every test here is a chmod reversal; Windows has no POSIX file modes. +plan skip_all => 'POSIX permission semantics required' if $windows_os; + +my $node = PostgreSQL::Test::Cluster->new('fileops_2pc'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +max_prepared_transactions = 10 +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; +my $file = "$datadir/twopc.dat"; + +# Poll until the file mode equals $want, or fail. +sub wait_for_mode +{ + my ($want, $desc) = @_; + my $ok = $node->poll_query_until('postgres', + qq{SELECT test_fileops_get_mode('$file') = $want}); + is($ok, '1', $desc); +} + +# 0644 = 420, 0700 = 448, 0600 = 384 (decimal) +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('twopc.dat')}); +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); +my $baseline = + $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($baseline, '420', 'baseline mode is 0644'); + +# ================================================================ +# Test 1: ROLLBACK PREPARED reverses the chmod +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_rb'; +)); +my $m = + $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 1: chmod applied and still visible while prepared'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_rb'}); +wait_for_mode(420, 'Test 1: ROLLBACK PREPARED reverses chmod (0700 -> 0644)'); + +# ================================================================ +# Test 2: COMMIT PREPARED keeps the chmod +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_cp'; +)); +$node->safe_psql('postgres', q{COMMIT PREPARED 'fo_cp'}); +$m = $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 2: COMMIT PREPARED keeps chmod (0700)'); + +# Reset to baseline for the crash test. +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); + +# ================================================================ +# Test 3: WAL retention across checkpoints while prepared +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 384); -- 0600 +PREPARE TRANSACTION 'fo_ckpt'; +)); +# Churn WAL and checkpoint; the prepared xact's UNDO batch must be retained. +$node->safe_psql('postgres', 'CHECKPOINT'); +for my $i (1 .. 20) +{ + $node->safe_psql('postgres', + "CREATE TABLE junk_$i(x int); " + . "INSERT INTO junk_$i SELECT g FROM generate_series(1,5000) g; " + . "DROP TABLE junk_$i;"); +} +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_ckpt'}); +wait_for_mode(420, + 'Test 3: ROLLBACK PREPARED still reverses after WAL churn + checkpoints'); + +# ================================================================ +# Test 4: crash recovery -- prepared UNDO xact survives, then reverts +# ================================================================ +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_crash'; +)); +$node->stop('immediate'); +$node->start; + +my $prep = $node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}); +is($prep, 'fo_crash', 'Test 4: prepared xact survives crash recovery'); +$m = $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 4: chmod still applied (prepared, not yet rolled back)'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_crash'}); +wait_for_mode(420, + 'Test 4: ROLLBACK PREPARED after crash+restart reverses chmod'); + +# Server healthy at the end. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after all 2PC UNDO scenarios'); + +$node->stop; +done_testing(); diff --git a/src/test/recovery/t/065_fileops_standby_divergence.pl b/src/test/recovery/t/065_fileops_standby_divergence.pl new file mode 100644 index 0000000000000..65a07fb873e04 --- /dev/null +++ b/src/test/recovery/t/065_fileops_standby_divergence.pl @@ -0,0 +1,124 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Regression test: physical-standby reproduction of FILEOPS deferred +# DESTRUCTIVE operations (DELETE, RENAME, RMDIR). +# +# These three ops execute on the primary at commit inside +# FileOpsDoPendingOps(). Each is logged at commit-EXECUTION time with a +# real, idempotent redo handler, so a physical standby (which only replays +# WAL, never running FileOpsDoPendingOps) reproduces them and its filesystem +# converges with the primary's. +# +# This test sets up a streaming physical standby, performs a committed +# deferred DELETE, RENAME, and RMDIR on the primary, waits for catchup, and +# asserts the standby's filesystem MATCHES the primary's. It MUST FAIL on the +# pre-fix (no-op redo) code -- see the header note in fileops.c and the +# "prove non-vacuity" comment below. +# +# All FILEOPS paths are RELATIVE to the data directory, matching every core +# FILEOPS caller (dbcommands.c, tablespace.c): the backend's cwd is PGDATA, so +# a relative path resolves to the right file on both the primary and a standby +# whose absolute data_dir differs. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('fileops_div_primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + "postgresql.conf", qq( +autovacuum = off +wal_level = replica +)); +$primary->start; + +if (!$primary->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $primary_dir = $primary->data_dir; + +# Fixture, all committed and WAL-logged so the standby has them before we +# start mutating. Use FileOps CREATE (WAL-logged, standby-reproduced) and +# FileOps MKDIR (WAL-logged) rather than raw open(), so the fixture itself +# replicates. Paths are relative to PGDATA. +$primary->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_create('div_delete_me.dat', 384); -- 0600 +SELECT test_fileops_create('div_rename_src.dat', 384); +SELECT test_fileops_mkdir('div_rmdir_me', 448); -- 0700 +COMMIT; +)); + +# Take the base backup AFTER the fixture exists so init_from_backup gives the +# standby the fixture files, then stream the mutations. +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('fileops_div_standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; + +my $standby_dir = $standby->data_dir; + +$primary->wait_for_catchup($standby); + +# Sanity: the fixture replicated to the standby. +ok(-e "$standby_dir/div_delete_me.dat", 'fixture: delete target on standby'); +ok(-e "$standby_dir/div_rename_src.dat", 'fixture: rename source on standby'); +ok(-d "$standby_dir/div_rmdir_me", 'fixture: rmdir target on standby'); + +# Perform the three deferred destructive ops in a committed transaction on the +# primary. Each goes through the FileOps deferred path (at_commit=true) and, +# with the fix, is WAL-logged at commit-execution. +$primary->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('div_delete_me.dat'); +SELECT test_fileops_rename('div_rename_src.dat', 'div_rename_dst.dat'); +SELECT test_fileops_rmdir('div_rmdir_me'); +COMMIT; +)); + +# Confirm the primary applied them (the op ran in FileOpsDoPendingOps). +ok(!-e "$primary_dir/div_delete_me.dat", 'primary: delete applied'); +ok(!-e "$primary_dir/div_rename_src.dat", 'primary: rename source gone'); +ok(-e "$primary_dir/div_rename_dst.dat", 'primary: rename dest present'); +ok(!-d "$primary_dir/div_rmdir_me", 'primary: rmdir applied'); + +$primary->wait_for_catchup($standby); + +# The diff the reviewer asked for: the standby's filesystem must MATCH the +# primary's. Pre-fix (no-op redo) the standby still has the original file / +# dir and lacks the renamed file -- these four assertions all fail, which is +# the non-vacuity proof. Post-fix they pass. +ok(!-e "$standby_dir/div_delete_me.dat", + 'deferred DELETE reproduced on standby (file gone)'); +ok(!-e "$standby_dir/div_rename_src.dat", + 'deferred RENAME reproduced on standby (old name gone)'); +ok(-e "$standby_dir/div_rename_dst.dat", + 'deferred RENAME reproduced on standby (new name present)'); +ok(!-d "$standby_dir/div_rmdir_me", + 'deferred RMDIR reproduced on standby (dir gone)'); + +# Idempotent redo across a standby restart: replaying the same records again +# after a crash must not error or diverge. +$standby->stop('immediate'); +$standby->start; +$primary->safe_psql('postgres', 'SELECT txid_current()'); +$primary->wait_for_catchup($standby); +ok(!-e "$standby_dir/div_delete_me.dat", + 'standby end state stable after restart (delete)'); +ok(-e "$standby_dir/div_rename_dst.dat", + 'standby end state stable after restart (rename)'); +ok(!-d "$standby_dir/div_rmdir_me", + 'standby end state stable after restart (rmdir)'); + +$standby->stop; +$primary->stop; + +done_testing(); diff --git a/src/test/recovery/t/066_fileops_crash_undo.pl b/src/test/recovery/t/066_fileops_crash_undo.pl new file mode 100644 index 0000000000000..f305b2814bdad --- /dev/null +++ b/src/test/recovery/t/066_fileops_crash_undo.pl @@ -0,0 +1,137 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# FILEOPS crash-recovery revert via the cluster-wide UNDO engine (common WAL). +# +# FILEOPS is a consumer of the cluster-wide UNDO engine: its structural +# operations (file existence, permissions, directory structure) are +# WAL-durable via RM_FILEOPS_ID forward records (fileops_redo), and each +# reversible op also writes an UNDO record into the common WAL +# (PrepareXactUndoData(UNDO_RMID_FILEOPS,...) / InsertXactUndoData) in the +# same critical section as its forward record. Filesystem structural +# operations are cluster-wide changes whose durability/revert belong in the +# common WAL. This test proves that model is crash-safe. +# +# A transaction produces FILEOPS create/chmod (immediate, WAL-flushed), then the +# server is crashed (immediate stop) BEFORE the transaction commits. On +# recovery: +# - fileops_redo() REDOES the forward create/chmod unconditionally, and +# - UndoRecoveryRollBack() (end of recovery) applies the never-committed +# transaction's FILEOPS UNDO chain via the RM_FILEOPS_ID rm_undo +# callbacks (fileops_undo.c), reversing every op: +# - the created file is unlinked (CREATE revert), and +# - the chmod'd file's mode is restored (CHMOD revert, before-state in WAL). +# +# FILEOPS only writes WAL at wal_level >= replica (XLogIsNeeded()), so the test +# runs at wal_level = replica, matching any FILEOPS deployment. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_crash'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +autovacuum = off +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; +my $create_path = "$datadir/fileops_crash_create.dat"; +my $chmod_path = "$datadir/fileops_crash_chmod.dat"; + +# --------------------------------------------------------------------------- +# Set up a pre-existing file to chmod (this part is committed). 0644 = 420. +# --------------------------------------------------------------------------- +$node->safe_psql('postgres', + q{SELECT test_fileops_create_tempfile('fileops_crash_chmod.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_chmod('$chmod_path', 420)}); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '420', + 'baseline mode of pre-existing file is 0644'); +} + +# --------------------------------------------------------------------------- +# Open a transaction that produces FILEOPS UNDO (create a new file and +# chmod the existing one) and DO NOT commit it. The FILEOPS forward operations +# execute and are WAL-flushed immediately (the file exists on disk, the mode is +# changed), but the wrapping transaction stays in progress. +# --------------------------------------------------------------------------- +my $bg = $node->background_psql('postgres'); +$bg->query_until( + qr/fileops_ready/, qq( +BEGIN; +SELECT test_fileops_create('$create_path', 384); -- 0600, new file +SELECT test_fileops_chmod('$chmod_path', 448); -- 0700, existing file +\\echo fileops_ready +)); + +# The forward ops are visible on disk while the xact is in progress. +ok(-e $create_path, + 'created file exists on disk before crash (in-progress xact)'); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '448', + 'chmod applied on disk before crash (in-progress xact)'); +} + +# --------------------------------------------------------------------------- +# Crash immediately: the transaction never commits. Stop the server before +# closing the client: closing it first would let the backend roll the +# transaction back online, which is a different case. The background psql +# connection dies with the server; abandon it (do not try to drain/finish it, +# which would hang waiting on the dead connection). +# --------------------------------------------------------------------------- +$node->stop('immediate'); +$bg->{run}->kill_kill; +$node->start; + +# --------------------------------------------------------------------------- +# Recovery redoes the forward ops from WAL, then UndoRecoveryRollBack() reverses +# the never-committed transaction's ops from its common-WAL UNDO chain: +# - the created file is unlinked, and +# - the chmod is restored to the original 0644. +# --------------------------------------------------------------------------- +ok( !-e $create_path, + 'FILEOPS COMMON-WAL CRASH RECOVERY: created file was REVERTED (unlinked) after crash' +); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '420', + 'FILEOPS COMMON-WAL CRASH RECOVERY: chmod was REVERTED (0700 -> 0644) after crash' + ); +} + +# Recovery must be clean. +my $rlog = slurp_file($node->logfile); +unlike( + $rlog, + qr/PANIC|inconsistent|corrupt/i, + 'clean crash recovery, no PANIC/corruption'); + +# Server healthy afterwards. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after crash-recovery undo'); + +$node->stop; +done_testing(); diff --git a/src/test/recovery/t/067_fileops_createdb_crash.pl b/src/test/recovery/t/067_fileops_createdb_crash.pl new file mode 100644 index 0000000000000..108a932fc0288 --- /dev/null +++ b/src/test/recovery/t/067_fileops_createdb_crash.pl @@ -0,0 +1,113 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +# A CREATE DATABASE that crashes, or fails, part way through its copy must +# leave no database directory behind, for either strategy. The directory +# is created by FILEOPS but filled by smgr (WAL_LOG) or copydir (FILE_COPY), +# so its UNDO must remove the whole tree, not just rmdir it. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('createdb_crash'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +shared_preload_libraries = 'injection_points' +wal_level = replica +autovacuum = off +}); +$node->start; +if (!$node->check_extension('injection_points')) +{ + plan skip_all => 'Extension injection_points not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); +$node->safe_psql('postgres', 'CREATE DATABASE tmpl'); +$node->safe_psql('tmpl', + "CREATE TABLE t AS SELECT g, repeat('x', 200) v FROM generate_series(1, 20000) g" +); + +my $base = $node->data_dir . '/base'; + +# Directories under base/ that belong to no database. +sub orphans +{ + my %known = map { $_ => 1 } split /\n/, + $node->safe_psql('postgres', 'SELECT oid FROM pg_database'); + opendir(my $dh, $base) or die "opendir $base: $!"; + my @o = grep { /^\d+$/ && !$known{$_} } readdir($dh); + closedir($dh); + return join(' ', sort @o); +} + +my %point = ( + wal_log => 'createdb-wal-log-after-copy-relation', + file_copy => 'createdb-file-copy-after-copydir'); + +foreach my $strategy ('wal_log', 'file_copy') +{ + my $ip = $point{$strategy}; + my %before = map { $_ => 1 } split / /, orphans(); + + # Crash while the new directory holds copied files. + $node->safe_psql('postgres', + "SELECT injection_points_attach('$ip', 'wait')"); + my $bg = $node->background_psql('postgres', on_error_stop => 0); + $bg->query_until( + qr/start/, qq(\\echo start +CREATE DATABASE c_$strategy TEMPLATE tmpl STRATEGY $strategy; +)); + $node->wait_for_event('client backend', $ip); + + my ($new) = grep { !$before{$_} } split / /, orphans(); + ok(defined $new, "$strategy: the new database directory exists"); + $new //= 'none'; + my @files = glob("$base/$new/*"); + cmp_ok(scalar(@files), '>', 1, + "$strategy: it holds copied files at the crash"); + + my $log_offset = -s $node->logfile; + $node->stop('immediate'); + $bg->{run}->finish; + $node->start; + + is(orphans(), '', "$strategy: no database directory left after a crash"); + my $log = slurp_file($node->logfile, $log_offset); + unlike( + $log, + qr/WARNING: FILEOPS UNDO/, + "$strategy: crash recovery applied the UNDO cleanly"); + + # The same, with the copy failing instead. + $node->safe_psql('postgres', + "SELECT injection_points_attach('$ip', 'error')"); + my ($ret, $out, $err) = $node->psql('postgres', + "CREATE DATABASE e_$strategy TEMPLATE tmpl STRATEGY $strategy"); + like($err, qr/ERROR/, "$strategy: CREATE DATABASE failed mid-copy"); + $node->safe_psql('postgres', "SELECT injection_points_detach('$ip')"); + is(orphans(), '', "$strategy: no database directory left after an error"); + + # A later CREATE DATABASE still works. + $node->safe_psql('postgres', + "CREATE DATABASE ok_$strategy TEMPLATE tmpl STRATEGY $strategy"); + is($node->safe_psql("ok_$strategy", 'SELECT count(*) FROM t'), + '20000', "$strategy: a new CREATE DATABASE succeeds"); +} + +# No failed rmdir of a non-empty directory anywhere in the log. +sleep(1); +unlike( + slurp_file($node->logfile), + qr/could not rmdir .*Directory not empty/, + 'no failed rmdir of a non-empty directory'); + +$node->stop; +done_testing(); diff --git a/src/tools/pginclude/headerscheck b/src/tools/pginclude/headerscheck index 785d6f867ad09..29c63abfb2ba8 100755 --- a/src/tools/pginclude/headerscheck +++ b/src/tools/pginclude/headerscheck @@ -129,6 +129,7 @@ do # These files are not meant to be included standalone, because # they contain lists that might have multiple use-cases. test "$f" = src/include/access/rmgrlist.h && continue + test "$f" = src/include/access/undormgrlist.h && continue test "$f" = src/include/parser/kwlist.h && continue test "$f" = src/include/postmaster/proctypelist.h && continue test "$f" = src/include/regex/regerrs.h && continue diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f10eb46926676..6e627beebb0fd 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -618,6 +618,7 @@ DBState DCHCacheEntry DEADLOCK_INFO DECountItem +DeferredUndoXact DH DIR DNSServiceErrorType @@ -835,6 +836,19 @@ ExtensionVersionInfo FDWCollateState FD_SET FILE +FileOpsSyncEntry +FileOpsTwoPhaseRecord +FileopsUndoChmod +FileopsUndoChown +FileopsUndoCreate +FileopsUndoLink +FileopsUndoMkdir +FileopsUndoRemovexattr +FileopsUndoRename +FileopsUndoSetxattr +FileopsUndoSymlink +FileopsUndoTruncate +FileopsUndoWrite FILETIME FPI FSMAddress @@ -1911,6 +1925,8 @@ PACE_HEADER PACL PATH PCtxtHandle +PendingFileOp +PendingFileOpType PERL_CONTEXT PERL_SI PFN @@ -3312,6 +3328,16 @@ UINT ULARGE_INTEGER ULONG ULONG_PTR +UndoApplyResult +UndoBatchData +UndoPersistenceLevel +UndoRecordHeader +UndoRecordSet +UndoRecoveryEntry +UndoRecPtr +UndoRmgrApplyFunc +UndoRmgrData +UndoRmgrDescFunc UV UVersionInfo UnicodeNormalizationForm @@ -3492,6 +3518,27 @@ X509_NAME_ENTRY X509_STORE X509_STORE_CTX X86FeatureId +XactUndoContext +XactUndoData +XactUndoInflightSlot +XactUndoSubTransactionState +xl_fileops_chmod +xl_fileops_chown +xl_fileops_create +xl_fileops_delete +xl_fileops_link +xl_fileops_mkdir +xl_fileops_removexattr +xl_fileops_rename +xl_fileops_rmdir +xl_fileops_setxattr +xl_fileops_symlink +xl_fileops_truncate +xl_fileops_write +xl_undo_batch +xl_undo_inflight +xl_undo_inflight_xact +xl_undo_rollback XLTW_Oper XLogCtlData XLogCtlInsert