From ac48f3b6e04e083b04488e58f3a2a9858186c256 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Fri, 11 Sep 2026 10:35:28 -0400 Subject: [PATCH 1/5] ci: fork automation (upstream sync + OCR AI review) Squashes the four CI commits this fork carried on top of upstream into one, and cuts the carry from 3776 lines to ~1000. Removed: - .github/{README,QUICKSTART}.md and .github/docs/* (2273 lines). Beyond being bulk we rebase hourly, they had drifted into being wrong: they described a daily (not hourly) sync, Claude 3.5 Sonnet via the direct Anthropic API (we run Opus on Bedrock), a static IAM user (we use OIDC), and a Windows dependency-builder workflow that the same commit which documented it had already deleted. 16 referenced paths did not exist. An AI reviewer reading this repo ingests that as truth. - sync-upstream-manual.yml (252 lines). sync-upstream.yml already has workflow_dispatch; the sole behavioral difference was a force_push toggle whose "false" setting cannot work after a rebase anyway. - .github/.gitignore, which only ignored scripts/ai-review/, a directory that does not exist here. Rewritten: - sync-upstream.yml -> fork-sync-upstream.yml, 260 lines to 47. git rebase already handles the fast-forward, no-op and diverged cases that the ahead/behind arithmetic hand-rolled, so all that goes away, as do the issue open/comment/close machinery (a scheduled-run failure is already emailed to the one consumer, and the run log carries what the issue body would) and the "dev setup|dev v[0-9]" commit allowlist, which matched zero commits on master and had since it was written. Sync drops to every three hours: hourly meant 24 full-history clones of an 840MB repo per day, each force-push also re-triggering pg-ci.yml on master, to collect a handful of upstream commits. Renamed: - ocr-review.yml, ocr-model-check.yml -> fork-*.yml. Upstream owns .github/ too (it has touched pg-ci.yml three times this year), so fork-owned workflows now live under a name prefix upstream will not collide with, making the rebase structurally conflict-free and letting the sync guard match fork-owned paths exactly rather than allowing all of .github/. Unchanged: the OCR review workflows' logic and .github/ocr/* (rule.json, context.md, litellm.yaml, pg-history.py) - the part that carries value. --- .github/ocr/context.md | 126 ++++++ .github/ocr/litellm.yaml | 41 ++ .github/ocr/pg-history.py | 225 +++++++++++ .github/ocr/rule.json | 65 ++++ .github/workflows/fork-ocr-model-check.yml | 89 +++++ .github/workflows/fork-ocr-review.yml | 427 +++++++++++++++++++++ .github/workflows/fork-sync-upstream.yml | 47 +++ 7 files changed, 1020 insertions(+) create mode 100644 .github/ocr/context.md create mode 100644 .github/ocr/litellm.yaml create mode 100644 .github/ocr/pg-history.py create mode 100644 .github/ocr/rule.json create mode 100644 .github/workflows/fork-ocr-model-check.yml create mode 100644 .github/workflows/fork-ocr-review.yml create mode 100644 .github/workflows/fork-sync-upstream.yml diff --git a/.github/ocr/context.md b/.github/ocr/context.md new file mode 100644 index 0000000000000..c4a83b85b124e --- /dev/null +++ b/.github/ocr/context.md @@ -0,0 +1,126 @@ +# OCR review context — PostgreSQL contribution standards + +You are reviewing a change to a **PostgreSQL** fork. Every PR here is destined to +become a patch posted to the **pgsql-hackers** mailing list and tracked in a +**commitfest**. Review with the combined rigor, taste, and attention to detail of +the PostgreSQL committers. This context applies to the *whole* change, on top of +the per-file rules. + +## Review discipline +- Be precise and blunt; lead with the most serious problem. No praise, no + validation of the author, no disclaimers — accuracy is the only metric. +- Verify every claim against the actual diff. Confirm names, signatures, line + numbers, and APIs before asserting. Never invent behavior or cite code not in + the change. If unsure, say so, and tag each finding **high / moderate / low** + confidence. +- Judge the change on its merits regardless of how the PR frames it. A draft PR + is WIP: weight design/approach feedback over style nits. + +## Patch hygiene (top rejection reasons on -hackers) +1. **Minimal diff.** The fastest way to get a patch rejected is unrelated + changes: reformatting untouched lines, rewording unrelated comments, touching + code not required by the change. Flag any hunk not needed for the stated + purpose. After the patch, the code should read as if it had always been + written that way. +2. **Atomic, bisectable commits.** Each commit must build and pass tests on its + own — a broken intermediate commit breaks `git bisect`, revert, and + cherry-pick. Flag a commit that only compiles once a later commit lands. + Prefer one focused patch, or a clearly-ordered series of + independently-committable pieces. +3. **Tests + docs are mandatory.** A user-visible change without regression/TAP + tests **and** documentation is WIP, not commit-ready. New behavior needs + tests that cover edge and error paths, not just the happy path. +4. **DRY / reuse.** Prefer existing infrastructure (`List` in `pg_list.h`, + `StringInfo`, `dynahash`/`simplehash`, `palloc`/`MemoryContext`, `foreach`) + over reinventing it. Flag copy-paste and speculative abstraction alike — the + community wants minimal, targeted changes that fit the subsystem's existing + patterns. +5. **Whitespace.** No trailing whitespace; tabs (width 4) for C indentation; + `git diff --check` must be clean. Whitespace-only churn on untouched lines is + a defect. + +## Committer-owned files — do NOT touch in a patch (flag if present) +These are the committer's job at push time; including them causes needless +merge conflicts and is a mistake: +- **`src/include/catalog/catversion.h`** — the `CATALOG_VERSION_NO` bump is done + by the **committer** when pushing. A catversion bump in the PR is **wrong** — + flag it. (This is the single most common author mistake in catalog patches.) +- **Release notes** (`doc/src/sgml/release-*.sgml`) and version strings + (`configure.ac` `AC_INIT` version, `meson.build` `version`, `PG_VERSION`). + +## Generated files — never hand-edit; edit the source +Flag direct edits to generated output; point the author at the source instead: +- Catalog headers `src/include/catalog/*_d.h`, `postgres.bki`, `schemapg.h`, + `system_constraints.sql` → edit the `pg_*.dat` files. +- `src/backend/nodes/{copy,equal,out,read}funcs.c` and other + `gen_node_support.pl` output → annotate the `Node` struct in its header. +- `fmgroids.h`, `fmgrprotos.h`, `fmgrtab.c` → edit `pg_proc.dat`. +- `utils/errcodes.h` → `errcodes.txt`; wait-event headers → + `wait_event_names.txt`; `lwlocknames.h` → `lwlocknames.txt`. +- `configure` → `configure.ac`; `*.po` translations are handled separately; + generated Unicode tables come from their source scripts. + +## Portability is a hard gate +PostgreSQL runs on Linux, Windows (MSVC), macOS, the BSDs and Solaris, across +**x86_64, ARM64, RISC-V, PPC64, s390x**, both endiannesses and 32/64-bit. Any +change must be portable across all of them: +- No unaligned memory access; no dependence on `char` signedness, integer/pointer + width, endianness, or struct padding for on-disk/wire formats. +- Use `int16/int32/int64`, `Size`, and `INT64_FORMAT`/`UINT64_FORMAT` (never + `%ld` for `int64`). +- Atomics/barriers only via `port/atomics` (`pg_atomic_*`, `pg_read/write_barrier`). +- **Windows/MSVC:** any `extern` variable used from another module or an + extension needs `PGDLLIMPORT` in its header; no VLAs or compiler-specific + extensions beyond the tree's C99 baseline. + +## Backward compatibility — the strongest constraint +Do not break SQL behavior, the libpq wire protocol, the logical-replication +protocol, dump/restore, `pg_upgrade`, or exported/`PGDLLIMPORT` APIs without +extraordinary justification. **ABI** matters for back-branches: changing the +size/layout of an exported struct or the signature of an exported function +breaks installed extensions. + +## Mailing-list context & etiquette +Because each PR becomes a pgsql-hackers email read by a busy, expert, opinionated +audience, also flag what reliably wastes reviewer time or draws rejection: +- A patch that **does more than one thing** or bundles unrelated cleanup — split it. +- **Footguns**: easy-to-misuse APIs, silent data-loss/corruption hazards, unsafe + defaults — name them explicitly. +- **Performance claims without a reproducible benchmark.** +- No reference to the **design discussion / prior -hackers thread** (Message-Id) + for a non-trivial change. +- **Do not bikeshed:** keep style nits proportionate and clearly separated from + substantive correctness findings. + +## Minimalism — the "ponytail" discipline +The best code is the code you never wrote (YAGNI). Before accepting new code, +apply the ladder: (1) Does this need to exist at all? (2) Can existing +code/infrastructure already do it? (3) Is this the simplest thing that works? +Flag: speculative scaffolding and config for a path that isn't wired yet; dead +code and unused "flexibility" (fields, params, abstractions, options with no +caller); premature abstraction (a helper used exactly once); knobs/GUCs/flags +nobody asked for. Minimal, targeted changes that fit the existing patterns beat +clever or general-purpose ones. + +## Comment & identity accuracy +- Comments must describe what the code does **now**. Flag aspirational/ + future-tense comments for behavior that already shipped ("will be", "for now", + "not yet", "future", and stale "TODO/FIXME/XXX/HACK"); comments that drifted + from the code they sit above; and incomplete/trailing comments. Comments + explain **why**, not what. No commented-out code. +- **ASCII only** in source and diffs — no smart quotes, em-dashes, or ellipsis + characters. + +## Commit & versioning discipline +- Conventional-commit style, imperative subject, one logical change per commit, + each commit building on its own. +- Do **not** bump version numbers or generated version stamps (including + `catversion.h`) — that is the maintainer's job at commit/release time. + +Understand common list shorthand so your comments are precise and not +miscommunicated: WIP (work in progress), GUC (config variable), WAL, LSN, OID, +TOAST, FSM, TAM (table access method), RLS, DSM, 2PC, PITR, CIC (concurrent index +creation), SAOP, ABI/API, backpatch (apply to supported back-branches), HEAD +(master tip), catversion (catalog version), pgindent, buildfarm, cfbot, +`s/x/y/` (suggested text substitution), footgun, bikeshedding, POLA (principle of +least astonishment). diff --git a/.github/ocr/litellm.yaml b/.github/ocr/litellm.yaml new file mode 100644 index 0000000000000..e23cc4eee6fe2 --- /dev/null +++ b/.github/ocr/litellm.yaml @@ -0,0 +1,41 @@ +# LiteLLM proxy config — bridges Open Code Review (OpenAI protocol) to AWS Bedrock. +# +# This proxy is NOT a hosted service. The ocr-review.yml workflow installs it +# (`pip install 'litellm[proxy]'`) and runs it as a background process bound to +# 127.0.0.1:4000 for the duration of a single GitHub Actions job, then it exits. +# +# Auth to Bedrock: LiteLLM uses boto3's default credential chain, which reads +# the temporary AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN +# minted by the workflow's OIDC "Configure AWS credentials" step; region from +# AWS_REGION. + +model_list: + - model_name: ocr-bedrock + litellm_params: + # Set the repo variable OCR_BEDROCK_MODEL to an Opus inference-profile id + # your account has access to, e.g.: + # bedrock/converse/us.anthropic.claude-opus-4-8 + # The 'converse/' prefix uses Bedrock's Converse API, which is the most + # reliable path for Claude tool-use (what OCR relies on). + model: os.environ/OCR_BEDROCK_MODEL + aws_region_name: os.environ/AWS_REGION + + # "High effort" review. Claude Opus 4.8 on Bedrock uses *adaptive* thinking + # controlled by output_config.effort. Set it DIRECTLY here — NOT via + # reasoning_effort, which LiteLLM still maps to the legacy + # thinking.type.enabled that Opus 4.8 rejects. LiteLLM forwards + # output_config into additionalModelRequestFields for Anthropic models; if + # the build doesn't recognize the effort param it is dropped with a warning + # (no error) and the model reviews at its default effort. + # Valid: low|medium|high|max|xhigh (auto-clamped to the model ceiling). + output_config: + effort: xhigh + max_tokens: 32000 + +litellm_settings: + drop_params: true # silently drop params a model doesn't support + modify_params: true # auto-fix minor request incompatibilities + request_timeout: 600 + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY diff --git a/.github/ocr/pg-history.py b/.github/ocr/pg-history.py new file mode 100644 index 0000000000000..5794f8a920bd7 --- /dev/null +++ b/.github/ocr/pg-history.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +""" +pg-history: tie a PR's changes to PostgreSQL git + pgsql-hackers email history. + +OCR (the code reviewer) cannot call MCP servers, so this is a separate agent: +it runs a Bedrock (Claude Opus) tool-use loop wired to the Agora MCP server at +https://pg.ddx.io/mcp, lets the model search the mailing-list archives / commit +history / commitfest data, and emits a Markdown summary linking the changes to +the relevant threads (https://pg.ddx.io/m/pgsql-hackers/). + +Env: + PG_HISTORY_MCP_URL MCP endpoint (default https://pg.ddx.io/mcp) + PG_HISTORY_MODEL Bedrock model id (e.g. us.anthropic.claude-opus-4-8) + AWS_REGION region (creds come from the OIDC step's env) + BASE_REF, HEAD_SHA PR base ref and head sha (for the git diff context) + GH_PR_TITLE PR title (optional, adds context) + PG_HISTORY_OUT output markdown path (default /tmp/pg-history.md) +Writes the markdown to PG_HISTORY_OUT; exits 0 even on soft failures (writes a note). +""" +import json, os, subprocess, sys, urllib.request + +MCP_URL = os.environ.get("PG_HISTORY_MCP_URL", "https://pg.ddx.io/mcp") +MODEL = os.environ.get("PG_HISTORY_MODEL", "us.anthropic.claude-opus-4-8").replace("bedrock/converse/", "").replace("bedrock/", "") +REGION = os.environ.get("AWS_REGION", "us-east-1") +BASE_REF = os.environ.get("BASE_REF", "") +HEAD_SHA = os.environ.get("HEAD_SHA", "") +PR_TITLE = os.environ.get("GH_PR_TITLE", "") +OUT = os.environ.get("PG_HISTORY_OUT", "/tmp/pg-history.md") +UA = "pg-history/0.1 (+github-actions)" + +# Curated subset of the 108 Agora tools — the ones useful for connecting a +# change to its discussion/commit history. Intersected with what the server +# actually exposes, so unknown names are harmless. +TOOL_WHITELIST = { + "find_related_discussions", "find_similar_messages", "get_thread", + "discussion_links", "get_author_messages", "browse_by_date", + "blame_symbol", "check_upstream_status", "find_related", + "find_entries_for_thread", "find_entries_for_author", "get_commit", + "search", "hybrid_search", "get_callers", "get_callees", "find_pattern", +} +MAX_ROUNDS = 14 +TOOL_RESULT_CAP = 8000 # chars per tool result fed back to the model + + +def _mcp_post(body, sid=None): + headers = {"Content-Type": "application/json", + "Accept": "application/json, text/event-stream", "User-Agent": UA} + if sid: + headers["Mcp-Session-Id"] = sid + req = urllib.request.Request(MCP_URL, data=json.dumps(body).encode(), headers=headers, method="POST") + resp = urllib.request.urlopen(req, timeout=60) + sid_out = resp.headers.get("Mcp-Session-Id") + result = None + for line in resp.read().decode().splitlines(): + line = line.strip() + if line.startswith("data:"): + line = line[5:].strip() + if not line or line.startswith("event:"): + continue + try: + obj = json.loads(line) + except Exception: + continue + if isinstance(obj, dict) and ("result" in obj or "error" in obj): + result = obj + return result, sid_out + + +class MCP: + def __init__(self): + init, self.sid = _mcp_post({"jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "pg-history", "version": "0.1"}}}) + if not init or "result" not in init: + raise RuntimeError(f"MCP initialize failed: {init}") + try: + _mcp_post({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, self.sid) + except Exception: + pass + self._id = 1 + + def list_tools(self): + self._id += 1 + res, _ = _mcp_post({"jsonrpc": "2.0", "id": self._id, "method": "tools/list", "params": {}}, self.sid) + return (res or {}).get("result", {}).get("tools", []) + + def call(self, name, args): + self._id += 1 + res, _ = _mcp_post({"jsonrpc": "2.0", "id": self._id, "method": "tools/call", + "params": {"name": name, "arguments": args or {}}}, self.sid) + if not res: + return "(no response)" + if "error" in res: + return f"ERROR: {json.dumps(res['error'])[:500]}" + parts = [] + for c in res.get("result", {}).get("content", []): + if c.get("type") == "text": + parts.append(c["text"]) + return ("\n".join(parts) or "(empty)")[:TOOL_RESULT_CAP] + + +def git(*args): + try: + return subprocess.check_output(["git", *args], text=True, stderr=subprocess.DEVNULL).strip() + except Exception: + return "" + + +def pr_context(): + base = f"origin/{BASE_REF}" if BASE_REF else "" + rng = f"{base}..{HEAD_SHA}" if base and HEAD_SHA else HEAD_SHA + commits = git("log", "--no-merges", "--format=%h %s", f"{rng}") if rng else "" + stat = git("diff", "--stat", rng) if rng else "" + files = git("diff", "--name-only", rng) if rng else "" + return commits[:4000], stat[:3000], files[:2000] + + +SYSTEM = """You are a PostgreSQL community research assistant. Given a pull request's +commits and changed files, use the available tools (backed by the Agora index of +pgsql-hackers mail, commit history, and commitfest data) to connect the change to +its history. Your goal: + +- Find the mailing-list thread(s) and prior discussion behind this change. +- Identify related/superseded prior commits and any commitfest entry. +- Note relevant prior art, rejected approaches, or design rationale. + +Rules (voice & rigor): +- Be precise and blunt. No praise, no filler, no hedging, no disclaimers. Accuracy is + the only success metric — not the author's approval. Lead with the most important finding. +- NEVER hallucinate. Verify every Message-ID, thread subject, commit hash, author name, + and date against an actual tool result before citing it. If a search returns nothing, + say so plainly — do not guess or fabricate a plausible-looking link. +- Assess the change on its merits, independent of how the PR frames it. +- Tag any inferred (not tool-confirmed) linkage with an explicit confidence level: + high / moderate / low. +- Be decisive and efficient: a handful of targeted tool calls, not exhaustive search. +- Cite every mailing-list message as a Markdown link: [subject](https://pg.ddx.io/m/pgsql-hackers/MESSAGE_ID). +- If you find nothing relevant, say so in one line — do not pad. + +When done, output ONLY Markdown (no preamble) with these sections, omitting any that are empty: +## 🧵 Related discussion +## 🔗 Related commits / prior art +## 📋 Commitfest +## 🧭 Context for reviewers +Keep it tight (use bullets; link generously).""" + + +def to_toolspec(t): + schema = t.get("inputSchema") or {"type": "object", "properties": {}} + return {"toolSpec": {"name": t["name"], + "description": (t.get("description") or "")[:600], + "inputSchema": {"json": schema}}} + + +def main(): + commits, stat, files = pr_context() + if not commits and not files: + open(OUT, "w").write("") # nothing to do + print("No PR diff context; skipping.") + return + user = (f"PR title: {PR_TITLE}\n\n" if PR_TITLE else "") + \ + f"Commits:\n{commits or '(none)'}\n\nChanged files:\n{files or '(none)'}\n\nDiffstat:\n{stat or '(none)'}\n" + + try: + mcp = MCP() + tools = [to_toolspec(t) for t in mcp.list_tools() if t.get("name") in TOOL_WHITELIST] + except Exception as e: + open(OUT, "w").write(f"_pg-history: could not reach the Agora MCP server ({MCP_URL}): {e}_\n") + print(f"MCP unavailable: {e}") + return + if not tools: + open(OUT, "w").write("_pg-history: no usable MCP tools available._\n") + return + + import boto3 + from botocore.config import Config + + # botocore's default read timeout (60s) is too short for a multi-round + # (MAX_ROUNDS) tool-use loop against a large PR diff on a reasoning model; + # each converse() call alone can take several minutes. Bump it well past + # what a single round needs; connect_timeout stays short since a stuck + # TCP handshake is a different (and much cheaper to detect) failure mode. + brt = boto3.client("bedrock-runtime", region_name=REGION, + config=Config(read_timeout=900, connect_timeout=10)) + messages = [{"role": "user", "content": [{"text": user}]}] + final_text = "" + try: + for _ in range(MAX_ROUNDS): + resp = brt.converse( + modelId=MODEL, + system=[{"text": SYSTEM}], + messages=messages, + toolConfig={"tools": tools}, + inferenceConfig={"maxTokens": 4096}, + ) + out = resp["output"]["message"] + messages.append(out) + if resp.get("stopReason") == "tool_use": + results = [] + for blk in out["content"]: + tu = blk.get("toolUse") + if not tu: + continue + res_text = mcp.call(tu["name"], tu.get("input") or {}) + results.append({"toolResult": {"toolUseId": tu["toolUseId"], + "content": [{"text": res_text}]}}) + messages.append({"role": "user", "content": results}) + continue + final_text = "".join(b.get("text", "") for b in out["content"]).strip() + break + except Exception as e: + open(OUT, "w").write(f"_pg-history: Bedrock call failed: {e}_\n") + print(f"Bedrock error: {e}") + return + + if not final_text: + final_text = "_pg-history: no related history found._" + body = "## 📜 Change history & discussion (Agora / pg.ddx.io)\n\n" + final_text + \ + "\n\nGenerated by pg-history via the Agora MCP server (pg.ddx.io).\n" + open(OUT, "w").write(body) + print(body) + + +if __name__ == "__main__": + main() diff --git a/.github/ocr/rule.json b/.github/ocr/rule.json new file mode 100644 index 0000000000000..60e13e73dcbe0 --- /dev/null +++ b/.github/ocr/rule.json @@ -0,0 +1,65 @@ +{ + "_comment": "OCR per-file review rules for PostgreSQL core + extensions. Cross-cutting contribution standards & mailing-list etiquette live in .github/ocr/context.md, passed via --background-file. OCR uses FIRST-MATCH-WINS in declaration order, so rules are ordered most-specific first. merge_system_rule:true keeps OCR's built-in fine-tuned checks (thread-safety, injection, NPE) alongside these PostgreSQL-specific rules.", + "rules": [ + { + "path": "src/test/**", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL tests. Coverage is mandatory for any behavioral change and must include edge cases (NULL, empty, boundary/overflow) and ERROR paths, not just the happy path. A test that still passes with the feature reverted is worthless — confirm it actually exercises and would catch regressions in the new code. Regression (.sql/expected): deterministic, portable output — ORDER BY where row order matters, no timing/plan-dependent output except intentional EXPLAIN, no absolute paths, locale-independent (C collation or explicit COLLATE), DROP objects the test creates; expected/ output must stay stable across platforms and under the parallel schedule. Concurrency/locking belongs in isolation tests (src/test/isolation, .spec + permutations). End-to-end/crash/replication/CLI behavior belongs in TAP tests (t/*.pl with PostgreSQL::Test::Cluster/Utils) — no hardcoded ports/paths, no sleep as synchronization (use poll_query_until/wait_for), skip cleanly when prerequisites are missing, and clean up nodes." + }, + { + "path": "**/*.{c,h}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL backend/frontend C — review as pgsql-hackers committers do, in priority order.\n\n(1) CORRECTNESS (highest): Memory — every palloc lives in the right MemoryContext; error paths via ereport/elog(ERROR) must not leak memory/buffers/locks/fds (rely on MemoryContext/ResourceOwner reset or PG_TRY/PG_FINALLY); no use-after-free; delete temp contexts. Concurrency — consistent lock ordering (deadlock-free), correct lock levels, balanced LWLockAcquire/Release and START_/END_CRIT_SECTION, no TOCTOU, CHECK_FOR_INTERRUPTS in long loops, async-signal-safe signal handlers (volatile sig_atomic_t). WAL — any change to shared on-disk state must be WAL-logged AND correctly replayed (redo path), crash- and replica-consistent. NULL/edge/overflow handling.\n\n(2) BACKWARD COMPATIBILITY / ABI: don't break behavior, dump/restore, pg_upgrade, libpq wire protocol, logical-replication protocol, or exported/PGDLLIMPORT'd APIs (struct size/layout, function signatures) without extraordinary justification.\n\n(3) CATALOG / GENERATED: new/changed catalog data goes in pg_*.dat, NOT the generated *_d.h/.bki. New Node types: ANNOTATE the struct in its header so gen_node_support.pl regenerates copy/equal/out/read — do NOT hand-edit *funcs.c. New SQL-callable functions: add to pg_proc.dat with an OID from the 8000-9999 developer range (src/include/catalog/unused_oids; check duplicate_oids); committer renumbers at commit. DO NOT bump CATALOG_VERSION_NO in the patch — flag any catversion.h change as a mistake (committer's job).\n\n(4) PERFORMANCE: no regression on hot paths; avoid O(n^2) where better is feasible; minimize work under contended locks; avoid needless palloc churn and large struct copies in hot paths.\n\n(5) SECURITY: bounded string ops (snprintf/strlcpy/strlcat — never strcpy/strcat/sprintf); integer/size-overflow checks before allocation; never user input as a format string; privilege checks via pg_*_aclcheck; beware search_path and SECURITY DEFINER.\n\n(6) PORTABILITY (hard gate): no unaligned access; no dependence on char signedness, int/long/pointer width, endianness, or struct padding for on-disk/wire formats; use int16/int32/int64 + INT64_FORMAT/UINT64_FORMAT (never %ld for int64); align contended shared structs (pg_attribute_aligned/cache-line pad). Atomics/barriers only via port/atomics (pg_atomic_*, pg_read/write_barrier) — never raw intrinsics or volatile-as-barrier. WINDOWS/MSVC: extern vars used cross-module/extension need PGDLLIMPORT; no VLAs or features beyond the C99 baseline the tree targets; use pg_pread/pg_pwrite. Applies across x86_64/ARM64/RISC-V/PPC64/s390x, big/little endian, 32/64-bit.\n\n(7) CONVENTIONS: errmsg starts lowercase, no trailing period, no embedded newlines; errdetail/errhint are complete capitalized sentences; correct ERRCODE_*; wrap user-facing text in _(); errmsg_plural for counts. Assert() only for can't-happen invariants (never user-reachable). Naming: snake_case with subsystem prefix (heap_insert) or CamelCase for major subsystems (ExecInitNode); ALL_CAPS macros. Must pgindent cleanly (tabs, width 4). Comments explain WHY not WHAT; no #ifdef 0 blocks, no commented-out code, no #ifdef fencing your feature. Reuse existing helpers (DRY)." + }, + { + "path": "**/*.dat", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL catalog data (pg_proc.dat, pg_type.dat, etc.) — the SOURCE for generated headers. The generated *_d.h, postgres.bki, fmgroids.h, fmgrtab.c must NOT be hand-edited (they regenerate from these files). OIDs: use a value from the developer range 8000-9999 (src/include/catalog/unused_oids; verify with duplicate_oids); committer renumbers to a final contiguous block, so stay in-range and unique but don't over-optimize the exact number. Keep proc entries complete/consistent (prosrc, provolatile, proparallel, prorettype/proargtypes, matching description). DO NOT bump CATALOG_VERSION_NO / catversion.h — committer's job at push time; flag any such change. New catalog columns/views need documentation in doc/src/sgml/catalogs.sgml." + }, + { + "path": "**/*.{sql,pgsql}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL SQL. Valid PostgreSQL dialect (not MySQL/Oracle); correct types (bigint vs int, text vs varchar); sound transaction/isolation and CTE-materialization assumptions. SECURITY: flag SQL injection in dynamic SQL (require quote_identifier/quote_literal or format() with %I/%L), SECURITY DEFINER without a locked-down search_path, inappropriate RLS bypass. Prefer set-based over row-at-a-time/N+1. BACKWARD COMPATIBILITY (a top rejection reason): changing existing SQL behavior, the output of existing functions, or default GUCs needs extraordinary justification. New SQL-callable objects belong in pg_*.dat with OIDs from the 8000-9999 range, not in generated files. Minimal diff; add regression tests + docs." + }, + { + "path": "**/*.{pl,pm}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Perl (TAP tests and build/catalog tooling). Require 'use strict; use warnings;'. Must be perltidy-clean with the tree's src/tools/pgindent/perltidyrc and pass src/tools/perlcheck/pgperlcritic. Use the framework: PostgreSQL::Test::Cluster, PostgreSQL::Test::Utils, Test::More; no hardcoded ports/paths/PIDs; use safe_psql/poll_query_until, not sleep; skippable without optional prerequisites; clean up nodes. PORTABILITY: run on Windows (no fork-only constructs, use File::Spec, avoid unavailable signals) and the minimum supported Perl. Robustness: avoid two-arg open and string system()/qx with interpolated data (use list forms). Generator scripts (gen_node_support.pl, catalog Perl) must be deterministic and stay in sync with inputs; do not commit their generated output." + }, + { + "path": "**/*.py", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Python (build/test tooling, oauth/pytest tests, src/tools). Follow surrounding style; keep imports to the standard library unless the dependency is already required by the tree (no surprise third-party deps in build/test tooling). PORTABILITY: support the project's minimum Python 3 and run on Windows and the BSDs (use os.path/pathlib, avoid POSIX-only calls and shell=True with interpolated input). Deterministic, self-cleaning tests; no hardcoded ports/paths; skip cleanly without prerequisites. For the Perl->pytest porting effort, confirm behavior parity with the TAP test replaced (same assertions/coverage), not a superficial translation. Minimal diff; match the tree's ruff/black config if present." + }, + { + "path": "**/*.{rs,toml}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. Rust PostgreSQL extension (pgrx) or Rust support crate. Not core C, but it runs inside/alongside the backend, so backend safety applies. SAFETY: in code reachable from an SQL call, a Rust panic aborts the Postgres process — forbid unwrap()/expect()/panic!/unreachable!/todo! and index-panics on reachable paths; use Result and pgrx error reporting (error!/ereport!). Every `unsafe` block needs a comment justifying its invariant; scrutinize raw pointers and FFI across the pg_sys boundary. pgrx: honor #[pg_guard] on extern C fns (correct panic/longjmp handling); never hold Rust references across SPI or anything that can longjmp (skips Rust destructors -> leaks); respect MemoryContext lifetimes for palloc'd data; datum<->Rust conversions must handle NULL. Concurrency uses Postgres shmem/LWLocks (pgrx shmem API), not std::sync alone. Lints: must pass `cargo clippy --all-targets --all-features -- -D warnings` and `cargo fmt --check`; deny unwrap_used/expect_used/panic in libraries; thiserror (libs) / anyhow (bins). Justify every new dependency. Tests: #[pg_test] for in-backend behavior, #[test] for pure logic; cover error and NULL paths. Minimal, idiomatic diff." + }, + { + "path": "**/{configure.ac,*.m4,aclocal.m4}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Autoconf. Edit configure.ac / the m4 macros — do NOT hand-edit generated 'configure' or pg_config.h.in in the same patch (regeneration is the committer's step; a patch that also rewrites generated configure output is suspect). Feature/header/function probes must be portable and not assume a specific OS/compiler. Every configure knob must be mirrored on the Meson side (meson_options.txt/meson.build) and documented. Minimal diff." + }, + { + "path": "**/{meson.build,meson_options.txt}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Meson build. Valid syntax; correct subdir()/dependency()/declare_dependency and install paths; new source files must be added here. CRITICAL: PostgreSQL maintains BOTH Meson and Autoconf/Make — any new file, option, or feature check must be mirrored on the configure.ac/Makefile side so the two never drift (a file built by only one system is a common defect). New options need matching docs and sensible defaults. Minimal diff." + }, + { + "path": "**/{Makefile,GNUmakefile,*.mk}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Makefile (GNU Make). $(VAR) refs; correct .PHONY; accurate dependencies (no parallel -j races); $(MAKE) for recursion; VPATH/out-of-tree build support; no hardcoded paths (use standard PostgreSQL makefile vars and $(top_builddir)); clean/distclean/maintainer-clean must remove new artifacts; extensions use PGXS. Must stay in sync with meson.build. Minimal diff." + }, + { + "path": "doc/**/*.sgml", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL documentation (DocBook SGML). Technically accurate/complete (parameters, limitations, version/compat notes); correct tag usage/nesting (, , , , , /); working cross-references; spell it 'PostgreSQL' in prose; SQL keywords uppercase in examples. Coverage: a new GUC -> config.sgml (and postgresql.conf.sample); new/changed catalogs or views -> catalogs.sgml; new SQL syntax -> the matching ref/*.sgml; new functions -> func.sgml. Do NOT edit release-notes (release-*.sgml) — written by the release team/committers; flag such edits. New user-facing behavior in this PR should ship with matching docs." + }, + { + "path": "**/*.md", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. Markdown docs. Clear heading hierarchy; fenced code blocks with language hints; accurate instructions/prerequisites; consistent PostgreSQL terminology; no broken relative links or stale claims. Minimal diff." + } + ] +} diff --git a/.github/workflows/fork-ocr-model-check.yml b/.github/workflows/fork-ocr-model-check.yml new file mode 100644 index 0000000000000..10d250528cf7c --- /dev/null +++ b/.github/workflows/fork-ocr-model-check.yml @@ -0,0 +1,89 @@ +# Checks AWS Bedrock weekly for a newer Claude Opus inference profile than the +# one OCR currently uses (vars.OCR_BEDROCK_MODEL) and, if found, opens/updates a +# single GitHub issue telling the maintainer to bump the variable. It does NOT +# change the model automatically: GITHUB_TOKEN cannot write Actions *variables* +# (that needs a PAT with admin), so this is a notify-only mechanism by design. +name: OCR model self-check + +on: + schedule: + - cron: '0 12 * * 1' # Mondays 12:00 UTC + workflow_dispatch: + +permissions: + id-token: write + contents: read + issues: write + +jobs: + check-model: + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: ocr-model-check-${{ github.run_id }} + + - name: Find newest Opus vs configured + id: check + env: + CURRENT: ${{ vars.OCR_BEDROCK_MODEL }} + AWS_REGION: ${{ vars.AWS_REGION }} + run: | + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import os, re, subprocess, json + region = os.environ.get("AWS_REGION", "us-east-1") + current = os.environ.get("CURRENT", "") + out = subprocess.run( + ["aws", "bedrock", "list-inference-profiles", "--region", region, + "--query", "inferenceProfileSummaries[].inferenceProfileId", "--output", "json"], + capture_output=True, text=True) + ids = json.loads(out.stdout or "[]") + # Parse claude-opus-- from any profile id (prefix us./global. ok). + def ver(s): + m = re.search(r"claude-opus-(\d+)-(\d+)", s) + return (int(m.group(1)), int(m.group(2))) if m else None + opus = [(ver(i), i) for i in ids if ver(i) and i.startswith(("us.", "global."))] + if not opus: + print("newer=false"); raise SystemExit(0) + best_ver, best_id = max(opus, key=lambda x: x[0]) + cur = ver(current) + newer = (cur is None) or (best_ver > cur) + print(f"newer={'true' if newer else 'false'}") + print(f"best_id={best_id}") + print(f"best_ver={best_ver[0]}.{best_ver[1]}") + print(f"cur_ver={'unknown' if cur is None else f'{cur[0]}.{cur[1]}'}") + PY + + - name: Open/update issue if a newer model exists + if: steps.check.outputs.newer == 'true' + uses: actions/github-script@v9 + with: + script: | + const best = '${{ steps.check.outputs.best_id }}'; + const bestVer = '${{ steps.check.outputs.best_ver }}'; + const curVer = '${{ steps.check.outputs.cur_ver }}'; + const marker = ''; + const title = `OCR: newer Claude Opus available (${bestVer} > ${curVer})`; + const body = `${marker}\n` + + `A newer Claude Opus inference profile is available on Bedrock.\n\n` + + `- **Configured** (\`vars.OCR_BEDROCK_MODEL\`): Opus ${curVer}\n` + + `- **Newest on Bedrock**: \`${best}\` (Opus ${bestVer})\n\n` + + `To upgrade, set the repo variable:\n\n` + + '```\n' + + `gh variable set OCR_BEDROCK_MODEL -R ${context.repo.owner}/${context.repo.repo} \\\n` + + ` -b "bedrock/converse/${best}"\n` + + '```\n\n' + + `Also confirm the \`ocr-bedrock-ci\` IAM inline policy allows invoking the new model ` + + `(the resource is scoped to \`anthropic.claude-opus-*\`), then re-run OCR.\n\n` + + `_Automated by \`.github/workflows/ocr-model-check.yml\`; this issue is upserted._`; + const q = `repo:${context.repo.owner}/${context.repo.repo} in:body "${marker}" state:open`; + const found = await github.rest.search.issuesAndPullRequests({ q, per_page: 1 }); + if (found.data.total_count > 0) { + const n = found.data.items[0].number; + await github.rest.issues.update({ owner: context.repo.owner, repo: context.repo.repo, issue_number: n, title, body }); + } else { + await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title, body }); + } diff --git a/.github/workflows/fork-ocr-review.yml b/.github/workflows/fork-ocr-review.yml new file mode 100644 index 0000000000000..0828af429b57c --- /dev/null +++ b/.github/workflows/fork-ocr-review.yml @@ -0,0 +1,427 @@ +# Open Code Review (OCR) — AI PR review backed by AWS Bedrock via a LiteLLM proxy. +# +# Flow: +# PR opened/updated (incl. DRAFTS) ─┐ +# /open-code-review PR comment ─┼─► start LiteLLM (127.0.0.1:4000 → Bedrock) +# manual workflow_dispatch ─┘ └► ocr review --format json +# └► post inline PR review comments +# +# Required (repo settings — all repo *variables*, no secrets; auth is via GitHub OIDC): +# vars.AWS_ROLE_ARN - IAM role to assume via OIDC (granting bedrock:InvokeModel*) +# vars.AWS_REGION - e.g. us-east-1 +# vars.OCR_BEDROCK_MODEL - LiteLLM model string for the Opus inference profile, e.g. +# bedrock/converse/us.anthropic.claude-opus-4-8 +# +# No static AWS keys are stored. GITHUB_TOKEN (auto) posts the review comments. + +name: OCR AI Review + +on: + pull_request: + # Note: no draft filter — drafts are reviewed too. + types: [opened, synchronize, reopened, ready_for_review] + issue_comment: + types: [created] + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to review' + required: true + type: number + +# One review per PR; cancel superseded runs to save Bedrock spend. +concurrency: + group: ocr-review-${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} + cancel-in-progress: true + +permissions: + id-token: write # required to mint the GitHub OIDC token for AWS role assumption + contents: read + pull-requests: write + +jobs: + ocr-review: + runs-on: ubuntu-latest + # PR events always; comment events only when the comment is on a PR and + # starts with the trigger keyword; manual dispatch always. + if: | + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/open-code-review') || + startsWith(github.event.comment.body, '@open-code-review'))) + + env: + # LiteLLM listens on localhost only; this key never leaves the runner. + LITELLM_MASTER_KEY: sk-ocr-ci-local + OCR_BEDROCK_MODEL: ${{ vars.OCR_BEDROCK_MODEL }} + # Region is a static var (safe at job level). AWS credentials are NOT set + # here — they're minted by the OIDC "Configure AWS credentials" step below + # and exported to the environment for the LiteLLM/boto3 Bedrock calls. + AWS_REGION: ${{ vars.AWS_REGION }} + + steps: + - name: Resolve PR context + id: pr + uses: actions/github-script@v9 + with: + script: | + let prNumber; + if (context.eventName === 'pull_request') { + prNumber = context.payload.pull_request.number; + } else if (context.eventName === 'issue_comment') { + prNumber = context.issue.number; + } else { + prNumber = parseInt('${{ github.event.inputs.pr_number }}', 10); + } + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + const { data: repo } = await github.rest.repos.get({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + core.setOutput('number', String(prNumber)); + core.setOutput('base_ref', pr.base.ref); + core.setOutput('head_ref', pr.head.ref); + core.setOutput('head_sha', pr.head.sha); + core.setOutput('default_branch', repo.default_branch); + core.setOutput('cross_repo', String(pr.head.repo.full_name !== pr.base.repo.full_name)); + + # NOTE: do NOT checkout the PR head. OCR reads the diff and file contents + # straight from git refs (git diff , git show :path, + # git grep ), so the working tree is irrelevant — but our OCR config + # lives on the default branch, not on the PR branch. We check out the repo + # (default ref), fetch the base/head objects, and materialize the config + # from origin/. + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Prepare git refs and OCR config + env: + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_REF: ${{ steps.pr.outputs.head_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + DEFAULT_BRANCH: ${{ steps.pr.outputs.default_branch }} + run: | + git fetch --no-tags origin "+refs/heads/${DEFAULT_BRANCH}:refs/remotes/origin/${DEFAULT_BRANCH}" || true + git fetch --no-tags origin "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" || true + git fetch --no-tags origin "+refs/heads/${HEAD_REF}:refs/remotes/origin/${HEAD_REF}" || true + git fetch --no-tags origin "${HEAD_SHA}" || true + + # OCR config lives on the default branch; materialize it independently + # of whatever ref is checked out. + mkdir -p "$RUNNER_TEMP/ocr" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/litellm.yaml" > "$RUNNER_TEMP/ocr/litellm.yaml" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/rule.json" > "$RUNNER_TEMP/ocr/rule.json" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/context.md" > "$RUNNER_TEMP/ocr/context.md" + echo "Config materialized:"; ls -l "$RUNNER_TEMP/ocr" + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: '20' + + - name: Install LiteLLM proxy + Open Code Review + run: | + python -m pip install --upgrade pip + # Pin LiteLLM to a main commit that supports Claude Opus 4.8 adaptive + # thinking (maps reasoning_effort -> output_config.effort, incl. xhigh). + # Not in any tagged release yet (PyPI latest 1.87.1 lacks the Opus + # normalizer). Bump this SHA once a release ships the feature. + pip install "litellm[proxy] @ git+https://github.com/BerriAI/litellm.git@5be0797d24a2f26eb2123e13788f90055a59d91d" + npm install -g @alibaba-group/open-code-review + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: ocr-review-${{ github.run_id }} + + - name: Start LiteLLM proxy (Bedrock bridge) + run: | + if [ -z "$OCR_BEDROCK_MODEL" ]; then + echo "::error::vars.OCR_BEDROCK_MODEL is not set (e.g. bedrock/converse/us.anthropic.claude-opus-4-1-20250805-v1:0)" + exit 1 + fi + nohup litellm --config "$RUNNER_TEMP/ocr/litellm.yaml" --host 127.0.0.1 --port 4000 \ + > /tmp/litellm.log 2>&1 & + echo "Waiting for LiteLLM to become ready..." + for i in $(seq 1 60); do + if curl -sf http://127.0.0.1:4000/health/readiness >/dev/null; then + echo "LiteLLM ready."; exit 0 + fi + sleep 2 + done + echo "::error::LiteLLM did not become ready in time"; cat /tmp/litellm.log; exit 1 + + - name: Configure OCR + run: | + ocr config set llm.url http://127.0.0.1:4000/v1/chat/completions + ocr config set llm.auth_token "$LITELLM_MASTER_KEY" + ocr config set llm.model ocr-bedrock + ocr config set llm.use_anthropic false + ocr config set language English + + - name: Run OCR review + run: | + ocr review \ + --from "origin/${{ steps.pr.outputs.base_ref }}" \ + --to "${{ steps.pr.outputs.head_sha }}" \ + --rule "$RUNNER_TEMP/ocr/rule.json" \ + --background-file "$RUNNER_TEMP/ocr/context.md" \ + --concurrency 3 \ + --timeout 20 \ + --format json \ + > /tmp/ocr-result.json 2>/tmp/ocr-stderr.log || true + echo "----- OCR stdout -----"; cat /tmp/ocr-result.json || true + echo "----- OCR stderr -----"; cat /tmp/ocr-stderr.log || true + echo "----- LiteLLM log (tail) -----"; tail -n 50 /tmp/litellm.log || true + + - name: Post review to PR + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const prNumber = parseInt('${{ steps.pr.outputs.number }}', 10); + const commitSha = '${{ steps.pr.outputs.head_sha }}'; + + // Opus at high effort can emit dozens of findings. Posting them all + // one-by-one trips GitHub's SECONDARY rate limit (403 "content + // creation"), which is what made every run fail after the review + // was already generated. We (a) cap inline comments and overflow + // the rest into the summary, (b) prefer a single bulk createReview, + // and (c) throttle + back off with Retry-After on any fallback. + const MAX_INLINE = 25; + const sleep = (ms) => new Promise(r => setTimeout(r, ms)); + + async function withRetry(fn, label) { + for (let attempt = 1; attempt <= 5; attempt++) { + try { return await fn(); } + catch (e) { + const status = e.status || (e.response && e.response.status); + const h = (e.response && e.response.headers) || {}; + const isRate = status === 403 || status === 429; + if (!isRate || attempt === 5) throw e; + let waitMs = 0; + if (h['retry-after']) waitMs = parseInt(h['retry-after'], 10) * 1000; + else if (h['x-ratelimit-reset']) waitMs = parseInt(h['x-ratelimit-reset'], 10) * 1000 - Date.now(); + if (!waitMs || Number.isNaN(waitMs) || waitMs < 0) waitMs = 1000 * Math.pow(2, attempt); + waitMs = Math.min(waitMs, 60000) + 500; + core.warning(`${label}: rate-limited (status ${status}); waiting ${Math.round(waitMs / 1000)}s (attempt ${attempt}/5)`); + await sleep(waitMs); + } + } + } + + let result; + try { + result = JSON.parse(fs.readFileSync('/tmp/ocr-result.json', 'utf8')); + } catch (e) { + const stderr = (() => { try { return fs.readFileSync('/tmp/ocr-stderr.log', 'utf8').trim(); } catch { return ''; } })(); + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `⚠️ **OCR** could not produce a review.\n\n\`\`\`\n${(stderr || e.message).slice(0, 8000)}\n\`\`\``, + }), 'error-comment'); + return; + } + + const comments = result.comments || []; + const warnings = result.warnings || []; + + const formatComment = (c) => { + let body = c.content || ''; + if (c.suggestion_code && c.existing_code) { + body += '\n\n```suggestion\n' + c.suggestion_code + (c.suggestion_code.endsWith('\n') ? '' : '\n') + '```'; + } + return body; + }; + const formatMarkdown = (c) => { + let md = `### 📄 \`${c.path}\``; + if (c.start_line && c.end_line) md += ` (L${c.start_line}-L${c.end_line})`; + md += '\n\n' + (c.content || ''); + if (c.suggestion_code && c.existing_code) { + md += '\n\n
💡 Suggested change\n\n'; + md += '**Before:**\n```\n' + c.existing_code + '\n```\n\n**After:**\n```\n' + c.suggestion_code + '\n```\n\n
'; + } + return md; + }; + + if (comments.length === 0) { + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `✅ **OCR**: ${result.message || 'No issues found.'}`, + }), 'no-issues-comment'); + return; + } + + const inlineAll = []; + const noLine = []; + for (const c of comments) { + const body = formatComment(c); + const hasLine = (c.start_line >= 1) || (c.end_line >= 1); + if (!hasLine) { noLine.push(c); continue; } + const rc = { path: c.path, body, side: 'RIGHT' }; + if (c.start_line >= 1 && c.end_line >= 1 && c.start_line !== c.end_line) { + rc.start_line = c.start_line; rc.line = c.end_line; rc.start_side = 'RIGHT'; + } else { + rc.line = c.end_line >= 1 ? c.end_line : c.start_line; + } + inlineAll.push({ rc, c }); + } + + const inline = inlineAll.slice(0, MAX_INLINE).map(x => x.rc); + const overflow = inlineAll.slice(MAX_INLINE).map(x => x.c); + + let summary = `🔍 **OCR** found **${comments.length}** issue(s).`; + summary += `\n- ${inline.length} inline, ${noLine.length + overflow.length} in summary`; + if (overflow.length) summary += ` (inline capped at ${MAX_INLINE})`; + if (warnings.length) summary += `\n- ⚠️ ${warnings.length} warning(s) during review`; + for (const c of noLine.concat(overflow)) summary += '\n\n---\n\n' + formatMarkdown(c); + + // Preferred path: ONE createReview carrying every inline comment. + try { + await withRetry(() => github.rest.pulls.createReview({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, body: summary, event: 'COMMENT', comments: inline, + }), 'bulk-review'); + return; + } catch (e) { + core.warning(`bulk createReview failed (${e.status || '?'}: ${e.message}); falling back to throttled per-comment posting`); + } + + // Fallback: an invalid inline position (line not in the diff -> 422) + // rejects the whole bulk review. Post the summary, then each comment + // individually with a delay + backoff, skipping ones GitHub rejects. + let ok = 0; const failed = []; + try { + await withRetry(() => github.rest.pulls.createReview({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, body: summary, event: 'COMMENT', + }), 'summary-review'); + } catch (err) { failed.push(`summary: ${err.message}`); } + + for (const rc of inline) { + try { + await withRetry(() => github.rest.pulls.createReviewComment({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, path: rc.path, body: rc.body, + ...(rc.start_line ? { start_line: rc.start_line, start_side: rc.start_side } : {}), + line: rc.line, side: rc.side, + }), `comment ${rc.path}:${rc.line}`); + ok++; + } catch (inner) { + failed.push(`\`${rc.path}\` L${rc.line}: ${inner.message}`); + } + await sleep(1200); // stay under the secondary content-creation limit + } + + if (failed.length) { + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `📊 OCR posted ${ok}/${inline.length} inline comment(s).\n\n
${failed.length} could not be posted\n\n${failed.join('\n')}\n
`, + }), 'summary-failures'); + } + + # Companion job: OCR can't call MCP, so this separate agent ties the PR's + # changes to PostgreSQL git + pgsql-hackers history via the Agora MCP server + # (pg.ddx.io) and posts a single, upserted "history & discussion" comment. + pg-history: + runs-on: ubuntu-latest + if: | + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/open-code-review') || + startsWith(github.event.comment.body, '@open-code-review') || + startsWith(github.event.comment.body, '/pg-history'))) + steps: + - name: Resolve PR context + id: pr + uses: actions/github-script@v9 + with: + script: | + let prNumber; + if (context.eventName === 'pull_request') prNumber = context.payload.pull_request.number; + else if (context.eventName === 'issue_comment') prNumber = context.issue.number; + else prNumber = parseInt('${{ github.event.inputs.pr_number }}', 10); + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber }); + core.setOutput('number', String(prNumber)); + core.setOutput('base_ref', pr.base.ref); + core.setOutput('head_sha', pr.head.sha); + core.setOutput('title', pr.title || ''); + + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Make base/head refs available + env: + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + run: | + git fetch --no-tags origin "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" || true + git fetch --no-tags origin "${HEAD_SHA}" || true + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: pg-history-${{ github.run_id }} + + - name: Install deps + run: pip install boto3 + + - name: Run pg-history (Agora MCP) + env: + PG_HISTORY_MODEL: ${{ vars.OCR_BEDROCK_MODEL }} + AWS_REGION: ${{ vars.AWS_REGION }} + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + GH_PR_TITLE: ${{ steps.pr.outputs.title }} + PG_HISTORY_OUT: ${{ runner.temp }}/pg-history.md + run: | + python .github/ocr/pg-history.py || true + echo "----- output -----"; cat "${{ runner.temp }}/pg-history.md" 2>/dev/null || echo "(no output)" + + - name: Upsert PR comment + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + const path = process.env.RUNNER_TEMP + '/pg-history.md'; + let body = ''; + try { body = fs.readFileSync(path, 'utf8').trim(); } catch (e) {} + if (!body) { console.log('pg-history: empty output, nothing to post'); return; } + const prNumber = parseInt('${{ steps.pr.outputs.number }}', 10); + const marker = ''; + body = marker + '\n' + body; + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, per_page: 100 }); + const mine = comments.find(c => c.user.type === 'Bot' && c.body && c.body.includes(marker)); + if (mine) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: mine.id, body }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, body }); + } diff --git a/.github/workflows/fork-sync-upstream.yml b/.github/workflows/fork-sync-upstream.yml new file mode 100644 index 0000000000000..e950821239968 --- /dev/null +++ b/.github/workflows/fork-sync-upstream.yml @@ -0,0 +1,47 @@ +# Keep this fork's master rebased on postgres/postgres master, carrying only +# the fork-owned CI files (.github/workflows/fork-*.yml and .github/ocr/). +# +# Requires the SYNC_TOKEN secret: a PAT with repo+workflow scope. The default +# GITHUB_TOKEN is not allowed to push commits that touch .github/workflows/, +# and every commit we carry does. + +name: Sync from upstream + +on: + schedule: + - cron: '17 */3 * * *' + workflow_dispatch: + +concurrency: + group: fork-sync-upstream + +permissions: + contents: read + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + token: ${{ secrets.SYNC_TOKEN }} + + - name: Rebase master onto upstream + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git remote add upstream https://github.com/postgres/postgres.git + git fetch --no-tags upstream master + + # Master carries fork CI only. Anything else means work landed here + # that belongs on a branch; force-rebasing it on a timer would be wrong. + if git diff --name-only upstream/master...HEAD | + grep -vE '^(\.github/workflows/fork-|\.github/ocr/)'; then + echo "::error::master carries non-CI changes (listed above); move them to a branch" + exit 1 + fi + + git rebase upstream/master + git push --force-with-lease From 70cc154f622b812c470388ae0a6694e16f388dde Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Fri, 11 Sep 2026 10:48:09 -0400 Subject: [PATCH 2/5] ci: move fork CI logic to gburd/ci-workflows The OCR review workflows and their config were the last bulk this fork carried on top of upstream: ~970 lines rebased onto postgres/postgres every few hours, editable only by amending a commit that gets force- pushed. They are now reusable (workflow_call) workflows in the sidecar repo gburd/ci-workflows, and master carries two caller stubs instead. Carry after this: 3 files, ~110 lines, one commit. The sidecar is also where the value is: ocr/rule.json and ocr/context.md (the PostgreSQL committer-grade review corpus) become normal reviewable files with normal PRs and history, rather than a blob inside a commit that is rewritten on every sync. Mechanics worth knowing: - The review jobs check ci-workflows out at github.job_workflow_sha, so the rules always come from the same commit as the logic using them, and pinning @v1 (or a sha) in the stub pins both. This replaces the old "git show origin/:.github/ocr/..." dance, which existed only because the config had to be read off a branch other than the PR's. - AWS settings are passed as explicit inputs rather than read from vars inside the reusable workflow. vars does resolve across the boundary, but the contract stays visible at the call site. - A reusable workflow does not change the OIDC subject: it stays repo:gburd/postgres:*, so the ocr-bedrock-ci trust policy is unchanged. - ci-workflows must stay public; callers read it with GITHUB_TOKEN. The sync guard narrows to .github/workflows/fork-* accordingly, and sync-upstream keeps needing SYNC_TOKEN: the stubs are still workflow files, which GITHUB_TOKEN may not push. --- .github/ocr/context.md | 126 ------- .github/ocr/litellm.yaml | 41 --- .github/ocr/pg-history.py | 225 ----------- .github/ocr/rule.json | 65 ---- .github/workflows/fork-ocr-model-check.yml | 91 +---- .github/workflows/fork-ocr-review.yml | 410 +-------------------- .github/workflows/fork-sync-upstream.yml | 5 +- 7 files changed, 26 insertions(+), 937 deletions(-) delete mode 100644 .github/ocr/context.md delete mode 100644 .github/ocr/litellm.yaml delete mode 100644 .github/ocr/pg-history.py delete mode 100644 .github/ocr/rule.json diff --git a/.github/ocr/context.md b/.github/ocr/context.md deleted file mode 100644 index c4a83b85b124e..0000000000000 --- a/.github/ocr/context.md +++ /dev/null @@ -1,126 +0,0 @@ -# OCR review context — PostgreSQL contribution standards - -You are reviewing a change to a **PostgreSQL** fork. Every PR here is destined to -become a patch posted to the **pgsql-hackers** mailing list and tracked in a -**commitfest**. Review with the combined rigor, taste, and attention to detail of -the PostgreSQL committers. This context applies to the *whole* change, on top of -the per-file rules. - -## Review discipline -- Be precise and blunt; lead with the most serious problem. No praise, no - validation of the author, no disclaimers — accuracy is the only metric. -- Verify every claim against the actual diff. Confirm names, signatures, line - numbers, and APIs before asserting. Never invent behavior or cite code not in - the change. If unsure, say so, and tag each finding **high / moderate / low** - confidence. -- Judge the change on its merits regardless of how the PR frames it. A draft PR - is WIP: weight design/approach feedback over style nits. - -## Patch hygiene (top rejection reasons on -hackers) -1. **Minimal diff.** The fastest way to get a patch rejected is unrelated - changes: reformatting untouched lines, rewording unrelated comments, touching - code not required by the change. Flag any hunk not needed for the stated - purpose. After the patch, the code should read as if it had always been - written that way. -2. **Atomic, bisectable commits.** Each commit must build and pass tests on its - own — a broken intermediate commit breaks `git bisect`, revert, and - cherry-pick. Flag a commit that only compiles once a later commit lands. - Prefer one focused patch, or a clearly-ordered series of - independently-committable pieces. -3. **Tests + docs are mandatory.** A user-visible change without regression/TAP - tests **and** documentation is WIP, not commit-ready. New behavior needs - tests that cover edge and error paths, not just the happy path. -4. **DRY / reuse.** Prefer existing infrastructure (`List` in `pg_list.h`, - `StringInfo`, `dynahash`/`simplehash`, `palloc`/`MemoryContext`, `foreach`) - over reinventing it. Flag copy-paste and speculative abstraction alike — the - community wants minimal, targeted changes that fit the subsystem's existing - patterns. -5. **Whitespace.** No trailing whitespace; tabs (width 4) for C indentation; - `git diff --check` must be clean. Whitespace-only churn on untouched lines is - a defect. - -## Committer-owned files — do NOT touch in a patch (flag if present) -These are the committer's job at push time; including them causes needless -merge conflicts and is a mistake: -- **`src/include/catalog/catversion.h`** — the `CATALOG_VERSION_NO` bump is done - by the **committer** when pushing. A catversion bump in the PR is **wrong** — - flag it. (This is the single most common author mistake in catalog patches.) -- **Release notes** (`doc/src/sgml/release-*.sgml`) and version strings - (`configure.ac` `AC_INIT` version, `meson.build` `version`, `PG_VERSION`). - -## Generated files — never hand-edit; edit the source -Flag direct edits to generated output; point the author at the source instead: -- Catalog headers `src/include/catalog/*_d.h`, `postgres.bki`, `schemapg.h`, - `system_constraints.sql` → edit the `pg_*.dat` files. -- `src/backend/nodes/{copy,equal,out,read}funcs.c` and other - `gen_node_support.pl` output → annotate the `Node` struct in its header. -- `fmgroids.h`, `fmgrprotos.h`, `fmgrtab.c` → edit `pg_proc.dat`. -- `utils/errcodes.h` → `errcodes.txt`; wait-event headers → - `wait_event_names.txt`; `lwlocknames.h` → `lwlocknames.txt`. -- `configure` → `configure.ac`; `*.po` translations are handled separately; - generated Unicode tables come from their source scripts. - -## Portability is a hard gate -PostgreSQL runs on Linux, Windows (MSVC), macOS, the BSDs and Solaris, across -**x86_64, ARM64, RISC-V, PPC64, s390x**, both endiannesses and 32/64-bit. Any -change must be portable across all of them: -- No unaligned memory access; no dependence on `char` signedness, integer/pointer - width, endianness, or struct padding for on-disk/wire formats. -- Use `int16/int32/int64`, `Size`, and `INT64_FORMAT`/`UINT64_FORMAT` (never - `%ld` for `int64`). -- Atomics/barriers only via `port/atomics` (`pg_atomic_*`, `pg_read/write_barrier`). -- **Windows/MSVC:** any `extern` variable used from another module or an - extension needs `PGDLLIMPORT` in its header; no VLAs or compiler-specific - extensions beyond the tree's C99 baseline. - -## Backward compatibility — the strongest constraint -Do not break SQL behavior, the libpq wire protocol, the logical-replication -protocol, dump/restore, `pg_upgrade`, or exported/`PGDLLIMPORT` APIs without -extraordinary justification. **ABI** matters for back-branches: changing the -size/layout of an exported struct or the signature of an exported function -breaks installed extensions. - -## Mailing-list context & etiquette -Because each PR becomes a pgsql-hackers email read by a busy, expert, opinionated -audience, also flag what reliably wastes reviewer time or draws rejection: -- A patch that **does more than one thing** or bundles unrelated cleanup — split it. -- **Footguns**: easy-to-misuse APIs, silent data-loss/corruption hazards, unsafe - defaults — name them explicitly. -- **Performance claims without a reproducible benchmark.** -- No reference to the **design discussion / prior -hackers thread** (Message-Id) - for a non-trivial change. -- **Do not bikeshed:** keep style nits proportionate and clearly separated from - substantive correctness findings. - -## Minimalism — the "ponytail" discipline -The best code is the code you never wrote (YAGNI). Before accepting new code, -apply the ladder: (1) Does this need to exist at all? (2) Can existing -code/infrastructure already do it? (3) Is this the simplest thing that works? -Flag: speculative scaffolding and config for a path that isn't wired yet; dead -code and unused "flexibility" (fields, params, abstractions, options with no -caller); premature abstraction (a helper used exactly once); knobs/GUCs/flags -nobody asked for. Minimal, targeted changes that fit the existing patterns beat -clever or general-purpose ones. - -## Comment & identity accuracy -- Comments must describe what the code does **now**. Flag aspirational/ - future-tense comments for behavior that already shipped ("will be", "for now", - "not yet", "future", and stale "TODO/FIXME/XXX/HACK"); comments that drifted - from the code they sit above; and incomplete/trailing comments. Comments - explain **why**, not what. No commented-out code. -- **ASCII only** in source and diffs — no smart quotes, em-dashes, or ellipsis - characters. - -## Commit & versioning discipline -- Conventional-commit style, imperative subject, one logical change per commit, - each commit building on its own. -- Do **not** bump version numbers or generated version stamps (including - `catversion.h`) — that is the maintainer's job at commit/release time. - -Understand common list shorthand so your comments are precise and not -miscommunicated: WIP (work in progress), GUC (config variable), WAL, LSN, OID, -TOAST, FSM, TAM (table access method), RLS, DSM, 2PC, PITR, CIC (concurrent index -creation), SAOP, ABI/API, backpatch (apply to supported back-branches), HEAD -(master tip), catversion (catalog version), pgindent, buildfarm, cfbot, -`s/x/y/` (suggested text substitution), footgun, bikeshedding, POLA (principle of -least astonishment). diff --git a/.github/ocr/litellm.yaml b/.github/ocr/litellm.yaml deleted file mode 100644 index e23cc4eee6fe2..0000000000000 --- a/.github/ocr/litellm.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# LiteLLM proxy config — bridges Open Code Review (OpenAI protocol) to AWS Bedrock. -# -# This proxy is NOT a hosted service. The ocr-review.yml workflow installs it -# (`pip install 'litellm[proxy]'`) and runs it as a background process bound to -# 127.0.0.1:4000 for the duration of a single GitHub Actions job, then it exits. -# -# Auth to Bedrock: LiteLLM uses boto3's default credential chain, which reads -# the temporary AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN -# minted by the workflow's OIDC "Configure AWS credentials" step; region from -# AWS_REGION. - -model_list: - - model_name: ocr-bedrock - litellm_params: - # Set the repo variable OCR_BEDROCK_MODEL to an Opus inference-profile id - # your account has access to, e.g.: - # bedrock/converse/us.anthropic.claude-opus-4-8 - # The 'converse/' prefix uses Bedrock's Converse API, which is the most - # reliable path for Claude tool-use (what OCR relies on). - model: os.environ/OCR_BEDROCK_MODEL - aws_region_name: os.environ/AWS_REGION - - # "High effort" review. Claude Opus 4.8 on Bedrock uses *adaptive* thinking - # controlled by output_config.effort. Set it DIRECTLY here — NOT via - # reasoning_effort, which LiteLLM still maps to the legacy - # thinking.type.enabled that Opus 4.8 rejects. LiteLLM forwards - # output_config into additionalModelRequestFields for Anthropic models; if - # the build doesn't recognize the effort param it is dropped with a warning - # (no error) and the model reviews at its default effort. - # Valid: low|medium|high|max|xhigh (auto-clamped to the model ceiling). - output_config: - effort: xhigh - max_tokens: 32000 - -litellm_settings: - drop_params: true # silently drop params a model doesn't support - modify_params: true # auto-fix minor request incompatibilities - request_timeout: 600 - -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY diff --git a/.github/ocr/pg-history.py b/.github/ocr/pg-history.py deleted file mode 100644 index 5794f8a920bd7..0000000000000 --- a/.github/ocr/pg-history.py +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env python3 -""" -pg-history: tie a PR's changes to PostgreSQL git + pgsql-hackers email history. - -OCR (the code reviewer) cannot call MCP servers, so this is a separate agent: -it runs a Bedrock (Claude Opus) tool-use loop wired to the Agora MCP server at -https://pg.ddx.io/mcp, lets the model search the mailing-list archives / commit -history / commitfest data, and emits a Markdown summary linking the changes to -the relevant threads (https://pg.ddx.io/m/pgsql-hackers/). - -Env: - PG_HISTORY_MCP_URL MCP endpoint (default https://pg.ddx.io/mcp) - PG_HISTORY_MODEL Bedrock model id (e.g. us.anthropic.claude-opus-4-8) - AWS_REGION region (creds come from the OIDC step's env) - BASE_REF, HEAD_SHA PR base ref and head sha (for the git diff context) - GH_PR_TITLE PR title (optional, adds context) - PG_HISTORY_OUT output markdown path (default /tmp/pg-history.md) -Writes the markdown to PG_HISTORY_OUT; exits 0 even on soft failures (writes a note). -""" -import json, os, subprocess, sys, urllib.request - -MCP_URL = os.environ.get("PG_HISTORY_MCP_URL", "https://pg.ddx.io/mcp") -MODEL = os.environ.get("PG_HISTORY_MODEL", "us.anthropic.claude-opus-4-8").replace("bedrock/converse/", "").replace("bedrock/", "") -REGION = os.environ.get("AWS_REGION", "us-east-1") -BASE_REF = os.environ.get("BASE_REF", "") -HEAD_SHA = os.environ.get("HEAD_SHA", "") -PR_TITLE = os.environ.get("GH_PR_TITLE", "") -OUT = os.environ.get("PG_HISTORY_OUT", "/tmp/pg-history.md") -UA = "pg-history/0.1 (+github-actions)" - -# Curated subset of the 108 Agora tools — the ones useful for connecting a -# change to its discussion/commit history. Intersected with what the server -# actually exposes, so unknown names are harmless. -TOOL_WHITELIST = { - "find_related_discussions", "find_similar_messages", "get_thread", - "discussion_links", "get_author_messages", "browse_by_date", - "blame_symbol", "check_upstream_status", "find_related", - "find_entries_for_thread", "find_entries_for_author", "get_commit", - "search", "hybrid_search", "get_callers", "get_callees", "find_pattern", -} -MAX_ROUNDS = 14 -TOOL_RESULT_CAP = 8000 # chars per tool result fed back to the model - - -def _mcp_post(body, sid=None): - headers = {"Content-Type": "application/json", - "Accept": "application/json, text/event-stream", "User-Agent": UA} - if sid: - headers["Mcp-Session-Id"] = sid - req = urllib.request.Request(MCP_URL, data=json.dumps(body).encode(), headers=headers, method="POST") - resp = urllib.request.urlopen(req, timeout=60) - sid_out = resp.headers.get("Mcp-Session-Id") - result = None - for line in resp.read().decode().splitlines(): - line = line.strip() - if line.startswith("data:"): - line = line[5:].strip() - if not line or line.startswith("event:"): - continue - try: - obj = json.loads(line) - except Exception: - continue - if isinstance(obj, dict) and ("result" in obj or "error" in obj): - result = obj - return result, sid_out - - -class MCP: - def __init__(self): - init, self.sid = _mcp_post({"jsonrpc": "2.0", "id": 1, "method": "initialize", - "params": {"protocolVersion": "2025-06-18", "capabilities": {}, - "clientInfo": {"name": "pg-history", "version": "0.1"}}}) - if not init or "result" not in init: - raise RuntimeError(f"MCP initialize failed: {init}") - try: - _mcp_post({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, self.sid) - except Exception: - pass - self._id = 1 - - def list_tools(self): - self._id += 1 - res, _ = _mcp_post({"jsonrpc": "2.0", "id": self._id, "method": "tools/list", "params": {}}, self.sid) - return (res or {}).get("result", {}).get("tools", []) - - def call(self, name, args): - self._id += 1 - res, _ = _mcp_post({"jsonrpc": "2.0", "id": self._id, "method": "tools/call", - "params": {"name": name, "arguments": args or {}}}, self.sid) - if not res: - return "(no response)" - if "error" in res: - return f"ERROR: {json.dumps(res['error'])[:500]}" - parts = [] - for c in res.get("result", {}).get("content", []): - if c.get("type") == "text": - parts.append(c["text"]) - return ("\n".join(parts) or "(empty)")[:TOOL_RESULT_CAP] - - -def git(*args): - try: - return subprocess.check_output(["git", *args], text=True, stderr=subprocess.DEVNULL).strip() - except Exception: - return "" - - -def pr_context(): - base = f"origin/{BASE_REF}" if BASE_REF else "" - rng = f"{base}..{HEAD_SHA}" if base and HEAD_SHA else HEAD_SHA - commits = git("log", "--no-merges", "--format=%h %s", f"{rng}") if rng else "" - stat = git("diff", "--stat", rng) if rng else "" - files = git("diff", "--name-only", rng) if rng else "" - return commits[:4000], stat[:3000], files[:2000] - - -SYSTEM = """You are a PostgreSQL community research assistant. Given a pull request's -commits and changed files, use the available tools (backed by the Agora index of -pgsql-hackers mail, commit history, and commitfest data) to connect the change to -its history. Your goal: - -- Find the mailing-list thread(s) and prior discussion behind this change. -- Identify related/superseded prior commits and any commitfest entry. -- Note relevant prior art, rejected approaches, or design rationale. - -Rules (voice & rigor): -- Be precise and blunt. No praise, no filler, no hedging, no disclaimers. Accuracy is - the only success metric — not the author's approval. Lead with the most important finding. -- NEVER hallucinate. Verify every Message-ID, thread subject, commit hash, author name, - and date against an actual tool result before citing it. If a search returns nothing, - say so plainly — do not guess or fabricate a plausible-looking link. -- Assess the change on its merits, independent of how the PR frames it. -- Tag any inferred (not tool-confirmed) linkage with an explicit confidence level: - high / moderate / low. -- Be decisive and efficient: a handful of targeted tool calls, not exhaustive search. -- Cite every mailing-list message as a Markdown link: [subject](https://pg.ddx.io/m/pgsql-hackers/MESSAGE_ID). -- If you find nothing relevant, say so in one line — do not pad. - -When done, output ONLY Markdown (no preamble) with these sections, omitting any that are empty: -## 🧵 Related discussion -## 🔗 Related commits / prior art -## 📋 Commitfest -## 🧭 Context for reviewers -Keep it tight (use bullets; link generously).""" - - -def to_toolspec(t): - schema = t.get("inputSchema") or {"type": "object", "properties": {}} - return {"toolSpec": {"name": t["name"], - "description": (t.get("description") or "")[:600], - "inputSchema": {"json": schema}}} - - -def main(): - commits, stat, files = pr_context() - if not commits and not files: - open(OUT, "w").write("") # nothing to do - print("No PR diff context; skipping.") - return - user = (f"PR title: {PR_TITLE}\n\n" if PR_TITLE else "") + \ - f"Commits:\n{commits or '(none)'}\n\nChanged files:\n{files or '(none)'}\n\nDiffstat:\n{stat or '(none)'}\n" - - try: - mcp = MCP() - tools = [to_toolspec(t) for t in mcp.list_tools() if t.get("name") in TOOL_WHITELIST] - except Exception as e: - open(OUT, "w").write(f"_pg-history: could not reach the Agora MCP server ({MCP_URL}): {e}_\n") - print(f"MCP unavailable: {e}") - return - if not tools: - open(OUT, "w").write("_pg-history: no usable MCP tools available._\n") - return - - import boto3 - from botocore.config import Config - - # botocore's default read timeout (60s) is too short for a multi-round - # (MAX_ROUNDS) tool-use loop against a large PR diff on a reasoning model; - # each converse() call alone can take several minutes. Bump it well past - # what a single round needs; connect_timeout stays short since a stuck - # TCP handshake is a different (and much cheaper to detect) failure mode. - brt = boto3.client("bedrock-runtime", region_name=REGION, - config=Config(read_timeout=900, connect_timeout=10)) - messages = [{"role": "user", "content": [{"text": user}]}] - final_text = "" - try: - for _ in range(MAX_ROUNDS): - resp = brt.converse( - modelId=MODEL, - system=[{"text": SYSTEM}], - messages=messages, - toolConfig={"tools": tools}, - inferenceConfig={"maxTokens": 4096}, - ) - out = resp["output"]["message"] - messages.append(out) - if resp.get("stopReason") == "tool_use": - results = [] - for blk in out["content"]: - tu = blk.get("toolUse") - if not tu: - continue - res_text = mcp.call(tu["name"], tu.get("input") or {}) - results.append({"toolResult": {"toolUseId": tu["toolUseId"], - "content": [{"text": res_text}]}}) - messages.append({"role": "user", "content": results}) - continue - final_text = "".join(b.get("text", "") for b in out["content"]).strip() - break - except Exception as e: - open(OUT, "w").write(f"_pg-history: Bedrock call failed: {e}_\n") - print(f"Bedrock error: {e}") - return - - if not final_text: - final_text = "_pg-history: no related history found._" - body = "## 📜 Change history & discussion (Agora / pg.ddx.io)\n\n" + final_text + \ - "\n\nGenerated by pg-history via the Agora MCP server (pg.ddx.io).\n" - open(OUT, "w").write(body) - print(body) - - -if __name__ == "__main__": - main() diff --git a/.github/ocr/rule.json b/.github/ocr/rule.json deleted file mode 100644 index 60e13e73dcbe0..0000000000000 --- a/.github/ocr/rule.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "_comment": "OCR per-file review rules for PostgreSQL core + extensions. Cross-cutting contribution standards & mailing-list etiquette live in .github/ocr/context.md, passed via --background-file. OCR uses FIRST-MATCH-WINS in declaration order, so rules are ordered most-specific first. merge_system_rule:true keeps OCR's built-in fine-tuned checks (thread-safety, injection, NPE) alongside these PostgreSQL-specific rules.", - "rules": [ - { - "path": "src/test/**", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL tests. Coverage is mandatory for any behavioral change and must include edge cases (NULL, empty, boundary/overflow) and ERROR paths, not just the happy path. A test that still passes with the feature reverted is worthless — confirm it actually exercises and would catch regressions in the new code. Regression (.sql/expected): deterministic, portable output — ORDER BY where row order matters, no timing/plan-dependent output except intentional EXPLAIN, no absolute paths, locale-independent (C collation or explicit COLLATE), DROP objects the test creates; expected/ output must stay stable across platforms and under the parallel schedule. Concurrency/locking belongs in isolation tests (src/test/isolation, .spec + permutations). End-to-end/crash/replication/CLI behavior belongs in TAP tests (t/*.pl with PostgreSQL::Test::Cluster/Utils) — no hardcoded ports/paths, no sleep as synchronization (use poll_query_until/wait_for), skip cleanly when prerequisites are missing, and clean up nodes." - }, - { - "path": "**/*.{c,h}", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL backend/frontend C — review as pgsql-hackers committers do, in priority order.\n\n(1) CORRECTNESS (highest): Memory — every palloc lives in the right MemoryContext; error paths via ereport/elog(ERROR) must not leak memory/buffers/locks/fds (rely on MemoryContext/ResourceOwner reset or PG_TRY/PG_FINALLY); no use-after-free; delete temp contexts. Concurrency — consistent lock ordering (deadlock-free), correct lock levels, balanced LWLockAcquire/Release and START_/END_CRIT_SECTION, no TOCTOU, CHECK_FOR_INTERRUPTS in long loops, async-signal-safe signal handlers (volatile sig_atomic_t). WAL — any change to shared on-disk state must be WAL-logged AND correctly replayed (redo path), crash- and replica-consistent. NULL/edge/overflow handling.\n\n(2) BACKWARD COMPATIBILITY / ABI: don't break behavior, dump/restore, pg_upgrade, libpq wire protocol, logical-replication protocol, or exported/PGDLLIMPORT'd APIs (struct size/layout, function signatures) without extraordinary justification.\n\n(3) CATALOG / GENERATED: new/changed catalog data goes in pg_*.dat, NOT the generated *_d.h/.bki. New Node types: ANNOTATE the struct in its header so gen_node_support.pl regenerates copy/equal/out/read — do NOT hand-edit *funcs.c. New SQL-callable functions: add to pg_proc.dat with an OID from the 8000-9999 developer range (src/include/catalog/unused_oids; check duplicate_oids); committer renumbers at commit. DO NOT bump CATALOG_VERSION_NO in the patch — flag any catversion.h change as a mistake (committer's job).\n\n(4) PERFORMANCE: no regression on hot paths; avoid O(n^2) where better is feasible; minimize work under contended locks; avoid needless palloc churn and large struct copies in hot paths.\n\n(5) SECURITY: bounded string ops (snprintf/strlcpy/strlcat — never strcpy/strcat/sprintf); integer/size-overflow checks before allocation; never user input as a format string; privilege checks via pg_*_aclcheck; beware search_path and SECURITY DEFINER.\n\n(6) PORTABILITY (hard gate): no unaligned access; no dependence on char signedness, int/long/pointer width, endianness, or struct padding for on-disk/wire formats; use int16/int32/int64 + INT64_FORMAT/UINT64_FORMAT (never %ld for int64); align contended shared structs (pg_attribute_aligned/cache-line pad). Atomics/barriers only via port/atomics (pg_atomic_*, pg_read/write_barrier) — never raw intrinsics or volatile-as-barrier. WINDOWS/MSVC: extern vars used cross-module/extension need PGDLLIMPORT; no VLAs or features beyond the C99 baseline the tree targets; use pg_pread/pg_pwrite. Applies across x86_64/ARM64/RISC-V/PPC64/s390x, big/little endian, 32/64-bit.\n\n(7) CONVENTIONS: errmsg starts lowercase, no trailing period, no embedded newlines; errdetail/errhint are complete capitalized sentences; correct ERRCODE_*; wrap user-facing text in _(); errmsg_plural for counts. Assert() only for can't-happen invariants (never user-reachable). Naming: snake_case with subsystem prefix (heap_insert) or CamelCase for major subsystems (ExecInitNode); ALL_CAPS macros. Must pgindent cleanly (tabs, width 4). Comments explain WHY not WHAT; no #ifdef 0 blocks, no commented-out code, no #ifdef fencing your feature. Reuse existing helpers (DRY)." - }, - { - "path": "**/*.dat", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL catalog data (pg_proc.dat, pg_type.dat, etc.) — the SOURCE for generated headers. The generated *_d.h, postgres.bki, fmgroids.h, fmgrtab.c must NOT be hand-edited (they regenerate from these files). OIDs: use a value from the developer range 8000-9999 (src/include/catalog/unused_oids; verify with duplicate_oids); committer renumbers to a final contiguous block, so stay in-range and unique but don't over-optimize the exact number. Keep proc entries complete/consistent (prosrc, provolatile, proparallel, prorettype/proargtypes, matching description). DO NOT bump CATALOG_VERSION_NO / catversion.h — committer's job at push time; flag any such change. New catalog columns/views need documentation in doc/src/sgml/catalogs.sgml." - }, - { - "path": "**/*.{sql,pgsql}", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL SQL. Valid PostgreSQL dialect (not MySQL/Oracle); correct types (bigint vs int, text vs varchar); sound transaction/isolation and CTE-materialization assumptions. SECURITY: flag SQL injection in dynamic SQL (require quote_identifier/quote_literal or format() with %I/%L), SECURITY DEFINER without a locked-down search_path, inappropriate RLS bypass. Prefer set-based over row-at-a-time/N+1. BACKWARD COMPATIBILITY (a top rejection reason): changing existing SQL behavior, the output of existing functions, or default GUCs needs extraordinary justification. New SQL-callable objects belong in pg_*.dat with OIDs from the 8000-9999 range, not in generated files. Minimal diff; add regression tests + docs." - }, - { - "path": "**/*.{pl,pm}", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Perl (TAP tests and build/catalog tooling). Require 'use strict; use warnings;'. Must be perltidy-clean with the tree's src/tools/pgindent/perltidyrc and pass src/tools/perlcheck/pgperlcritic. Use the framework: PostgreSQL::Test::Cluster, PostgreSQL::Test::Utils, Test::More; no hardcoded ports/paths/PIDs; use safe_psql/poll_query_until, not sleep; skippable without optional prerequisites; clean up nodes. PORTABILITY: run on Windows (no fork-only constructs, use File::Spec, avoid unavailable signals) and the minimum supported Perl. Robustness: avoid two-arg open and string system()/qx with interpolated data (use list forms). Generator scripts (gen_node_support.pl, catalog Perl) must be deterministic and stay in sync with inputs; do not commit their generated output." - }, - { - "path": "**/*.py", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Python (build/test tooling, oauth/pytest tests, src/tools). Follow surrounding style; keep imports to the standard library unless the dependency is already required by the tree (no surprise third-party deps in build/test tooling). PORTABILITY: support the project's minimum Python 3 and run on Windows and the BSDs (use os.path/pathlib, avoid POSIX-only calls and shell=True with interpolated input). Deterministic, self-cleaning tests; no hardcoded ports/paths; skip cleanly without prerequisites. For the Perl->pytest porting effort, confirm behavior parity with the TAP test replaced (same assertions/coverage), not a superficial translation. Minimal diff; match the tree's ruff/black config if present." - }, - { - "path": "**/*.{rs,toml}", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. Rust PostgreSQL extension (pgrx) or Rust support crate. Not core C, but it runs inside/alongside the backend, so backend safety applies. SAFETY: in code reachable from an SQL call, a Rust panic aborts the Postgres process — forbid unwrap()/expect()/panic!/unreachable!/todo! and index-panics on reachable paths; use Result and pgrx error reporting (error!/ereport!). Every `unsafe` block needs a comment justifying its invariant; scrutinize raw pointers and FFI across the pg_sys boundary. pgrx: honor #[pg_guard] on extern C fns (correct panic/longjmp handling); never hold Rust references across SPI or anything that can longjmp (skips Rust destructors -> leaks); respect MemoryContext lifetimes for palloc'd data; datum<->Rust conversions must handle NULL. Concurrency uses Postgres shmem/LWLocks (pgrx shmem API), not std::sync alone. Lints: must pass `cargo clippy --all-targets --all-features -- -D warnings` and `cargo fmt --check`; deny unwrap_used/expect_used/panic in libraries; thiserror (libs) / anyhow (bins). Justify every new dependency. Tests: #[pg_test] for in-backend behavior, #[test] for pure logic; cover error and NULL paths. Minimal, idiomatic diff." - }, - { - "path": "**/{configure.ac,*.m4,aclocal.m4}", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Autoconf. Edit configure.ac / the m4 macros — do NOT hand-edit generated 'configure' or pg_config.h.in in the same patch (regeneration is the committer's step; a patch that also rewrites generated configure output is suspect). Feature/header/function probes must be portable and not assume a specific OS/compiler. Every configure knob must be mirrored on the Meson side (meson_options.txt/meson.build) and documented. Minimal diff." - }, - { - "path": "**/{meson.build,meson_options.txt}", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Meson build. Valid syntax; correct subdir()/dependency()/declare_dependency and install paths; new source files must be added here. CRITICAL: PostgreSQL maintains BOTH Meson and Autoconf/Make — any new file, option, or feature check must be mirrored on the configure.ac/Makefile side so the two never drift (a file built by only one system is a common defect). New options need matching docs and sensible defaults. Minimal diff." - }, - { - "path": "**/{Makefile,GNUmakefile,*.mk}", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Makefile (GNU Make). $(VAR) refs; correct .PHONY; accurate dependencies (no parallel -j races); $(MAKE) for recursion; VPATH/out-of-tree build support; no hardcoded paths (use standard PostgreSQL makefile vars and $(top_builddir)); clean/distclean/maintainer-clean must remove new artifacts; extensions use PGXS. Must stay in sync with meson.build. Minimal diff." - }, - { - "path": "doc/**/*.sgml", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL documentation (DocBook SGML). Technically accurate/complete (parameters, limitations, version/compat notes); correct tag usage/nesting (, , , , , /); working cross-references; spell it 'PostgreSQL' in prose; SQL keywords uppercase in examples. Coverage: a new GUC -> config.sgml (and postgresql.conf.sample); new/changed catalogs or views -> catalogs.sgml; new SQL syntax -> the matching ref/*.sgml; new functions -> func.sgml. Do NOT edit release-notes (release-*.sgml) — written by the release team/committers; flag such edits. New user-facing behavior in this PR should ship with matching docs." - }, - { - "path": "**/*.md", - "merge_system_rule": true, - "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. Markdown docs. Clear heading hierarchy; fenced code blocks with language hints; accurate instructions/prerequisites; consistent PostgreSQL terminology; no broken relative links or stale claims. Minimal diff." - } - ] -} diff --git a/.github/workflows/fork-ocr-model-check.yml b/.github/workflows/fork-ocr-model-check.yml index 10d250528cf7c..b85a990c66fd7 100644 --- a/.github/workflows/fork-ocr-model-check.yml +++ b/.github/workflows/fork-ocr-model-check.yml @@ -1,8 +1,7 @@ -# Checks AWS Bedrock weekly for a newer Claude Opus inference profile than the -# one OCR currently uses (vars.OCR_BEDROCK_MODEL) and, if found, opens/updates a -# single GitHub issue telling the maintainer to bump the variable. It does NOT -# change the model automatically: GITHUB_TOKEN cannot write Actions *variables* -# (that needs a PAT with admin), so this is a notify-only mechanism by design. +# Weekly check for a newer Claude Opus on Bedrock; opens/updates one issue here +# when OCR_BEDROCK_MODEL is behind. Notify-only (GITHUB_TOKEN cannot write +# Actions variables). Logic lives in gburd/ci-workflows. + name: OCR model self-check on: @@ -10,80 +9,10 @@ on: - cron: '0 12 * * 1' # Mondays 12:00 UTC workflow_dispatch: -permissions: - id-token: write - contents: read - issues: write - jobs: - check-model: - runs-on: ubuntu-latest - steps: - - name: Configure AWS credentials (OIDC) - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ vars.AWS_ROLE_ARN }} - aws-region: ${{ vars.AWS_REGION }} - role-session-name: ocr-model-check-${{ github.run_id }} - - - name: Find newest Opus vs configured - id: check - env: - CURRENT: ${{ vars.OCR_BEDROCK_MODEL }} - AWS_REGION: ${{ vars.AWS_REGION }} - run: | - python3 - <<'PY' >> "$GITHUB_OUTPUT" - import os, re, subprocess, json - region = os.environ.get("AWS_REGION", "us-east-1") - current = os.environ.get("CURRENT", "") - out = subprocess.run( - ["aws", "bedrock", "list-inference-profiles", "--region", region, - "--query", "inferenceProfileSummaries[].inferenceProfileId", "--output", "json"], - capture_output=True, text=True) - ids = json.loads(out.stdout or "[]") - # Parse claude-opus-- from any profile id (prefix us./global. ok). - def ver(s): - m = re.search(r"claude-opus-(\d+)-(\d+)", s) - return (int(m.group(1)), int(m.group(2))) if m else None - opus = [(ver(i), i) for i in ids if ver(i) and i.startswith(("us.", "global."))] - if not opus: - print("newer=false"); raise SystemExit(0) - best_ver, best_id = max(opus, key=lambda x: x[0]) - cur = ver(current) - newer = (cur is None) or (best_ver > cur) - print(f"newer={'true' if newer else 'false'}") - print(f"best_id={best_id}") - print(f"best_ver={best_ver[0]}.{best_ver[1]}") - print(f"cur_ver={'unknown' if cur is None else f'{cur[0]}.{cur[1]}'}") - PY - - - name: Open/update issue if a newer model exists - if: steps.check.outputs.newer == 'true' - uses: actions/github-script@v9 - with: - script: | - const best = '${{ steps.check.outputs.best_id }}'; - const bestVer = '${{ steps.check.outputs.best_ver }}'; - const curVer = '${{ steps.check.outputs.cur_ver }}'; - const marker = ''; - const title = `OCR: newer Claude Opus available (${bestVer} > ${curVer})`; - const body = `${marker}\n` + - `A newer Claude Opus inference profile is available on Bedrock.\n\n` + - `- **Configured** (\`vars.OCR_BEDROCK_MODEL\`): Opus ${curVer}\n` + - `- **Newest on Bedrock**: \`${best}\` (Opus ${bestVer})\n\n` + - `To upgrade, set the repo variable:\n\n` + - '```\n' + - `gh variable set OCR_BEDROCK_MODEL -R ${context.repo.owner}/${context.repo.repo} \\\n` + - ` -b "bedrock/converse/${best}"\n` + - '```\n\n' + - `Also confirm the \`ocr-bedrock-ci\` IAM inline policy allows invoking the new model ` + - `(the resource is scoped to \`anthropic.claude-opus-*\`), then re-run OCR.\n\n` + - `_Automated by \`.github/workflows/ocr-model-check.yml\`; this issue is upserted._`; - const q = `repo:${context.repo.owner}/${context.repo.repo} in:body "${marker}" state:open`; - const found = await github.rest.search.issuesAndPullRequests({ q, per_page: 1 }); - if (found.data.total_count > 0) { - const n = found.data.items[0].number; - await github.rest.issues.update({ owner: context.repo.owner, repo: context.repo.repo, issue_number: n, title, body }); - } else { - await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title, body }); - } + check: + uses: gburd/ci-workflows/.github/workflows/ocr-model-check.yml@v1 + with: + aws_role_arn: ${{ vars.AWS_ROLE_ARN }} + aws_region: ${{ vars.AWS_REGION }} + bedrock_model: ${{ vars.OCR_BEDROCK_MODEL }} diff --git a/.github/workflows/fork-ocr-review.yml b/.github/workflows/fork-ocr-review.yml index 0828af429b57c..ae938754339b4 100644 --- a/.github/workflows/fork-ocr-review.yml +++ b/.github/workflows/fork-ocr-review.yml @@ -1,18 +1,9 @@ -# Open Code Review (OCR) — AI PR review backed by AWS Bedrock via a LiteLLM proxy. +# AI PR review (Claude Opus on Bedrock). Logic and review rules live in +# gburd/ci-workflows so this fork's master carries a stub instead of ~1000 +# lines that have to be rebased onto upstream forever. # -# Flow: -# PR opened/updated (incl. DRAFTS) ─┐ -# /open-code-review PR comment ─┼─► start LiteLLM (127.0.0.1:4000 → Bedrock) -# manual workflow_dispatch ─┘ └► ocr review --format json -# └► post inline PR review comments -# -# Required (repo settings — all repo *variables*, no secrets; auth is via GitHub OIDC): -# vars.AWS_ROLE_ARN - IAM role to assume via OIDC (granting bedrock:InvokeModel*) -# vars.AWS_REGION - e.g. us-east-1 -# vars.OCR_BEDROCK_MODEL - LiteLLM model string for the Opus inference profile, e.g. -# bedrock/converse/us.anthropic.claude-opus-4-8 -# -# No static AWS keys are stored. GITHUB_TOKEN (auto) posts the review comments. +# Needs repo variables AWS_ROLE_ARN, AWS_REGION, OCR_BEDROCK_MODEL. Auth is +# GitHub OIDC; no static AWS keys. See the ci-workflows README. name: OCR AI Review @@ -34,311 +25,10 @@ concurrency: group: ocr-review-${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} cancel-in-progress: true -permissions: - id-token: write # required to mint the GitHub OIDC token for AWS role assumption - contents: read - pull-requests: write - jobs: - ocr-review: - runs-on: ubuntu-latest - # PR events always; comment events only when the comment is on a PR and - # starts with the trigger keyword; manual dispatch always. - if: | - github.event_name == 'pull_request' || - github.event_name == 'workflow_dispatch' || - (github.event_name == 'issue_comment' && github.event.issue.pull_request && - (startsWith(github.event.comment.body, '/open-code-review') || - startsWith(github.event.comment.body, '@open-code-review'))) - - env: - # LiteLLM listens on localhost only; this key never leaves the runner. - LITELLM_MASTER_KEY: sk-ocr-ci-local - OCR_BEDROCK_MODEL: ${{ vars.OCR_BEDROCK_MODEL }} - # Region is a static var (safe at job level). AWS credentials are NOT set - # here — they're minted by the OIDC "Configure AWS credentials" step below - # and exported to the environment for the LiteLLM/boto3 Bedrock calls. - AWS_REGION: ${{ vars.AWS_REGION }} - - steps: - - name: Resolve PR context - id: pr - uses: actions/github-script@v9 - with: - script: | - let prNumber; - if (context.eventName === 'pull_request') { - prNumber = context.payload.pull_request.number; - } else if (context.eventName === 'issue_comment') { - prNumber = context.issue.number; - } else { - prNumber = parseInt('${{ github.event.inputs.pr_number }}', 10); - } - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - }); - const { data: repo } = await github.rest.repos.get({ - owner: context.repo.owner, - repo: context.repo.repo, - }); - core.setOutput('number', String(prNumber)); - core.setOutput('base_ref', pr.base.ref); - core.setOutput('head_ref', pr.head.ref); - core.setOutput('head_sha', pr.head.sha); - core.setOutput('default_branch', repo.default_branch); - core.setOutput('cross_repo', String(pr.head.repo.full_name !== pr.base.repo.full_name)); - - # NOTE: do NOT checkout the PR head. OCR reads the diff and file contents - # straight from git refs (git diff , git show :path, - # git grep ), so the working tree is irrelevant — but our OCR config - # lives on the default branch, not on the PR branch. We check out the repo - # (default ref), fetch the base/head objects, and materialize the config - # from origin/. - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Prepare git refs and OCR config - env: - BASE_REF: ${{ steps.pr.outputs.base_ref }} - HEAD_REF: ${{ steps.pr.outputs.head_ref }} - HEAD_SHA: ${{ steps.pr.outputs.head_sha }} - DEFAULT_BRANCH: ${{ steps.pr.outputs.default_branch }} - run: | - git fetch --no-tags origin "+refs/heads/${DEFAULT_BRANCH}:refs/remotes/origin/${DEFAULT_BRANCH}" || true - git fetch --no-tags origin "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" || true - git fetch --no-tags origin "+refs/heads/${HEAD_REF}:refs/remotes/origin/${HEAD_REF}" || true - git fetch --no-tags origin "${HEAD_SHA}" || true - - # OCR config lives on the default branch; materialize it independently - # of whatever ref is checked out. - mkdir -p "$RUNNER_TEMP/ocr" - git show "origin/${DEFAULT_BRANCH}:.github/ocr/litellm.yaml" > "$RUNNER_TEMP/ocr/litellm.yaml" - git show "origin/${DEFAULT_BRANCH}:.github/ocr/rule.json" > "$RUNNER_TEMP/ocr/rule.json" - git show "origin/${DEFAULT_BRANCH}:.github/ocr/context.md" > "$RUNNER_TEMP/ocr/context.md" - echo "Config materialized:"; ls -l "$RUNNER_TEMP/ocr" - - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '3.12' - - - name: Setup Node.js - uses: actions/setup-node@v6 - with: - node-version: '20' - - - name: Install LiteLLM proxy + Open Code Review - run: | - python -m pip install --upgrade pip - # Pin LiteLLM to a main commit that supports Claude Opus 4.8 adaptive - # thinking (maps reasoning_effort -> output_config.effort, incl. xhigh). - # Not in any tagged release yet (PyPI latest 1.87.1 lacks the Opus - # normalizer). Bump this SHA once a release ships the feature. - pip install "litellm[proxy] @ git+https://github.com/BerriAI/litellm.git@5be0797d24a2f26eb2123e13788f90055a59d91d" - npm install -g @alibaba-group/open-code-review - - - name: Configure AWS credentials (OIDC) - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ vars.AWS_ROLE_ARN }} - aws-region: ${{ vars.AWS_REGION }} - role-session-name: ocr-review-${{ github.run_id }} - - - name: Start LiteLLM proxy (Bedrock bridge) - run: | - if [ -z "$OCR_BEDROCK_MODEL" ]; then - echo "::error::vars.OCR_BEDROCK_MODEL is not set (e.g. bedrock/converse/us.anthropic.claude-opus-4-1-20250805-v1:0)" - exit 1 - fi - nohup litellm --config "$RUNNER_TEMP/ocr/litellm.yaml" --host 127.0.0.1 --port 4000 \ - > /tmp/litellm.log 2>&1 & - echo "Waiting for LiteLLM to become ready..." - for i in $(seq 1 60); do - if curl -sf http://127.0.0.1:4000/health/readiness >/dev/null; then - echo "LiteLLM ready."; exit 0 - fi - sleep 2 - done - echo "::error::LiteLLM did not become ready in time"; cat /tmp/litellm.log; exit 1 - - - name: Configure OCR - run: | - ocr config set llm.url http://127.0.0.1:4000/v1/chat/completions - ocr config set llm.auth_token "$LITELLM_MASTER_KEY" - ocr config set llm.model ocr-bedrock - ocr config set llm.use_anthropic false - ocr config set language English - - - name: Run OCR review - run: | - ocr review \ - --from "origin/${{ steps.pr.outputs.base_ref }}" \ - --to "${{ steps.pr.outputs.head_sha }}" \ - --rule "$RUNNER_TEMP/ocr/rule.json" \ - --background-file "$RUNNER_TEMP/ocr/context.md" \ - --concurrency 3 \ - --timeout 20 \ - --format json \ - > /tmp/ocr-result.json 2>/tmp/ocr-stderr.log || true - echo "----- OCR stdout -----"; cat /tmp/ocr-result.json || true - echo "----- OCR stderr -----"; cat /tmp/ocr-stderr.log || true - echo "----- LiteLLM log (tail) -----"; tail -n 50 /tmp/litellm.log || true - - - name: Post review to PR - uses: actions/github-script@v9 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const fs = require('fs'); - const prNumber = parseInt('${{ steps.pr.outputs.number }}', 10); - const commitSha = '${{ steps.pr.outputs.head_sha }}'; - - // Opus at high effort can emit dozens of findings. Posting them all - // one-by-one trips GitHub's SECONDARY rate limit (403 "content - // creation"), which is what made every run fail after the review - // was already generated. We (a) cap inline comments and overflow - // the rest into the summary, (b) prefer a single bulk createReview, - // and (c) throttle + back off with Retry-After on any fallback. - const MAX_INLINE = 25; - const sleep = (ms) => new Promise(r => setTimeout(r, ms)); - - async function withRetry(fn, label) { - for (let attempt = 1; attempt <= 5; attempt++) { - try { return await fn(); } - catch (e) { - const status = e.status || (e.response && e.response.status); - const h = (e.response && e.response.headers) || {}; - const isRate = status === 403 || status === 429; - if (!isRate || attempt === 5) throw e; - let waitMs = 0; - if (h['retry-after']) waitMs = parseInt(h['retry-after'], 10) * 1000; - else if (h['x-ratelimit-reset']) waitMs = parseInt(h['x-ratelimit-reset'], 10) * 1000 - Date.now(); - if (!waitMs || Number.isNaN(waitMs) || waitMs < 0) waitMs = 1000 * Math.pow(2, attempt); - waitMs = Math.min(waitMs, 60000) + 500; - core.warning(`${label}: rate-limited (status ${status}); waiting ${Math.round(waitMs / 1000)}s (attempt ${attempt}/5)`); - await sleep(waitMs); - } - } - } - - let result; - try { - result = JSON.parse(fs.readFileSync('/tmp/ocr-result.json', 'utf8')); - } catch (e) { - const stderr = (() => { try { return fs.readFileSync('/tmp/ocr-stderr.log', 'utf8').trim(); } catch { return ''; } })(); - await withRetry(() => github.rest.issues.createComment({ - owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, - body: `⚠️ **OCR** could not produce a review.\n\n\`\`\`\n${(stderr || e.message).slice(0, 8000)}\n\`\`\``, - }), 'error-comment'); - return; - } - - const comments = result.comments || []; - const warnings = result.warnings || []; - - const formatComment = (c) => { - let body = c.content || ''; - if (c.suggestion_code && c.existing_code) { - body += '\n\n```suggestion\n' + c.suggestion_code + (c.suggestion_code.endsWith('\n') ? '' : '\n') + '```'; - } - return body; - }; - const formatMarkdown = (c) => { - let md = `### 📄 \`${c.path}\``; - if (c.start_line && c.end_line) md += ` (L${c.start_line}-L${c.end_line})`; - md += '\n\n' + (c.content || ''); - if (c.suggestion_code && c.existing_code) { - md += '\n\n
💡 Suggested change\n\n'; - md += '**Before:**\n```\n' + c.existing_code + '\n```\n\n**After:**\n```\n' + c.suggestion_code + '\n```\n\n
'; - } - return md; - }; - - if (comments.length === 0) { - await withRetry(() => github.rest.issues.createComment({ - owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, - body: `✅ **OCR**: ${result.message || 'No issues found.'}`, - }), 'no-issues-comment'); - return; - } - - const inlineAll = []; - const noLine = []; - for (const c of comments) { - const body = formatComment(c); - const hasLine = (c.start_line >= 1) || (c.end_line >= 1); - if (!hasLine) { noLine.push(c); continue; } - const rc = { path: c.path, body, side: 'RIGHT' }; - if (c.start_line >= 1 && c.end_line >= 1 && c.start_line !== c.end_line) { - rc.start_line = c.start_line; rc.line = c.end_line; rc.start_side = 'RIGHT'; - } else { - rc.line = c.end_line >= 1 ? c.end_line : c.start_line; - } - inlineAll.push({ rc, c }); - } - - const inline = inlineAll.slice(0, MAX_INLINE).map(x => x.rc); - const overflow = inlineAll.slice(MAX_INLINE).map(x => x.c); - - let summary = `🔍 **OCR** found **${comments.length}** issue(s).`; - summary += `\n- ${inline.length} inline, ${noLine.length + overflow.length} in summary`; - if (overflow.length) summary += ` (inline capped at ${MAX_INLINE})`; - if (warnings.length) summary += `\n- ⚠️ ${warnings.length} warning(s) during review`; - for (const c of noLine.concat(overflow)) summary += '\n\n---\n\n' + formatMarkdown(c); - - // Preferred path: ONE createReview carrying every inline comment. - try { - await withRetry(() => github.rest.pulls.createReview({ - owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, - commit_id: commitSha, body: summary, event: 'COMMENT', comments: inline, - }), 'bulk-review'); - return; - } catch (e) { - core.warning(`bulk createReview failed (${e.status || '?'}: ${e.message}); falling back to throttled per-comment posting`); - } - - // Fallback: an invalid inline position (line not in the diff -> 422) - // rejects the whole bulk review. Post the summary, then each comment - // individually with a delay + backoff, skipping ones GitHub rejects. - let ok = 0; const failed = []; - try { - await withRetry(() => github.rest.pulls.createReview({ - owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, - commit_id: commitSha, body: summary, event: 'COMMENT', - }), 'summary-review'); - } catch (err) { failed.push(`summary: ${err.message}`); } - - for (const rc of inline) { - try { - await withRetry(() => github.rest.pulls.createReviewComment({ - owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, - commit_id: commitSha, path: rc.path, body: rc.body, - ...(rc.start_line ? { start_line: rc.start_line, start_side: rc.start_side } : {}), - line: rc.line, side: rc.side, - }), `comment ${rc.path}:${rc.line}`); - ok++; - } catch (inner) { - failed.push(`\`${rc.path}\` L${rc.line}: ${inner.message}`); - } - await sleep(1200); // stay under the secondary content-creation limit - } - - if (failed.length) { - await withRetry(() => github.rest.issues.createComment({ - owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, - body: `📊 OCR posted ${ok}/${inline.length} inline comment(s).\n\n
${failed.length} could not be posted\n\n${failed.join('\n')}\n
`, - }), 'summary-failures'); - } - - # Companion job: OCR can't call MCP, so this separate agent ties the PR's - # changes to PostgreSQL git + pgsql-hackers history via the Agora MCP server - # (pg.ddx.io) and posts a single, upserted "history & discussion" comment. - pg-history: - runs-on: ubuntu-latest + review: + # Comment events only when the comment is on a PR and starts with the + # trigger keyword; PR events and manual dispatch always. if: | github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || @@ -346,82 +36,8 @@ jobs: (startsWith(github.event.comment.body, '/open-code-review') || startsWith(github.event.comment.body, '@open-code-review') || startsWith(github.event.comment.body, '/pg-history'))) - steps: - - name: Resolve PR context - id: pr - uses: actions/github-script@v9 - with: - script: | - let prNumber; - if (context.eventName === 'pull_request') prNumber = context.payload.pull_request.number; - else if (context.eventName === 'issue_comment') prNumber = context.issue.number; - else prNumber = parseInt('${{ github.event.inputs.pr_number }}', 10); - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber }); - core.setOutput('number', String(prNumber)); - core.setOutput('base_ref', pr.base.ref); - core.setOutput('head_sha', pr.head.sha); - core.setOutput('title', pr.title || ''); - - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Make base/head refs available - env: - BASE_REF: ${{ steps.pr.outputs.base_ref }} - HEAD_SHA: ${{ steps.pr.outputs.head_sha }} - run: | - git fetch --no-tags origin "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" || true - git fetch --no-tags origin "${HEAD_SHA}" || true - - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '3.12' - - - name: Configure AWS credentials (OIDC) - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ vars.AWS_ROLE_ARN }} - aws-region: ${{ vars.AWS_REGION }} - role-session-name: pg-history-${{ github.run_id }} - - - name: Install deps - run: pip install boto3 - - - name: Run pg-history (Agora MCP) - env: - PG_HISTORY_MODEL: ${{ vars.OCR_BEDROCK_MODEL }} - AWS_REGION: ${{ vars.AWS_REGION }} - BASE_REF: ${{ steps.pr.outputs.base_ref }} - HEAD_SHA: ${{ steps.pr.outputs.head_sha }} - GH_PR_TITLE: ${{ steps.pr.outputs.title }} - PG_HISTORY_OUT: ${{ runner.temp }}/pg-history.md - run: | - python .github/ocr/pg-history.py || true - echo "----- output -----"; cat "${{ runner.temp }}/pg-history.md" 2>/dev/null || echo "(no output)" - - - name: Upsert PR comment - uses: actions/github-script@v9 - with: - script: | - const fs = require('fs'); - const path = process.env.RUNNER_TEMP + '/pg-history.md'; - let body = ''; - try { body = fs.readFileSync(path, 'utf8').trim(); } catch (e) {} - if (!body) { console.log('pg-history: empty output, nothing to post'); return; } - const prNumber = parseInt('${{ steps.pr.outputs.number }}', 10); - const marker = ''; - body = marker + '\n' + body; - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, per_page: 100 }); - const mine = comments.find(c => c.user.type === 'Bot' && c.body && c.body.includes(marker)); - if (mine) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, repo: context.repo.repo, comment_id: mine.id, body }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, body }); - } + uses: gburd/ci-workflows/.github/workflows/ocr-review.yml@v1 + with: + aws_role_arn: ${{ vars.AWS_ROLE_ARN }} + aws_region: ${{ vars.AWS_REGION }} + bedrock_model: ${{ vars.OCR_BEDROCK_MODEL }} diff --git a/.github/workflows/fork-sync-upstream.yml b/.github/workflows/fork-sync-upstream.yml index e950821239968..022b3101b5a9d 100644 --- a/.github/workflows/fork-sync-upstream.yml +++ b/.github/workflows/fork-sync-upstream.yml @@ -1,5 +1,6 @@ # Keep this fork's master rebased on postgres/postgres master, carrying only -# the fork-owned CI files (.github/workflows/fork-*.yml and .github/ocr/). +# the fork-owned CI stubs (.github/workflows/fork-*.yml). The workflows they +# call, and the OCR review rules, live in gburd/ci-workflows. # # Requires the SYNC_TOKEN secret: a PAT with repo+workflow scope. The default # GITHUB_TOKEN is not allowed to push commits that touch .github/workflows/, @@ -38,7 +39,7 @@ jobs: # Master carries fork CI only. Anything else means work landed here # that belongs on a branch; force-rebasing it on a timer would be wrong. if git diff --name-only upstream/master...HEAD | - grep -vE '^(\.github/workflows/fork-|\.github/ocr/)'; then + grep -v '^\.github/workflows/fork-'; then echo "::error::master carries non-CI changes (listed above); move them to a branch" exit 1 fi From 942c59c95373c0655edf44bae903820666c41b14 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 14 Sep 2026 13:03:39 +0000 Subject: [PATCH 3/5] Add a cluster-wide UNDO engine with records in the WAL stream Add a cluster-wide UNDO engine whose records live in the WAL stream. A consumer appends records describing how to reverse a change to its transaction's UndoRecordSet, and the set is written as an XLOG_UNDO_BATCH record in the critical section of the consumer's own WAL record. Each batch links to the transaction's previous one, so a transaction's UNDO forms a chain that rollback walks backward. * Rollback. AbortTransaction() and ROLLBACK PREPARED apply the chain synchronously (ApplyUndoChainFromWAL()); a subtransaction abort applies the batches written since the subtransaction started. Each record goes to the UNDO resource manager its header names (undormgr.c), registered through the initially empty access/undormgrlist.h, so the engine never interprets a payload. * Crash recovery. Redo tracks the transactions that wrote UNDO, through XLOG_UNDO_BATCH records and, for batches before the redo pointer, the XLOG_UNDO_INFLIGHT record each online checkpoint writes. Once WAL is writable, the startup process rolls back the transactions the crash left in progress and logs each rollback in an XLOG_UNDO_ROLLBACK record, whose redo applies the same records on a standby and in a later recovery. * WAL retention. Each backend publishes the LSN of its transaction's first batch, and KeepLogSeg() keeps WAL from the oldest one, or a prepared transaction's chain head, onward. * A buffer-manager helper, BufferLockReleaseAll(), that the inline apply uses to release content locks an aborting statement left held, and documentation. The next patch adds the first UNDO-writing consumer, transactional filesystem operations (FILEOPS). This cluster-wide UNDO engine draws on two lines of prior work proposed on pgsql-hackers: the zheap storage engine and its undo-log infrastructure, and the zedstore columnar engine whose design discussion shaped the record and apply model used here. Author: Greg Burd Co-authored-by: Amit Kapila Co-authored-by: Dilip Kumar Co-authored-by: Thomas Munro Co-authored-by: Kuntal Ghosh Co-authored-by: Rafia Sabih Co-authored-by: Mithun Cy Co-authored-by: Ashutosh Sharma Co-authored-by: Andres Freund Co-authored-by: Antonin Houska Co-authored-by: Ashwin Agrawal Co-authored-by: Heikki Linnakangas Co-authored-by: Taylor Vesely Co-authored-by: Melanie Plageman Co-authored-by: Alexandra Wang Reviewed-by: shihao zhong Reviewed-by: Manuel Reyes Bravo Discussion: https://postgr.es/m/7BNP_kRdfRAWyq4swht7lEjtTjEyIPgbDLkR7PXvp9qgNPfXb_fbg4k8ceAd6j-4genQldy9CY9j9KthZUI0SducvaQsxSDoa51e0GdEtQo=@burd.me Changes since v2: - Remove the unused table AM prepare hook [a1ef4339f9] - Headerscheck: skip undormgrlist.h [3e9124969b] - Describe the cluster-wide UNDO recovery model as implemented [3a1784e689] - Release buffer content locks before applying UNDO inline at abort [1fdfe0a8a9] - Do not claim a failed abort keeps its WAL hold until backend exit [4825775611] - Doc: make the UNDO page a chapter, as postgres.sgml includes it [479155221f] Changes since v7: - An abort whose last batch LSN is not a valid UNDO batch no longer leaks its "Inline UNDO Apply" memory context; the early exit skipped the switch back to the caller's context and the delete, so the rest of the abort ran in, and returned into, a context that was never freed (test_fileops/023, with the FILEOPS commit's test module; injection point inline-undo-invalid-batch) - The startup undo phase counts a record that fails to apply as a failure, and its closing message says how many there were, as a WARNING. It counted every record as applied and reported the transaction as rolled back (test_fileops/024: a real EACCES on a crash loser's inverse chmod) - Remove what only the commits that are no longer in the series used: undo_bufmgr.c/h, MarkBufferDirtyShared(), DropRelationBufferRange(), the deferred-batch and two-part record entry points the index AMs called, the per-backend UNDO fields in TransactionStateData, the nbtree/hash UNDO rmgr IDs, the revert worker's LWLock tranche, and the legacy no-op stubs and getters that had no caller at all, with AtAbort_XactUndo's and AtProcExit_XactUndo's calls to two of them, which the FILEOPS commit used to remove; about 1,700 lines - Comments, log messages, the README and the documentation chapter no longer describe the ATM, the logical revert worker, per-backend UNDO, index UNDO (index_undo) or FLUX. The startup undo phase's message for a transaction it cannot apply now says that, not that it was handed to a revert worker. doc/src/sgml/undo.sgml is rewritten around what this series does (FILEOPS) and drops pg_stat_undo, index_undo and the table AM DML records, none of which exist here - undostats.c no longer marks its three SQL functions with PG_FUNCTION_INFO_V1. They are built-in (pg_proc.dat), so fmgrprotos.h already declares them, and MSVC rejected the second declaration with C2375 Changes since v9: - Each online checkpoint writes an XLOG_UNDO_INFLIGHT record listing the UNDO chain heads of transactions in flight, and redo starts tracking them from it. Redo starts at the redo pointer, so a transaction whose UNDO batches all preceded it was never seen by the startup undo phase and was not rolled back (Shihao; test_fileops/005). InsertXactUndoData() delays the checkpoint's redo-pointer phase while it inserts a batch and publishes the chain head, so no batch can fall between the two - The UNDO chapter describes the abort record carrying the inverses, the syscall-first ordering and its window, and the in-flight record Changes since v10: - The undo phase logs the rollback of each crash loser in an XLOG_UNDO_ROLLBACK record, which carries the UNDO records that applied, and redo of that record applies them again. It wrote no WAL before, so a streaming standby kept a crash loser's changes until it was promoted, and then, like an archive recovery that crossed a crash of the primary, reverted the loser only at its own end, after later transactions might have reused its names (Manu; test_fileops/027 in the next commit). A standby no longer needs the loser's UNDO batches either, which a restartpoint may have removed (test_fileops/029). To write WAL the undo phase now runs in StartupXLOG(), just before the end-of-recovery checkpoint, instead of at the end of PerformWalRecovery(); the series no longer touches xlogrecovery.c - XLOG_UNDO_APPLY_RECORD, xl_undo_apply and its redo are gone, with the README text and comments that described CLRs: nothing in the series wrote one. Their opcode now names XLOG_UNDO_ROLLBACK. The rm_undo contract in undormgr.h says a callback may run more than once and must not write WAL. The undo phase no longer keeps a list of transactions it could not apply, which nothing in the series could add to - typedefs.list gains XactUndoInflightSlot, xl_undo_inflight and xl_undo_inflight_xact; pgindent - Remove the per-phase checkpoint timing (eight timers and an extra LOG line in CreateCheckPoint()), leftover diagnostics unrelated to UNDO - Both UNDO apply loops refuse a record whose payload length exceeds the record; the UNDO discard worker uses the standard SignalHandlerForShutdownRequest(); LSNs print as %X/%08X; undo_desc() describes an unknown record type; drop the unused SizeOfXactPrepare and a static assertion that checked nothing - Remove what nothing in the series uses: the UNDO discard worker, the three statistics functions and their pg_proc entries, the per-log control array and its states, the redo of five record types nothing writes (the remaining three are renumbered from 0x00), the deferred batch path and its two GUCs, urec_prev and the other record-set fields nothing read, the tableam.h am_supports_undo flag, and the wait events and LWLock tranche they used; about 3,300 lines. The README is rewritten around what the engine does now. The series no longer needs a catversion bump --- doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/postgres.sgml | 1 + doc/src/sgml/undo.sgml | 137 +++ src/backend/access/Makefile | 3 +- src/backend/access/meson.build | 1 + src/backend/access/rmgrdesc/Makefile | 1 + src/backend/access/rmgrdesc/meson.build | 1 + src/backend/access/rmgrdesc/undodesc.c | 107 +++ src/backend/access/transam/rmgr.c | 1 + src/backend/access/transam/twophase.c | 130 ++- src/backend/access/transam/xact.c | 111 +++ src/backend/access/transam/xlog.c | 40 + src/backend/access/undo/Makefile | 25 + src/backend/access/undo/README | 100 ++ src/backend/access/undo/meson.build | 12 + src/backend/access/undo/undo.c | 212 +++++ src/backend/access/undo/undo_xlog.c | 878 +++++++++++++++++ src/backend/access/undo/undoapply.c | 312 ++++++ src/backend/access/undo/undoinsert.c | 127 +++ src/backend/access/undo/undolog.c | 153 +++ src/backend/access/undo/undorecord.c | 360 +++++++ src/backend/access/undo/undormgr.c | 70 ++ src/backend/access/undo/xactundo.c | 1149 +++++++++++++++++++++++ src/backend/commands/tablecmds.c | 1 + src/backend/storage/buffer/bufmgr.c | 73 ++ src/backend/tcop/utility.c | 1 + src/backend/utils/init/postinit.c | 4 + src/bin/pg_dump/pg_dump.c | 1 + src/bin/pg_waldump/rmgrdesc.c | 1 + src/bin/pg_waldump/t/001_basic.pl | 3 +- src/include/access/heapam.h | 1 + src/include/access/rmgrlist.h | 1 + src/include/access/twophase.h | 3 + src/include/access/undo.h | 35 + src/include/access/undo_xlog.h | 169 ++++ src/include/access/undodefs.h | 47 + src/include/access/undolog.h | 41 + src/include/access/undorecord.h | 166 ++++ src/include/access/undormgr.h | 117 +++ src/include/access/undormgrlist.h | 33 + src/include/access/undormgrs.h | 28 + src/include/access/xact.h | 36 + src/include/access/xactundo.h | 88 ++ src/include/access/xlog_internal.h | 2 +- src/include/storage/bufmgr.h | 1 + src/include/storage/subsystemlist.h | 3 + src/tools/pginclude/headerscheck | 1 + src/tools/pgindent/typedefs.list | 19 + 48 files changed, 4803 insertions(+), 4 deletions(-) create mode 100644 doc/src/sgml/undo.sgml create mode 100644 src/backend/access/rmgrdesc/undodesc.c create mode 100644 src/backend/access/undo/Makefile create mode 100644 src/backend/access/undo/README create mode 100644 src/backend/access/undo/meson.build create mode 100644 src/backend/access/undo/undo.c create mode 100644 src/backend/access/undo/undo_xlog.c create mode 100644 src/backend/access/undo/undoapply.c create mode 100644 src/backend/access/undo/undoinsert.c create mode 100644 src/backend/access/undo/undolog.c create mode 100644 src/backend/access/undo/undorecord.c create mode 100644 src/backend/access/undo/undormgr.c create mode 100644 src/backend/access/undo/xactundo.c create mode 100644 src/include/access/undo.h create mode 100644 src/include/access/undo_xlog.h create mode 100644 src/include/access/undodefs.h create mode 100644 src/include/access/undolog.h create mode 100644 src/include/access/undorecord.h create mode 100644 src/include/access/undormgr.h create mode 100644 src/include/access/undormgrlist.h create mode 100644 src/include/access/undormgrs.h create mode 100644 src/include/access/xactundo.h diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 0797dcf96da3a..7800bf9944b7a 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -49,6 +49,7 @@ + diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index 2101442c90fcb..0940a557ffa2e 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -164,6 +164,7 @@ break is not needed in a wider output rendering. &high-availability; &monitoring; &wal; + &undo; &logical-replication; &jit; ®ress; diff --git a/doc/src/sgml/undo.sgml b/doc/src/sgml/undo.sgml new file mode 100644 index 0000000000000..e9a7debfe4d6d --- /dev/null +++ b/doc/src/sgml/undo.sgml @@ -0,0 +1,137 @@ + + + + UNDO Logging + + + UNDO logging + + + + PostgreSQL provides UNDO logging infrastructure + that records the inverse of a change, so that the change can be reversed if + the transaction that made it rolls back or is still in progress when the + server crashes. UNDO records are written into the WAL stream, so they are + durable, replicated and archived like every other WAL record. + + + + UNDO is not a cluster-wide switch. It is always-on infrastructure that is + used only by the subsystems that write UNDO records. The default + heap table access method does not, and ordinary heap and + index changes carry no UNDO overhead. The one user of UNDO in + PostgreSQL today is transactional filesystem + operations (FILEOPS), which make the directory and file changes done by + CREATE DATABASE, ALTER DATABASE SET + TABLESPACE, CREATE TABLESPACE and + DROP TABLESPACE roll back with the transaction. + + + + Transactional Filesystem Operations + + + When a transaction creates, renames or writes a file, creates a directory, + changes a file's mode or owner, truncates a file, or creates a link, + FILEOPS makes the change and then writes a WAL record describing it and an + UNDO record describing how to reverse it. A change that fails is an + ordinary ERROR and leaves nothing in the WAL. If the + transaction commits, the UNDO record is never used. If it rolls back, the + abort record carries the inverses, and the backend applies them, newest + first, before ROLLBACK returns: a created file is + removed, a changed mode is restored, a new database directory is removed + with everything in it, and so on. Removing a file or directory is not + done immediately; it is deferred to commit and simply dropped on rollback. + + + + Between the change and its WAL record there is a short window in which a + crash leaves the change in place with nothing to reverse it, as it would + without FILEOPS. Doing it the other way round, logging first, would turn + every ordinary filesystem error into a PANIC. + + + + FILEOPS writes WAL and UNDO only when is + replica or higher. At minimal it + writes neither, and a crash in the middle of a transaction can leave a + partly done filesystem change behind, as without FILEOPS. + + + + + Crash Recovery and Replication + + + At the end of crash recovery, after redo, the startup process finds every + transaction that wrote UNDO records but neither committed nor aborted + before the crash, applies its UNDO records, and writes a WAL record of the + rollback, before the server accepts connections. Each checkpoint lists + the UNDO chains of transactions in progress, so this includes a + transaction whose UNDO records all precede the checkpoint redo starts + from. This step takes time proportional to + the number of UNDO records those transactions wrote. A prepared + transaction is left prepared; ROLLBACK PREPARED + applies its UNDO records later, and COMMIT PREPARED + discards them. + + + + An UNDO record that cannot be applied, for example because the server no + longer has permission to change the file it describes, is reported with a + WARNING and is not retried, and the change it describes + stays in place. The end-of-recovery summary then also reports how many + records could not be applied. + + + + A physical standby replays the WAL record of each filesystem change, and + the inverses carried by each abort record, so it ends up with the same + files as the primary. Crash recovery replays them the same way. A + transaction lost in a crash of the primary writes no abort record; the + primary's crash recovery writes the rollback record instead, and a + standby replays it. + + + + + UNDO Space Management + + + UNDO records live in the WAL stream as XLOG_UNDO_BATCH + records; there are no separate UNDO files. A checkpoint does not remove + WAL that still holds the UNDO records of a transaction that is in progress + or prepared, so a long-running or long-prepared transaction that wrote UNDO + holds back WAL removal until it ends, much as a replication slot does. + + + + + Monitoring + + + The server log shows UNDO being applied during crash recovery: + + + +LOG: UNDO recovery: rolling back transaction 742 +LOG: UNDO recovery complete: 1 transactions rolled back, 3 records applied + + + + and, if some records could not be applied: + + + +WARNING: UNDO recovery complete: 1 transactions processed, 0 records applied, 1 records could not be applied +DETAIL: The changes those records describe were not rolled back. + + + + pg_waldump shows UNDO records with + and the filesystem changes themselves with + . + + + + diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile index e88d72ea0397d..5778d0d7f8225 100644 --- a/src/backend/access/Makefile +++ b/src/backend/access/Makefile @@ -22,6 +22,7 @@ SUBDIRS = \ sequence \ table \ tablesample \ - transam + transam \ + undo \ include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/meson.build b/src/backend/access/meson.build index 5fd18de74f92b..d569ac4e6e32a 100644 --- a/src/backend/access/meson.build +++ b/src/backend/access/meson.build @@ -14,3 +14,4 @@ subdir('spgist') subdir('table') subdir('tablesample') subdir('transam') +subdir('undo') diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile index cd95eec37f148..bf6709e738d99 100644 --- a/src/backend/access/rmgrdesc/Makefile +++ b/src/backend/access/rmgrdesc/Makefile @@ -29,6 +29,7 @@ OBJS = \ spgdesc.o \ standbydesc.o \ tblspcdesc.o \ + undodesc.o \ xactdesc.o \ xlogdesc.o diff --git a/src/backend/access/rmgrdesc/meson.build b/src/backend/access/rmgrdesc/meson.build index d9000ccd9fd10..d0dc4cb229a18 100644 --- a/src/backend/access/rmgrdesc/meson.build +++ b/src/backend/access/rmgrdesc/meson.build @@ -22,6 +22,7 @@ rmgr_desc_sources = files( 'spgdesc.c', 'standbydesc.c', 'tblspcdesc.c', + 'undodesc.c', 'xactdesc.c', 'xlogdesc.c', ) diff --git a/src/backend/access/rmgrdesc/undodesc.c b/src/backend/access/rmgrdesc/undodesc.c new file mode 100644 index 0000000000000..ba7ed874b5f1e --- /dev/null +++ b/src/backend/access/rmgrdesc/undodesc.c @@ -0,0 +1,107 @@ +/*------------------------------------------------------------------------- + * + * undodesc.c + * rmgr descriptor routines for access/undo + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/rmgrdesc/undodesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undo_xlog.h" +#include "access/xlogreader.h" + +/* + * undo_desc - Describe an UNDO WAL record for pg_waldump + * + * This function generates human-readable output for UNDO WAL records, + * used by pg_waldump and other debugging tools. + */ +void +undo_desc(StringInfo buf, XLogReaderState *record) +{ + char *rec = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_UNDO_ROLLBACK: + { + xl_undo_rollback *xlrec = (xl_undo_rollback *) rec; + + appendStringInfo(buf, "xid %u, nrecords %u", + xlrec->xid, xlrec->nrecords); + } + break; + + case XLOG_UNDO_INFLIGHT: + { + xl_undo_inflight *xlrec = (xl_undo_inflight *) rec; + + appendStringInfo(buf, "nxacts %u", xlrec->nxacts); + for (uint32 i = 0; i < xlrec->nxacts; i++) + { + xl_undo_inflight_xact *x = &xlrec->xacts[i]; + + appendStringInfo(buf, "; xid %u", x->xid); + for (int j = 0; j < NUndoPersistenceLevels; j++) + if (XLogRecPtrIsValid(x->last_batch_lsn[j])) + appendStringInfo(buf, " p%d %X/%08X", j, + LSN_FORMAT_ARGS(x->last_batch_lsn[j])); + } + } + break; + + case XLOG_UNDO_BATCH: + { + xl_undo_batch *xlrec = (xl_undo_batch *) rec; + + appendStringInfo(buf, + "undo batch: xid %u, nrecords %u, " + "total_len %u, chain_prev %X/%08X, " + "primary_reloid %u, persistence %d", + xlrec->xid, + xlrec->nrecords, + xlrec->total_len, + LSN_FORMAT_ARGS(xlrec->chain_prev), + xlrec->primary_reloid, + xlrec->persistence); + } + break; + + default: + appendStringInfo(buf, "unknown undo op code %u", info); + break; + } +} + +/* + * undo_identify - Identify an UNDO WAL record type + * + * Returns a string identifying the operation type for debugging output. + */ +const char * +undo_identify(uint8 info) +{ + const char *id = NULL; + + switch (info & ~XLR_INFO_MASK) + { + case XLOG_UNDO_ROLLBACK: + id = "ROLLBACK"; + break; + case XLOG_UNDO_BATCH: + id = "BATCH"; + break; + case XLOG_UNDO_INFLIGHT: + id = "INFLIGHT"; + break; + } + + return id; +} diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c index 4fda03a3cfcc6..aa427b3a3b09d 100644 --- a/src/backend/access/transam/rmgr.c +++ b/src/backend/access/transam/rmgr.c @@ -30,6 +30,7 @@ #include "access/multixact.h" #include "access/nbtxlog.h" #include "access/spgxlog.h" +#include "access/undo_xlog.h" #include "access/xact.h" #include "catalog/storage_xlog.h" #include "commands/dbcommands_xlog.h" diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index d89afb093209c..6b1b466d859b5 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -77,6 +77,8 @@ #include #include "access/commit_ts.h" +#include "access/undorecord.h" +#include "access/xactundo.h" #include "access/htup_details.h" #include "access/subtrans.h" #include "access/transam.h" @@ -163,6 +165,17 @@ typedef struct GlobalTransactionData */ XLogRecPtr prepare_start_lsn; /* XLOG offset of prepare record start */ XLogRecPtr prepare_end_lsn; /* XLOG offset of prepare record end */ + + /* + * Permanent-level UNDO chain-head LSN for this prepared xact, or + * InvalidXLogRecPtr if it generated no cluster-wide UNDO. Set at PREPARE + * (MarkAsPreparingGuts), at redo (PrepareRedoAdd), and when a stale 2PC + * file is recovered from disk. UndoGetOldestBatchLSN() scans this so the + * UNDO-batch WAL a still-prepared xact needs for ROLLBACK PREPARED is not + * recycled -- the per-backend retention slot is cleared when the + * preparing backend exits, so it does not cover this case. + */ + XLogRecPtr undo_batch_lsn; FullTransactionId fxid; /* The GXACT full xid */ Oid owner; /* ID of user that executed the xact */ @@ -493,6 +506,7 @@ MarkAsPreparingGuts(GlobalTransaction gxact, FullTransactionId fxid, gxact->locking_backend = MyProcNumber; gxact->valid = false; gxact->inredo = false; + gxact->undo_batch_lsn = GetCurrentXactLastBatchLSN(UNDOPERSISTENCE_PERMANENT); strlcpy(gxact->gid, gid, GIDSIZE); /* @@ -979,8 +993,14 @@ TwoPhaseFilePath(char *path, FullTransactionId fxid) /* * Header for a 2PC state file + * + * TWOPHASE_MAGIC must be bumped whenever xl_xact_prepare changes layout. + * The struct gained last_batch_lsn[NUndoPersistenceLevels] (24 bytes) for + * UNDO chain tracking across 2PC boundaries, requiring this bump from + * 0x57F94534 to 0x57F94535 to prevent old servers from silently misreading + * the variable-length arrays that follow the fixed header at the wrong offsets. */ -#define TWOPHASE_MAGIC 0x57F94534 /* format identifier */ +#define TWOPHASE_MAGIC 0x57F94535 /* format identifier */ typedef xl_xact_prepare TwoPhaseFileHeader; @@ -1102,6 +1122,10 @@ StartPrepare(GlobalTransaction gxact) hdr.origin_lsn = InvalidXLogRecPtr; hdr.origin_timestamp = 0; + /* Save UNDO chain head LSNs so recovery can find UNDO records */ + for (int j = 0; j < NUndoPersistenceLevels; j++) + hdr.last_batch_lsn[j] = GetCurrentXactLastBatchLSN(j); + save_state_data(&hdr, sizeof(TwoPhaseFileHeader)); save_state_data(gxact->gid, hdr.gidlen); @@ -1499,6 +1523,47 @@ StandbyTransactionIdIsPrepared(TransactionId xid) return result; } +/* + * RecoveryTransactionIdIsPrepared + * Check if a transaction ID is in the in-memory prepared transaction list. + * + * This is used during crash recovery UNDO phase, before prepared transaction + * files exist on disk. It checks the in-memory TwoPhaseState that was + * reconstructed from WAL replay. + */ +bool +RecoveryTransactionIdIsPrepared(TransactionId xid) +{ + int i; + FullTransactionId fxid; + + Assert(TransactionIdIsValid(xid)); + + if (max_prepared_xacts <= 0) + return false; /* 2PC not enabled */ + + if (TwoPhaseState == NULL) + return false; /* 2PC not initialized yet */ + + fxid = AdjustToFullTransactionId(xid); + + LWLockAcquire(TwoPhaseStateLock, LW_SHARED); + + for (i = 0; i < TwoPhaseState->numPrepXacts; i++) + { + GlobalTransaction gxact = TwoPhaseState->prepXacts[i]; + + if (FullTransactionIdEquals(gxact->fxid, fxid)) + { + LWLockRelease(TwoPhaseStateLock); + return true; + } + } + + LWLockRelease(TwoPhaseStateLock); + return false; +} + /* * FinishPreparedTransaction: execute COMMIT PREPARED or ROLLBACK PREPARED */ @@ -1586,6 +1651,7 @@ FinishPreparedTransaction(const char *gid, bool isCommit) hdr->ninvalmsgs, invalmsgs, hdr->initfileinval, gid); else + { RecordTransactionAbortPrepared(xid, hdr->nsubxacts, children, hdr->nabortrels, abortrels, @@ -1593,6 +1659,26 @@ FinishPreparedTransaction(const char *gid, bool isCommit) abortstats, gid); + /* + * ROLLBACK PREPARED: apply any cluster-wide UNDO chain synchronously, + * here, before the prepared transaction's resources are torn down. + * The permanent-level chain-head LSN was durably saved in the 2PC + * header at PREPARE; ApplyUndoChainFromWAL() walks the UNDO chain + * backwards from that LSN, restoring before-images. + * + * Only the permanent level is applied: TEMP undo is gone (the + * originating backend exited) and UNLOGGED forks are reset on crash; + * this matches the crash-recovery UNDO phase in undo_xlog.c. + */ + { + XLogRecPtr perm_lsn = + hdr->last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; + + if (XLogRecPtrIsValid(perm_lsn)) + (void) ApplyUndoChainFromWAL(perm_lsn); + } + } + ProcArrayRemove(proc, latestXid); /* @@ -2173,6 +2259,14 @@ RecoverPreparedTransactions(void) hdr->prepared_at, hdr->owner, hdr->database); + /* + * MarkAsPreparingGuts read the live backend's last_batch_lsn (wrong + * xact during recovery); restore this xact's real value from the 2PC + * header so WAL retention pins its UNDO batch until ROLLBACK + * PREPARED. + */ + gxact->undo_batch_lsn = hdr->last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; + /* recovered, so reset the flag for entries generated by redo */ gxact->inredo = false; @@ -2646,6 +2740,7 @@ PrepareRedoAdd(FullTransactionId fxid, char *buf, gxact->valid = false; gxact->ondisk = !XLogRecPtrIsValid(start_lsn); gxact->inredo = true; /* yes, added in redo */ + gxact->undo_batch_lsn = hdr->last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; strlcpy(gxact->gid, gid, GIDSIZE); /* And insert it into the active array */ @@ -2927,3 +3022,36 @@ TwoPhaseGetOldestXidInCommit(void) return oldestRunningXid; } + +/* + * TwoPhaseGetOldestUndoBatchLSN + * Return the oldest permanent-level UNDO chain-head LSN across all + * prepared transactions, or InvalidXLogRecPtr if none generated UNDO. + * + * UndoGetOldestBatchLSN() folds this into the WAL-retention horizon so the + * UNDO-batch WAL a still-prepared xact needs for ROLLBACK PREPARED survives + * checkpoints and backend exit (see undo_batch_lsn in GlobalTransactionData). + */ +XLogRecPtr +TwoPhaseGetOldestUndoBatchLSN(void) +{ + XLogRecPtr oldest = InvalidXLogRecPtr; + + if (max_prepared_xacts <= 0) + return InvalidXLogRecPtr; + + LWLockAcquire(TwoPhaseStateLock, LW_SHARED); + + for (int i = 0; i < TwoPhaseState->numPrepXacts; i++) + { + XLogRecPtr lsn = TwoPhaseState->prepXacts[i]->undo_batch_lsn; + + if (XLogRecPtrIsValid(lsn) && + (!XLogRecPtrIsValid(oldest) || lsn < oldest)) + oldest = lsn; + } + + LWLockRelease(TwoPhaseStateLock); + + return oldest; +} diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index d8d91bed42764..0ad6c376c4542 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -26,6 +26,10 @@ #include "access/subtrans.h" #include "access/transam.h" #include "access/twophase.h" +#include "access/undo_xlog.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/xactundo.h" #include "access/xact.h" #include "access/xlog.h" #include "access/xloginsert.h" @@ -317,6 +321,16 @@ typedef struct XactCallbackItem static XactCallbackItem *Xact_callbacks = NULL; +/* + * Pending-structural-file-operation hooks, mirroring smgr's + * own pending-deletes housekeeping ordering (see access/xact.h). NULL when + * no such subsystem is compiled in. + */ +void (*PendingPhysOpsDo_hook) (bool isCommit) = NULL; +void (*PendingPhysOpsPostPrepare_hook) (void) = NULL; +void (*PendingPhysOpsAtSubCommit_hook) (void) = NULL; +void (*PendingPhysOpsAtSubAbort_hook) (void) = NULL; + /* * List of add-on start- and end-of-subxact callbacks */ @@ -419,6 +433,37 @@ IsAbortedTransactionBlockState(void) } +/* + * EnterInlineUndoApplyState / LeaveInlineUndoApplyState + * + * AtAbort_XactUndo() may apply UNDO records synchronously in this backend + * while AbortTransaction() has already set s->state = TRANS_ABORT but has + * not yet torn down the relcache, locks, or resource owner. UNDO appliers + * (e.g. a table AM's undo-apply callback) need to open relations, which trips + * Assert(IsTransactionState()) because TRANS_ABORT is not TRANS_INPROGRESS. + * + * These helpers temporarily present TRANS_INPROGRESS for the duration of the + * inline apply. This is safe because all backing resources are still live at + * this point in the abort sequence; only the state enum has advanced. The + * caller MUST pair Enter/Leave (use PG_TRY/PG_FINALLY) so the real abort state + * is always restored before the rest of AbortTransaction() proceeds. + */ +int +EnterInlineUndoApplyState(void) +{ + TransactionState s = CurrentTransactionState; + int saved = (int) s->state; + + s->state = TRANS_INPROGRESS; + return saved; +} + +void +LeaveInlineUndoApplyState(int saved) +{ + CurrentTransactionState->state = (TransState) saved; +} + /* * GetTopTransactionId * @@ -2464,6 +2509,9 @@ CommitTransaction(void) CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_COMMIT : XACT_EVENT_COMMIT); + /* Clean up transaction undo state (free per-persistence record sets) */ + AtCommit_XactUndo(); + CurrentResourceOwner = NULL; ResourceOwnerRelease(TopTransactionResourceOwner, RESOURCE_RELEASE_BEFORE_LOCKS, @@ -2508,6 +2556,8 @@ CommitTransaction(void) * attempt to access affected files. */ smgrDoPendingDeletes(true); + if (PendingPhysOpsDo_hook) + PendingPhysOpsDo_hook(true); /* * Send out notification signals to other backends (and do other @@ -2796,6 +2846,8 @@ PrepareTransaction(void) PostPrepare_Inval(); PostPrepare_smgr(); + if (PendingPhysOpsPostPrepare_hook) + PendingPhysOpsPostPrepare_hook(); PostPrepare_MultiXact(fxid); @@ -2834,6 +2886,16 @@ PrepareTransaction(void) AtEOXact_LogicalCtl(); pgstat_report_xact_timestamp(0); + /* + * Drop this backend's references to the transaction's cluster-wide UNDO + * record sets. Ownership passed to the 2PC state (undo_batch_lsn) in + * EndPrepare; the record sets live in TopTransactionContext, which + * AtCommit_Memory() destroys just below, so the backend-private pointers + * must be dropped first (otherwise AtProcExit_XactUndo frees freed + * memory). + */ + AtPrepare_XactUndo(); + CurrentResourceOwner = NULL; ResourceOwnerDelete(TopTransactionResourceOwner); s->curTransactionOwner = NULL; @@ -2978,6 +3040,9 @@ AbortTransaction(void) s->parallelModeLevel = 0; s->parallelChildXact = false; /* should be false already */ + /* Clean up transaction undo state (free per-persistence record sets) */ + AtAbort_XactUndo(); + /* * do abort processing */ @@ -3045,7 +3110,10 @@ AbortTransaction(void) ResourceOwnerRelease(TopTransactionResourceOwner, RESOURCE_RELEASE_AFTER_LOCKS, false, true); + smgrDoPendingDeletes(false); + if (PendingPhysOpsDo_hook) + PendingPhysOpsDo_hook(false); AtEOXact_GUC(false, 1); AtEOXact_SPI(false); @@ -5262,6 +5330,8 @@ CommitSubTransaction(void) AtEOSubXact_TypeCache(); AtEOSubXact_Inval(true); AtSubCommit_smgr(); + if (PendingPhysOpsAtSubCommit_hook) + PendingPhysOpsAtSubCommit_hook(); /* * The only lock we actually release here is the subtransaction XID lock. @@ -5453,6 +5523,8 @@ AbortSubTransaction(void) RESOURCE_RELEASE_AFTER_LOCKS, false, false); AtSubAbort_smgr(); + if (PendingPhysOpsAtSubAbort_hook) + PendingPhysOpsAtSubAbort_hook(); AtEOXact_GUC(false, s->gucNestLevel); AtEOSubXact_SPI(false, s->subTransactionId); @@ -6478,6 +6550,11 @@ xact_redo(XLogReaderState *record) ParseCommitRecord(XLogRecGetInfo(record), xlrec, &parsed); xact_redo_commit(&parsed, XLogRecGetXid(record), record->EndRecPtr, XLogRecGetOrigin(record)); + + /* + * Remove from UNDO recovery tracking; committed, no rollback needed + */ + UndoRecoveryRemoveXid(XLogRecGetXid(record)); } else if (info == XLOG_XACT_COMMIT_PREPARED) { @@ -6492,6 +6569,9 @@ xact_redo(XLogReaderState *record) LWLockAcquire(TwoPhaseStateLock, LW_EXCLUSIVE); PrepareRedoRemove(parsed.twophase_xid, false); LWLockRelease(TwoPhaseStateLock); + + /* Remove from UNDO recovery tracking */ + UndoRecoveryRemoveXid(parsed.twophase_xid); } else if (info == XLOG_XACT_ABORT) { @@ -6501,6 +6581,13 @@ xact_redo(XLogReaderState *record) ParseAbortRecord(XLogRecGetInfo(record), xlrec, &parsed); xact_redo_abort(&parsed, XLogRecGetXid(record), record->EndRecPtr, XLogRecGetOrigin(record)); + + /* + * Remove from UNDO recovery tracking; abort record present means the + * UNDO rollback was already completed (or will be handled by the + * abort record's own redo logic). + */ + UndoRecoveryRemoveXid(XLogRecGetXid(record)); } else if (info == XLOG_XACT_ABORT_PREPARED) { @@ -6515,12 +6602,21 @@ xact_redo(XLogReaderState *record) LWLockAcquire(TwoPhaseStateLock, LW_EXCLUSIVE); PrepareRedoRemove(parsed.twophase_xid, false); LWLockRelease(TwoPhaseStateLock); + + /* Remove from UNDO recovery tracking */ + UndoRecoveryRemoveXid(parsed.twophase_xid); } else if (info == XLOG_XACT_PREPARE) { + xl_xact_prepare *xlrec = (xl_xact_prepare *) XLogRecGetData(record); + /* * Store xid and start/end pointers of the WAL record in TwoPhaseState * gxact entry. + * + * NB: xl_xact_prepare includes last_batch_lsn[NUndoPersistenceLevels] + * for UNDO chain tracking across 2PC boundaries, which extended the + * on-disk struct and required an XLOG_PAGE_MAGIC bump. */ LWLockAcquire(TwoPhaseStateLock, LW_EXCLUSIVE); PrepareRedoAdd(InvalidFullTransactionId, @@ -6529,6 +6625,21 @@ xact_redo(XLogReaderState *record) record->EndRecPtr, XLogRecGetOrigin(record)); LWLockRelease(TwoPhaseStateLock); + + /* + * Restore UNDO recovery tracking for the prepared transaction. The + * UNDO chain LSNs were saved in the prepare record so that if the + * server crashes after PREPARE but before COMMIT/ROLLBACK PREPARED, + * recovery can still find and roll back UNDO records. + */ + for (int j = 0; j < NUndoPersistenceLevels; j++) + { + if (!XLogRecPtrIsInvalid(xlrec->last_batch_lsn[j])) + UndoRecoveryTrackBatch(xlrec->xid, + xlrec->last_batch_lsn[j], + InvalidXLogRecPtr, + (UndoPersistenceLevel) j); + } } else if (info == XLOG_XACT_ASSIGNMENT) { diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index fd37ea4f607f8..4754d8cd9360a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -55,7 +55,10 @@ #include "access/timeline.h" #include "access/transam.h" #include "access/twophase.h" +#include "access/undolog.h" +#include "access/undo_xlog.h" #include "access/xact.h" +#include "access/xactundo.h" #include "access/xlog_internal.h" #include "access/xlogarchive.h" #include "access/xloginsert.h" @@ -6836,6 +6839,14 @@ StartupXLOG(void) Insert->fullPageWrites = lastFullPageWrites; UpdateFullPageWrites(); + /* + * Roll back the transactions the crash left in progress that wrote UNDO + * records, and WAL-log each rollback, before the end-of-recovery + * checkpoint or record below makes the rollbacks durable. + */ + if (performedWalRecovery) + UndoRecoveryRollBack(); + /* * Emit checkpoint or end-of-recovery record in XLOG, if required. */ @@ -8055,6 +8066,14 @@ CreateCheckPoint(int flags) if (!shutdown && XLogStandbyInfoActive()) LogStandbySnapshot(); + /* + * Tell crash recovery, which starts at our redo pointer, about the UNDO + * chains of transactions still in flight; it would not replay the batches + * they wrote before the redo pointer. + */ + if (!shutdown) + LogXactUndoInflight(); + START_CRIT_SECTION(); /* @@ -8231,6 +8250,7 @@ CreateCheckPoint(int flags) if (!RecoveryInProgress()) TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning()); + /* Real work is done; log and update stats. */ LogCheckpointEnd(false, flags); @@ -8971,6 +8991,26 @@ KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo) segno = unsummarized_segno; } + /* + * If UNDO-in-WAL is active, retain WAL segments that contain UNDO records + * still needed for rollback of in-progress transactions. + * + * Scan live per-backend UNDO batch LSN slots at every checkpoint rather + * than using the worker-updated cached horizon, to ensure WAL retention + * is accurate even when the UNDO worker lags. + */ + { + keep = UndoGetOldestBatchLSN(); + if (XLogRecPtrIsValid(keep)) + { + XLogSegNo undo_segno; + + XLByteToSeg(keep, undo_segno, wal_segment_size); + if (undo_segno < segno) + segno = undo_segno; + } + } + /* but, keep at least wal_keep_size if that's set */ if (wal_keep_size_mb > 0) { diff --git a/src/backend/access/undo/Makefile b/src/backend/access/undo/Makefile new file mode 100644 index 0000000000000..06a8a9eeda79a --- /dev/null +++ b/src/backend/access/undo/Makefile @@ -0,0 +1,25 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for access/undo +# +# IDENTIFICATION +# src/backend/access/undo/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/access/undo +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = \ + undo.o \ + undo_xlog.o \ + undoapply.o \ + undoinsert.o \ + undolog.o \ + undorecord.o \ + undormgr.o \ + xactundo.o + +include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/undo/README b/src/backend/access/undo/README new file mode 100644 index 0000000000000..0716d69d263a6 --- /dev/null +++ b/src/backend/access/undo/README @@ -0,0 +1,100 @@ +UNDO for PostgreSQL +=================== + +This directory implements a cluster-wide UNDO engine: a way for a +transaction to record how to reverse a change it has made, so that the +change is reversed if the transaction rolls back, or if a crash leaves it +in progress. The first consumer is FILEOPS, the transactional filesystem +operations in storage/file/fileops.c; see storage/file/README. + +Records and chains +------------------ + +A consumer writes UNDO through xactundo.c. PrepareXactUndoData() appends a +record (an UndoRecordHeader, undorecord.h, and an opaque payload) to the +transaction's UndoRecordSet for the record's persistence level, and +InsertXactUndoData() writes the set as one XLOG_UNDO_BATCH WAL record, +inside the critical section of the consumer's own WAL record. There are no +UNDO files: the records live in the WAL stream. + +Each batch carries the LSN of the transaction's previous batch +(xl_undo_batch.chain_prev), so a transaction's UNDO forms a chain from its +newest batch back to its first. Every record carries the top-level xid, +even one written by a subtransaction. urec_rmid names the UNDO resource +manager that applies the record (undormgr.h); consumers register theirs +through access/undormgrlist.h, and the engine never interprets a payload. + +A record is applied by handing it to its resource manager's rm_undo +callback. The engine keeps no "already applied" marker, and the same +record can be applied more than once, so a callback must leave alone an +object that no longer shows the record's effect. A callback must not +write WAL, since redo of XLOG_UNDO_ROLLBACK (below) calls it. + +Rollback +-------- + +AbortTransaction() calls AtAbort_XactUndo(), which walks the chain from its +newest batch, applying each batch's records newest first +(ApplyUndoChainFromWAL(), undoapply.c). A subtransaction abort applies only +the batches written since the subtransaction started, and a subtransaction +commit hands its chain to the parent. ROLLBACK PREPARED applies the chain +whose head PREPARE saved in the 2PC state. + +A consumer decides how its rollback reaches a standby. FILEOPS carries its +inverses in the abort record and applies them from there, so the +transaction-abort walk skips its records (FileOpsSkipUndo). + +Crash recovery +-------------- + +During redo, each XLOG_UNDO_BATCH record registers its transaction in an +in-memory table (undo_xlog.c), and the transaction's commit or abort +record removes it. Redo starts at the checkpoint's redo pointer, so it +replays none of the batches written before it; each online checkpoint +therefore writes an XLOG_UNDO_INFLIGHT record listing the chain heads of +the transactions in progress (LogXactUndoInflight()), which redo adds to +the table. InsertXactUndoData() holds off the checkpoint's redo-pointer +phase while it inserts a batch and publishes its chain head, so no batch +can fall between the two. + +What remains in the table at the end of redo, apart from prepared +transactions, are the transactions the crash left in progress. +StartupXLOG() calls UndoRecoveryRollBack() once WAL is writable, just +before the end-of-recovery checkpoint or record. For each such loser it +applies the UNDO records, newest first, then writes an XLOG_UNDO_ROLLBACK +record carrying the records that applied, and forgets the loser. Redo of +that record applies the same records again and forgets the loser too. So +a standby rolls the loser back at the same point in the WAL as the primary +did, without reading the loser's chain (which a restartpoint may already +have removed), and a later recovery never reverts it a second time. + +A crash before the rollback record reaches disk leaves the loser tracked, +and the next recovery rolls it back again. A record whose callback fails +is left out of the rollback record, so the change it describes stays in +place everywhere, and the summary at the end of the undo phase says how +many records could not be applied. + +Recovery applies only PERMANENT records: temporary relations do not +survive a restart, and unlogged ones are reset to their init forks. + +WAL retention +------------- + +Each backend publishes the LSN of its current transaction's first UNDO +batch in a shared slot (undolog.c), and a prepared transaction's chain +head is kept in its 2PC state. KeepLogSeg() keeps WAL from the oldest of +them onward, so a long-running or long-prepared transaction that wrote +UNDO holds back WAL removal until it ends, much as a replication slot +does. + +Files +----- + + undo.c shared memory set-up and per-backend initialization + undolog.c per-backend first-batch LSN slots for WAL retention + undorecord.c UndoRecordHeader and UndoRecordSet + undoinsert.c writing an UndoRecordSet as an XLOG_UNDO_BATCH record + xactundo.c per-transaction UNDO, the checkpoint in-flight record + undoapply.c online rollback: walking a chain and applying it + undo_xlog.c RM_UNDO_ID redo, the recovery undo phase, batch reads + undormgr.c UNDO resource manager registry diff --git a/src/backend/access/undo/meson.build b/src/backend/access/undo/meson.build new file mode 100644 index 0000000000000..e2f871f7194fd --- /dev/null +++ b/src/backend/access/undo/meson.build @@ -0,0 +1,12 @@ +# Copyright (c) 2022-2026, PostgreSQL Global Development Group + +backend_sources += files( + 'undo.c', + 'undo_xlog.c', + 'undoapply.c', + 'undoinsert.c', + 'undolog.c', + 'undorecord.c', + 'undormgr.c', + 'xactundo.c', +) diff --git a/src/backend/access/undo/undo.c b/src/backend/access/undo/undo.c new file mode 100644 index 0000000000000..d45d2c59f4d65 --- /dev/null +++ b/src/backend/access/undo/undo.c @@ -0,0 +1,212 @@ +/*------------------------------------------------------------------------- + * + * undo.c + * Common undo layer coordination + * + * The undo subsystem consists of several logically separate subsystems + * that work together to achieve a common goal. The code in this file + * provides a limited amount of common infrastructure that can be used + * by all of those various subsystems, and helps coordinate activities + * such as shared memory initialization and startup/shutdown. + * + * This file has no compile-time or link-time dependency on any specific + * table or index access method. Shared memory for AM-specific tracking + * structures (such as an optional per-tuple tracking hash an AM may register) + * is registered as its own sibling entry in storage/subsystemlist.h, not + * sized or initialized here. Likewise, the + * set of UNDO resource managers to register comes from access/undormgrlist.h + * (mirroring the subsystemlist.h idiom): a new consumer adds itself to that + * list, and never edits this file. + * + * Shared memory initialization uses the PG_SHMEM_SUBSYSTEM pattern: + * UndoShmemCallbacks is registered in subsystemlist.h, and the framework + * calls UndoShmemRequest() and UndoShmemInit() at the appropriate times + * during postmaster startup. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/backend/access/undo/undo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undo.h" +#include "access/undolog.h" +#include "access/undormgr.h" +#include "access/undormgrs.h" +#include "access/xactundo.h" +#include "miscadmin.h" +#include "storage/ipc.h" +#include "storage/shmem.h" +#include "storage/subsystems.h" +#include "utils/memutils.h" + +/* + * UndoContext is a child of TopMemoryContext which is never reset. The only + * reason for having a separate context is to make it easier to spot leaks or + * excessive memory utilization related to undo operations. + */ +MemoryContext UndoContext = NULL; + +static void AtProcExit_Undo(int code, Datum arg); +static void UndoShmemRequest_internal(void *arg); +static void UndoShmemInit_internal(void *arg); +static void UndoShmemAttach_internal(void *arg); +static void RegisterUndoRmgrs(void); + +/* + * ShmemCallbacks for the UNDO subsystem. + * + * Registered via PG_SHMEM_SUBSYSTEM(UndoShmemCallbacks) in subsystemlist.h. + * + * init_fn initializes the contents of all UNDO shared memory structures. + */ +const ShmemCallbacks UndoShmemCallbacks = { + .request_fn = UndoShmemRequest_internal, + .init_fn = UndoShmemInit_internal, + .attach_fn = UndoShmemAttach_internal, +}; + +/* + * UndoShmemRequest_internal + * Register shared memory needs for UNDO subsystems. + * + * Called during the request_fn phase of postmaster startup, before shared + * memory is allocated. + */ +static void +UndoShmemRequest_internal(void *arg) +{ + /* Shared areas of the UNDO sub-modules. */ + UndoLogShmemRequest(); + XactUndoShmemRequest(); +} + +/* + * UndoShmemInit / UndoShmemInit_internal + * Initialize undo-related shared memory. + * + * Also, perform other initialization steps that need to be done very early. + * This is called once during postmaster startup via the ShmemCallbacks + * framework. + */ +static void +UndoShmemInit_internal(void *arg) +{ + UndoShmemInit(); +} + +void +UndoShmemInit(void) +{ + /* + * Initialize the undo memory context. If it already exists (crash restart + * via reset_shared()), reset it instead. + */ + if (UndoContext) + MemoryContextReset(UndoContext); + else + UndoContext = AllocSetContextCreate(TopMemoryContext, "Undo", + ALLOCSET_DEFAULT_SIZES); + + /* Now give various undo subsystems a chance to initialize. */ + UndoLogShmemInit(); + XactUndoShmemInit(); + + /* + * Initialize the UNDO resource manager dispatch table and register the + * built-in resource managers listed in access/undormgrlist.h. + */ + RegisterUndoRmgrs(); +} + +/* + * UndoShmemAttach_internal + * Re-establish per-process UNDO state in EXEC_BACKEND children. + * + * Under EXEC_BACKEND (Windows, or --enable-exec-backend builds) a child does + * not inherit the postmaster's address space, so module-scope state populated + * by UndoShmemInit() is not present and must be rebuilt here. Two distinct + * kinds of state need re-establishing: + * + * 1. Process-local pointers derived from the requested shmem areas. Every + * UNDO shmem area is requested with ShmemRequestStruct() from + * UndoShmemRequest_internal(), so the framework has already set each + * module's base pointer before calling us; the sub-modules' *ShmemInit() + * functions initialize shared contents and must not be called here. + * + * 2. The UndoRmgrs[] dispatch table and each registered AM's undo hook + * function pointers, both rebuilt by RegisterUndoRmgrs() (idempotent: + * InitUndoRmgrs() zeroes the table first). + * + * Any AM-specific shared structure handles its own EXEC_BACKEND re-attach + * via its own PG_SHMEM_SUBSYSTEM entry (see storage/subsystemlist.h). + */ +static void +UndoShmemAttach_internal(void *arg) +{ + RegisterUndoRmgrs(); +} + +/* + * RegisterUndoRmgrs + * Initialize the UNDO resource manager dispatch table and register the + * built-in resource managers listed in access/undormgrlist.h. + * + * Called from both UndoShmemInit() (postmaster/standalone) and + * UndoShmemAttach_internal() (EXEC_BACKEND children). Idempotent: + * InitUndoRmgrs() zeroes the dispatch table before the *UndoRmgrInit() calls + * re-register, so RegisterUndoRmgr()'s double-registration guard is not + * tripped on a second invocation in the same process. + * + * This function has no compile-time knowledge of which resource managers + * exist: it just expands access/undormgrlist.h through the UNDO_RMGR_INIT + * macro. A new UNDO-writing consumer (a new index AM, table AM, or other + * subsystem) adds one line to that list; this function and the rest of + * undo.c are never modified. + */ +static void +RegisterUndoRmgrs(void) +{ + /* + * Initialize the UNDO resource manager dispatch table. + */ + InitUndoRmgrs(); + + /* + * Register every built-in resource manager listed in + * access/undormgrlist.h. + */ +#define UNDO_RMGR_INIT(initfunc) \ + initfunc(); +#include "access/undormgrlist.h" +#undef UNDO_RMGR_INIT +} + +/* + * InitializeUndo + * Per-backend initialization for the undo subsystem. + * + * Called once per backend from InitPostgres(). + */ +void +InitializeUndo(void) +{ + InitializeXactUndo(); + + on_shmem_exit(AtProcExit_Undo, 0); +} + +/* + * AtProcExit_Undo + * Shut down undo subsystems in the correct order. + * + * Higher-level stuff should be shut down first. + */ +static void +AtProcExit_Undo(int code, Datum arg) +{ + AtProcExit_XactUndo(); +} diff --git a/src/backend/access/undo/undo_xlog.c b/src/backend/access/undo/undo_xlog.c new file mode 100644 index 0000000000000..3e3a87cde67cc --- /dev/null +++ b/src/backend/access/undo/undo_xlog.c @@ -0,0 +1,878 @@ +/*------------------------------------------------------------------------- + * + * undo_xlog.c + * UNDO resource manager WAL redo routines + * + * This module implements the WAL redo callback for the RM_UNDO_ID resource + * manager. It handles replay of: + * + * XLOG_UNDO_ROLLBACK - Reapply a crash loser's rollback + * XLOG_UNDO_BATCH - Track a transaction's UNDO for recovery + * XLOG_UNDO_INFLIGHT - Track UNDO chains in flight at a checkpoint + * + * It also holds the recovery undo phase (UndoRecoveryRollBack), which rolls + * back the transactions a crash left in progress, and the code that reads + * UNDO batches back from WAL. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undo_xlog.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/transam.h" +#include "access/twophase.h" +#include "access/undo_xlog.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/xlog.h" +#include "access/xlog_internal.h" +#include "access/xloginsert.h" +#include "access/xlogreader.h" +#include "access/xlogrecovery.h" +#include "access/xlogutils.h" +#include "miscadmin.h" +#include "utils/memutils.h" + +static int undo_apply_records(TransactionId xid, const char *pos, Size len, + int *failed, StringInfo done); + +/* + * undo_redo - Replay an UNDO WAL record during crash recovery + */ +void +undo_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_UNDO_ROLLBACK: + { + xl_undo_rollback *xlrec = (xl_undo_rollback *) XLogRecGetData(record); + + (void) undo_apply_records(xlrec->xid, + XLogRecGetData(record) + SizeOfUndoRollback, + XLogRecGetDataLen(record) - SizeOfUndoRollback, + NULL, NULL); + UndoRecoveryRemoveXid(xlrec->xid); + } + break; + + case XLOG_UNDO_INFLIGHT: + { + xl_undo_inflight *xlrec = (xl_undo_inflight *) XLogRecGetData(record); + + for (uint32 i = 0; i < xlrec->nxacts; i++) + UndoRecoveryTrackInflight(&xlrec->xacts[i]); + } + break; + + case XLOG_UNDO_BATCH: + { + xl_undo_batch *xlrec = (xl_undo_batch *) XLogRecGetData(record); + + /* + * During recovery, track this batch for incomplete + * transaction detection. After redo completes, any + * transaction that wrote UNDO batches but did not commit will + * need its UNDO chain walked for rollback. + * + * The batch payload (serialized UNDO records) is part of the + * WAL record and can be re-read later via XLogReadRecord() + * during the undo phase. + */ + UndoRecoveryTrackBatch(xlrec->xid, record->ReadRecPtr, + xlrec->chain_prev, + xlrec->persistence); + + ereport(DEBUG2, + (errmsg("undo_redo: BATCH xid %u, nrecords %u, " + "total_len %u, chain_prev %X/%08X", + xlrec->xid, xlrec->nrecords, + xlrec->total_len, + LSN_FORMAT_ARGS(xlrec->chain_prev)))); + } + break; + + default: + elog(PANIC, "undo_redo: unknown op code %u", info); + } +} + +/* ---------------------------------------------------------------- + * UNDO recovery tracking + * + * During WAL redo, we track which transactions wrote UNDO batches. + * When a commit/abort record is redone, the XID is removed. + * After redo completes, remaining entries represent incomplete + * transactions that need their UNDO chains walked. + * ---------------------------------------------------------------- + */ + +/* Hash table entry for tracking incomplete UNDO transactions */ +typedef struct UndoRecoveryEntry +{ + TransactionId xid; /* hash key */ + XLogRecPtr last_batch_lsn[NUndoPersistenceLevels]; /* chain heads per + * persistence level */ + char status; /* in use */ +} UndoRecoveryEntry; + +/* Simple dynamic array for recovery tracking (used during startup only) */ +static UndoRecoveryEntry *undo_recovery_entries = NULL; +static int undo_recovery_nentries = 0; +static int undo_recovery_capacity = 0; + +/* + * Safety cap to prevent OOM during recovery. If more than this many + * distinct in-flight XIDs are found in WAL at crash time, we stop + * tracking new ones and log a warning. The untracked transactions will + * need manual resolution (e.g. via pg_resetwal or targeted UNDO apply). + * + * 1 million entries x ~40 bytes each ~ 40 MB, which is reasonable for + * a recovery-only allocation. + */ +#define UNDO_RECOVERY_MAX_ENTRIES 1048576 + +/* + * undo_recovery_entry - Find or create the recovery entry for xid + * + * Returns NULL if the entry would exceed UNDO_RECOVERY_MAX_ENTRIES. + */ +static UndoRecoveryEntry * +undo_recovery_entry(TransactionId xid) +{ + int i; + UndoRecoveryEntry *entry = NULL; + + /* Find existing entry for this XID */ + for (i = 0; i < undo_recovery_nentries; i++) + { + if (undo_recovery_entries[i].xid == xid) + return &undo_recovery_entries[i]; + } + + /* Safety cap: refuse to track more XIDs to prevent OOM */ + if (undo_recovery_nentries >= UNDO_RECOVERY_MAX_ENTRIES) + { + static bool warned = false; + + if (!warned) + { + ereport(WARNING, + (errmsg("UNDO recovery: reached maximum tracked transaction limit (%d)", + UNDO_RECOVERY_MAX_ENTRIES), + errhint("Transactions beyond this limit will not be automatically rolled back. " + "Manual intervention may be required after recovery completes."))); + warned = true; + } + return NULL; + } + + if (undo_recovery_nentries >= undo_recovery_capacity) + { + int new_capacity = (undo_recovery_capacity == 0) ? 64 : + undo_recovery_capacity * 2; + UndoRecoveryEntry *new_entries; + + /* Clamp doubling to not exceed the safety cap */ + if (new_capacity > UNDO_RECOVERY_MAX_ENTRIES) + new_capacity = UNDO_RECOVERY_MAX_ENTRIES; + + if (undo_recovery_entries == NULL) + { + new_entries = (UndoRecoveryEntry *) + palloc0(sizeof(UndoRecoveryEntry) * new_capacity); + } + else + { + new_entries = (UndoRecoveryEntry *) + repalloc(undo_recovery_entries, + sizeof(UndoRecoveryEntry) * new_capacity); + memset(&new_entries[undo_recovery_capacity], 0, + sizeof(UndoRecoveryEntry) * (new_capacity - undo_recovery_capacity)); + } + undo_recovery_entries = new_entries; + undo_recovery_capacity = new_capacity; + } + + entry = &undo_recovery_entries[undo_recovery_nentries++]; + entry->xid = xid; + for (i = 0; i < NUndoPersistenceLevels; i++) + entry->last_batch_lsn[i] = InvalidXLogRecPtr; + + return entry; +} + +/* + * UndoRecoveryTrackBatch - Record an UNDO batch during WAL redo + * + * Called from the XLOG_UNDO_BATCH redo handler to track which + * transactions have UNDO data that may need rollback. + */ +void +UndoRecoveryTrackBatch(TransactionId xid, XLogRecPtr batch_lsn, + XLogRecPtr chain_prev, + UndoPersistenceLevel persistence) +{ + UndoRecoveryEntry *entry; + + if (!TransactionIdIsValid(xid)) + return; + entry = undo_recovery_entry(xid); + if (entry == NULL) + return; + + /* Update the chain head for this persistence level */ + if (persistence < NUndoPersistenceLevels) + entry->last_batch_lsn[persistence] = batch_lsn; +} + +/* + * UndoRecoveryTrackInflight - Record a chain listed by XLOG_UNDO_INFLIGHT + * + * The checkpoint that wrote the record saw the transaction in flight. Redo + * started at its redo pointer, so batches the transaction wrote before it + * were not replayed and the entry may not exist yet. A batch already seen + * by redo is at least as new as the one listed, so keep the later head. A + * transaction whose commit or abort precedes the record was listed after it + * ended; its outcome is already in pg_xact, so it is ignored. + */ +void +UndoRecoveryTrackInflight(const xl_undo_inflight_xact *x) +{ + UndoRecoveryEntry *entry; + + if (!TransactionIdIsValid(x->xid) || + TransactionIdDidCommit(x->xid) || TransactionIdDidAbort(x->xid)) + return; + entry = undo_recovery_entry(x->xid); + if (entry == NULL) + return; + for (int i = 0; i < NUndoPersistenceLevels; i++) + { + if (XLogRecPtrIsValid(x->last_batch_lsn[i]) && + (!XLogRecPtrIsValid(entry->last_batch_lsn[i]) || + x->last_batch_lsn[i] > entry->last_batch_lsn[i])) + entry->last_batch_lsn[i] = x->last_batch_lsn[i]; + } +} + +/* + * UndoRecoveryRemoveXid - Remove an XID from recovery tracking + * + * Called when a commit or abort record is redone during recovery. + * Committed transactions don't need UNDO rollback. Aborted transactions + * that were already fully rolled back (abort record present) also don't + * need further work. + */ +void +UndoRecoveryRemoveXid(TransactionId xid) +{ + int i; + + if (!TransactionIdIsValid(xid)) + return; + + for (i = 0; i < undo_recovery_nentries; i++) + { + if (undo_recovery_entries[i].xid == xid) + { + /* Mark as removed by zeroing XID */ + undo_recovery_entries[i].xid = InvalidTransactionId; + break; + } + } +} + +/* + * UndoRecoveryNeeded - Check if there are incomplete transactions needing UNDO + * + * Returns true if any tracked transactions remain after redo is complete. + */ +static bool +UndoRecoveryNeeded(void) +{ + int i; + + for (i = 0; i < undo_recovery_nentries; i++) + { + if (TransactionIdIsValid(undo_recovery_entries[i].xid)) + return true; + } + + return false; +} + +/* + * UndoRecoveryRollingBack is true while UndoRecoveryRollBack() applies UNDO + * records. That runs after InRecovery has been cleared, but an object an + * UNDO record describes may have been recreated by redo, as during redo + * itself, so a resource manager that tests InRecovery should test this too. + */ +bool UndoRecoveryRollingBack = false; + +/* + * undo_apply_records - Apply a packed stream of UNDO records, in order + * + * Shared by the undo phase and by redo of XLOG_UNDO_ROLLBACK, so the two + * apply the same records the same way. The stream is the payload format of + * XLOG_UNDO_BATCH: each record is an UndoRecordHeader and its payload, + * urec_len bytes in all. Returns the number of records applied. If failed + * is not NULL, the number of records whose apply function reported an error + * is added to it, and if done is not NULL, each record that applied is + * appended to it. + */ +static int +undo_apply_records(TransactionId xid, const char *pos, Size len, + int *failed, StringInfo done) +{ + const char *end = pos + len; + int applied = 0; + + while ((Size) (end - pos) >= SizeOfUndoRecordHeader) + { + UndoRecordHeader header; + const UndoRmgrData *rmgr; + UndoApplyResult result; + + memcpy(&header, pos, SizeOfUndoRecordHeader); + if (header.urec_len < SizeOfUndoRecordHeader || + (Size) (end - pos) < header.urec_len || + header.urec_payload_len > header.urec_len - SizeOfUndoRecordHeader) + break; + + rmgr = GetUndoRmgr(header.urec_rmid); + if (rmgr != NULL) + { + result = rmgr->rm_undo(header.urec_rmid, header.urec_info, xid, + header.urec_reloid, + header.urec_payload_len > 0 ? + pos + SizeOfUndoRecordHeader : NULL, + header.urec_payload_len, + InvalidUndoRecPtr); + if (result == UNDO_APPLY_ERROR) + { + if (failed != NULL) + (*failed)++; + } + else + { + applied++; + if (done != NULL) + appendBinaryStringInfo(done, pos, header.urec_len); + } + } + pos += header.urec_len; + } + + return applied; +} + +/* + * undo_collect_chain - Gather one persistence level of a loser's UNDO chain + * + * Walks the chain from its newest batch back to its first and appends each + * batch's records to buf, newest batch first, so that applying buf in order + * applies every record newest first, as an online abort does. + * + * Returns false, having warned, if a batch could not be read: buf then holds + * the records of the newer batches only. + */ +static bool +undo_collect_chain(TransactionId xid, XLogRecPtr batch_lsn, StringInfo buf) +{ + int maxstarts = 64; + const char **starts = palloc_array(const char *, maxstarts); + + while (XLogRecPtrIsValid(batch_lsn)) + { + int nstarts; + UndoBatchData *batch; + XLogRecPtr next_lsn; + const char *pos; + const char *end; + + batch = UndoReadBatchFromWAL(batch_lsn); + if (batch == NULL) + { + /* + * The batch's WAL is gone or unreadable, for example because a + * restartpoint on a standby removed the segment it was in. Skip + * the rest of the transaction's rollback rather than PANIC, which + * would leave the cluster unable to start. + */ + ereport(WARNING, + (errmsg("UNDO recovery: could not read batch at %X/%08X for transaction %u; skipping rollback", + LSN_FORMAT_ARGS(batch_lsn), xid))); + pfree(starts); + return false; + } + + /* + * Find the whole records, then copy them newest first. A batch holds + * them in the order they were written. + */ + pos = batch->payload; + end = pos + batch->payload_len; + nstarts = 0; + while ((Size) (end - pos) >= SizeOfUndoRecordHeader) + { + UndoRecordHeader header; + + memcpy(&header, pos, SizeOfUndoRecordHeader); + if (header.urec_len < SizeOfUndoRecordHeader || + (Size) (end - pos) < header.urec_len) + break; + if (nstarts == maxstarts) + { + maxstarts *= 2; + starts = repalloc_array(starts, const char *, maxstarts); + } + starts[nstarts++] = pos; + pos += header.urec_len; + } + for (int i = nstarts - 1; i >= 0; i--) + { + UndoRecordHeader header; + + memcpy(&header, starts[i], SizeOfUndoRecordHeader); + appendBinaryStringInfo(buf, starts[i], header.urec_len); + } + + /* + * chain_prev must point strictly backward, or the walk would never + * end. + */ + next_lsn = batch->header.chain_prev; + if (XLogRecPtrIsValid(next_lsn) && next_lsn >= batch_lsn) + ereport(PANIC, + (errmsg("UNDO recovery: chain_prev %X/%08X >= batch_lsn %X/%08X for transaction %u; corrupt UNDO chain", + LSN_FORMAT_ARGS(next_lsn), + LSN_FORMAT_ARGS(batch_lsn), xid))); + UndoFreeBatchData(batch); + batch_lsn = next_lsn; + } + + pfree(starts); + return true; +} + +/* + * UndoRecoveryRollBack - Roll back the transactions a crash left in progress + * + * This is the undo phase. StartupXLOG() calls it once WAL can be written + * again, where resource managers may write cleanup records, just before the + * end-of-recovery checkpoint or record. For each transaction that redo found + * writing UNDO records but neither committing nor aborting, other than a + * prepared one, it applies the records, newest first, then writes an + * XLOG_UNDO_ROLLBACK record carrying the ones that applied, and forgets the + * transaction. A standby and any later recovery replay that record instead + * of reading the chain from WAL; see xl_undo_rollback. + * + * TEMP and UNLOGGED records are not applied: temporary relations do not + * survive a restart, and unlogged ones are reset to their init forks. + * + * A record whose apply function fails is left out of the rollback record, + * so the change it describes stays in place everywhere and no later replay + * retries it. The same goes for the records of a batch that could not be + * read, and of the batches older than it. The summary at the end says how + * many records could not be applied. + */ +void +UndoRecoveryRollBack(void) +{ + int total_xacts = 0; + int total_records = 0; + int failed_records = 0; + int incomplete_xacts = 0; + int pending_xacts = 0; + + if (!UndoRecoveryNeeded()) + return; + for (int i = 0; i < undo_recovery_nentries; i++) + { + if (TransactionIdIsValid(undo_recovery_entries[i].xid)) + pending_xacts++; + } + + ereport(LOG, + (errmsg("UNDO recovery: %d incomplete transaction(s) to roll back", + pending_xacts))); + + for (int i = 0; i < undo_recovery_nentries; i++) + { + UndoRecoveryEntry *entry = &undo_recovery_entries[i]; + xl_undo_rollback xlrec; + StringInfoData buf; + StringInfoData done; + int nrecords; + + if (!TransactionIdIsValid(entry->xid)) + continue; + + /* + * A prepared transaction stays prepared; COMMIT PREPARED or ROLLBACK + * PREPARED settles it later. + */ + if (RecoveryTransactionIdIsPrepared(entry->xid)) + { + ereport(LOG, + (errmsg("UNDO recovery: skipping prepared transaction %u " + "(will remain in prepared state)", + entry->xid))); + continue; + } + + total_xacts++; + ereport(LOG, + (errmsg("UNDO recovery: rolling back transaction %u", + entry->xid))); + + /* + * FILEOPS caps the UNDO a transaction can write (see + * FILEOPS_MAX_ABORT_BYTES), which keeps this record far below the WAL + * record size limit. + */ + initStringInfo(&buf); + initStringInfo(&done); + if (!undo_collect_chain(entry->xid, + entry->last_batch_lsn[UNDOPERSISTENCE_PERMANENT], + &buf)) + incomplete_xacts++; + + UndoRecoveryRollingBack = true; + nrecords = undo_apply_records(entry->xid, buf.data, buf.len, + &failed_records, &done); + UndoRecoveryRollingBack = false; + total_records += nrecords; + + /* + * Log the rollback, and only then forget the transaction. The apply + * functions made their changes durable, and a crash before this + * record reaches disk leaves the loser to be rolled back again. The + * end-of-recovery checkpoint or record flushes it. Write it even if + * nothing applied, so that a standby forgets the loser too. + */ + xlrec.xid = entry->xid; + xlrec.nrecords = nrecords; + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfUndoRollback); + XLogRegisterData(done.data, done.len); + (void) XLogInsert(RM_UNDO_ID, XLOG_UNDO_ROLLBACK); + + pfree(buf.data); + pfree(done.data); + entry->xid = InvalidTransactionId; + } + + if (failed_records > 0 || incomplete_xacts > 0) + ereport(WARNING, + (errmsg("UNDO recovery complete: %d transactions processed, " + "%d records applied, %d records could not be applied", + total_xacts, total_records, failed_records), + errdetail("The changes those records describe were not " + "rolled back."))); + else if (total_xacts > 0) + ereport(LOG, + (errmsg("UNDO recovery complete: %d transactions rolled back, " + "%d records applied", + total_xacts, total_records))); + + /* Only prepared transactions are left, and nothing tracks them now. */ + pfree(undo_recovery_entries); + undo_recovery_entries = NULL; + undo_recovery_nentries = 0; + undo_recovery_capacity = 0; +} + +/* ---------------------------------------------------------------- + * UNDO batch reading from WAL + * ---------------------------------------------------------------- + */ + +/* + * UndoReadBatchFromWAL - Read a single XLOG_UNDO_BATCH record from WAL + * + * Uses XLogReader to read the WAL record at the given LSN. + * Returns a palloc'd UndoBatchData containing the header and a copy + * of the payload. The caller must pfree via UndoFreeBatchData(). + * + * Returns NULL if the record cannot be read or is not an UNDO batch. + */ +/* + * Module-level cached XLogReader for UndoReadBatchFromWAL. + * Allocated once and reused across calls to avoid per-batch + * open/close overhead on WAL segment files during rollback. + */ +static XLogReaderState *undo_batch_reader = NULL; +static XLogReaderRoutine undo_batch_reader_routine = { + .page_read = read_local_xlog_page, + .segment_open = wal_segment_open, + .segment_close = wal_segment_close, +}; + +/* + * UndoValidateBatchLSN + * Quick check that the WAL record at batch_lsn is a valid UNDO source. + * + * Returns true if the record is a standalone RM_UNDO_ID batch. Returns + * false for any other unrecognized record type. Used by the inline UNDO + * path to avoid calling ApplyUndoChainFromWAL on a batch_lsn that points to + * the wrong record type. + */ +bool +UndoValidateBatchLSN(XLogRecPtr batch_lsn) +{ + XLogRecord *record_hdr; + char *errormsg = NULL; + uint8 rmid; + MemoryContext old_ctx; + bool result = false; + + if (!XLogRecPtrIsValid(batch_lsn)) + return false; + + /* + * Perform the entire read under TopMemoryContext. The cached reader and, + * critically, its lazily-allocated decode_buffer (xlogreader.c) are + * palloc'd in CurrentMemoryContext. Callers such as the inline-abort + * path run with a transient context that is deleted right after; + * allocating the reader state there would leave the static + * undo_batch_reader (and its decode buffer) dangling, crashing the next + * abort. TopMemoryContext makes the cache truly persistent. + */ + old_ctx = MemoryContextSwitchTo(TopMemoryContext); + + if (undo_batch_reader == NULL) + { + undo_batch_reader = XLogReaderAllocate(wal_segment_size, NULL, + &undo_batch_reader_routine, NULL); + if (undo_batch_reader == NULL) + { + MemoryContextSwitchTo(old_ctx); + return false; + } + } + + XLogBeginRead(undo_batch_reader, batch_lsn); + record_hdr = XLogReadRecord(undo_batch_reader, &errormsg); + if (record_hdr == NULL) + { + MemoryContextSwitchTo(old_ctx); + return false; + } + + rmid = XLogRecGetRmid(undo_batch_reader); + + /* Only a standalone UNDO batch is a valid inline UNDO source */ + if (rmid == RM_UNDO_ID) + result = true; + + MemoryContextSwitchTo(old_ctx); + return result; +} + +UndoBatchData * +UndoReadBatchFromWAL(XLogRecPtr batch_lsn) +{ + XLogRecord *record_hdr; + char *errormsg = NULL; + UndoBatchData *result; + xl_undo_batch *xlrec; + char *record_data; + Size record_len; + Size payload_offset; + + if (!XLogRecPtrIsValid(batch_lsn)) + return NULL; + + /* + * Safety check: verify the WAL segment containing this LSN has not been + * recycled by a checkpoint. If the LSN is behind the current redo + * pointer and the segment file doesn't exist, reading would cause SIGBUS + * (signal 10: Bus error) or SIGSEGV. Return NULL gracefully instead. + * + * Compare against GetRedoRecPtr(); if our target is well behind the redo + * pointer AND behind the last checkpoint's redo location, the segment may + * have been recycled. + */ + { + XLogRecPtr redo_ptr = GetRedoRecPtr(); + XLogSegNo target_segno; + char path[MAXPGPATH]; + TimeLineID tli; + + if (batch_lsn < redo_ptr) + { + /* + * Determine the timeline for the segment path. + * GetWALInsertionTimeLine() is only valid once recovery is DONE + * (it asserts RECOVERY_STATE_DONE); during the crash-recovery + * undo phase we must instead use the replay timeline. + * RecoveryInProgress() distinguishes the two callers + * (crash-recovery undo phase vs. online abort). + */ + if (RecoveryInProgress()) + { + (void) GetXLogReplayRecPtr(&tli); + if (tli == 0) + tli = 1; /* no replay position yet: default to TLI 1 */ + } + else + tli = GetWALInsertionTimeLine(); + + XLByteToSeg(batch_lsn, target_segno, wal_segment_size); + XLogFilePath(path, tli, target_segno, + wal_segment_size); + + if (access(path, F_OK) != 0) + { + ereport(WARNING, + (errmsg("UNDO batch at %X/%08X: WAL segment \"%s\" no longer " + "exists (recycled by checkpoint); skipping rollback", + LSN_FORMAT_ARGS(batch_lsn), path))); + return NULL; + } + } + } + + /* + * Allocate the reader and perform the read under TopMemoryContext. Both + * the cached reader and its lazily-allocated decode_buffer (xlogreader.c) + * are palloc'd in CurrentMemoryContext; if that is a transient caller + * context (e.g. the inline-abort context) it gets deleted, leaving the + * static undo_batch_reader and its decode buffer dangling and crashing + * the next abort. We switch back to the caller's context before + * allocating the returned UndoBatchData, which the caller owns. See the + * matching comment in UndoValidateBatchLSN. + */ + { + MemoryContext read_ctx = MemoryContextSwitchTo(TopMemoryContext); + + if (undo_batch_reader == NULL) + { + undo_batch_reader = XLogReaderAllocate(wal_segment_size, NULL, + &undo_batch_reader_routine, NULL); + if (undo_batch_reader == NULL) + { + MemoryContextSwitchTo(read_ctx); + ereport(WARNING, + (errmsg("could not allocate XLogReader for UNDO batch read"))); + return NULL; + } + } + + /* Position the reader at the target LSN, then read */ + XLogBeginRead(undo_batch_reader, batch_lsn); + record_hdr = XLogReadRecord(undo_batch_reader, &errormsg); + MemoryContextSwitchTo(read_ctx); + } + if (record_hdr == NULL) + { + if (errormsg) + ereport(WARNING, + (errmsg("could not read WAL record at %X/%08X: %s", + LSN_FORMAT_ARGS(batch_lsn), errormsg))); + return NULL; + } + + /* + * Determine record format: a standalone XLOG_UNDO_BATCH record carrying + * the UNDO payload. + */ + record_data = XLogRecGetData(undo_batch_reader); + record_len = XLogRecGetDataLen(undo_batch_reader); + + if (XLogRecGetRmid(undo_batch_reader) == RM_UNDO_ID && + (XLogRecGetInfo(undo_batch_reader) & ~XLR_INFO_MASK) == XLOG_UNDO_BATCH) + { + /* Standalone XLOG_UNDO_BATCH record */ + if (record_len < SizeOfUndoBatch) + { + ereport(WARNING, + (errmsg("UNDO batch record at %X/%08X too short: %zu bytes", + LSN_FORMAT_ARGS(batch_lsn), record_len))); + return NULL; + } + + xlrec = (xl_undo_batch *) record_data; + payload_offset = SizeOfUndoBatch; + } + else + { + ereport(WARNING, + (errmsg("WAL record at %X/%08X is not an UNDO batch (rmid=%u, info=0x%02x)", + LSN_FORMAT_ARGS(batch_lsn), + XLogRecGetRmid(undo_batch_reader), + XLogRecGetInfo(undo_batch_reader) & ~XLR_INFO_MASK))); + return NULL; + } + + /* + * Validate that the claimed payload length fits within the WAL record. A + * mismatch here means the WAL segment was recycled and overwritten (the + * LSN now points to a different record), or the record is corrupt. + * Without this check, the memcpy below would read past the XLogReader's + * internal buffer, potentially accessing unmapped memory + * (SIGBUS/SIGSEGV). + */ + if (payload_offset + (Size) xlrec->total_len > record_len) + { + ereport(WARNING, + (errmsg("UNDO batch at %X/%08X: payload length %u exceeds " + "record data (offset %zu, record_len %zu)", + LSN_FORMAT_ARGS(batch_lsn), + xlrec->total_len, payload_offset, record_len))); + return NULL; + } + + /* + * Allocate UndoBatchData. We use palloc (CurrentMemoryContext) because + * this structure is only needed until ApplyUndoChainFromWAL processes the + * batch. We intentionally do NOT pfree in UndoFreeBatchData() because + * calling pfree on BumpContext memory would ERROR. The memory will be + * reclaimed when the current memory context is reset. + */ + result = (UndoBatchData *) palloc(sizeof(UndoBatchData)); + memcpy(&result->header, xlrec, SizeOfUndoBatch); + result->payload_len = (Size) xlrec->total_len; + if (result->payload_len > 0) + { + result->payload = (char *) palloc(result->payload_len); + memcpy(result->payload, record_data + payload_offset, + result->payload_len); + } + else + { + result->payload = NULL; + } + + /* Do not free reader -- it is cached for reuse. */ + return result; +} + +/* + * UndoFreeBatchData - Release a UndoBatchData structure + * + * This is a no-op function. We don't actually pfree the batch or payload + * because they were allocated with palloc() from CurrentMemoryContext, which + * may be a BumpContext. Calling pfree on BumpContext memory would ERROR. + * The memory will be automatically reclaimed when the current memory context + * is reset (e.g., at end of query, transaction, or subtransaction). + * + * This function exists to maintain API compatibility and to serve as a + * clear marker in the code where batch data is no longer needed. + */ +void +UndoFreeBatchData(UndoBatchData *batch) +{ + /* Intentionally empty - memory reclaimed by context reset */ + (void) batch; +} diff --git a/src/backend/access/undo/undoapply.c b/src/backend/access/undo/undoapply.c new file mode 100644 index 0000000000000..772ec11431ca2 --- /dev/null +++ b/src/backend/access/undo/undoapply.c @@ -0,0 +1,312 @@ +/*------------------------------------------------------------------------- + * + * undoapply.c + * Generic UNDO record application during transaction rollback + * + * When a transaction aborts, this module walks the UNDO chain backward + * from the most recent record to the first. For each record, it + * dispatches to the appropriate resource manager's rm_undo callback + * based on the urec_rmid field in the record header. + * + * This module is AM-agnostic: it contains no AM- or subsystem-specific + * code. All such UNDO application logic lives in the respective resource + * managers, each registered via RegisterUndoRmgr() (see access/undormgr.h). + * + * The dispatch pattern is analogous to WAL resource managers: each RM + * registers its callbacks via RegisterUndoRmgr(), and this module + * routes UNDO records to the correct handler. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undoapply.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/undo_xlog.h" +#include "miscadmin.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" + +/* + * ApplyOneUndoRecord - Apply a single UNDO record via RM dispatch + * + * UNDO records carry no "already applied" marker, so a record can be applied + * more than once; each resource manager's rm_undo callback must leave alone + * an object that no longer shows the record's effect. + * + * Returns true if successfully applied, false if skipped. + */ +static bool +ApplyOneUndoRecord(UndoRecordHeader *header, char *payload, + UndoRecPtr urec_ptr) +{ + const UndoRmgrData *rmgr; + UndoApplyResult result; + + /* + * Look up the resource manager for this record. + */ + rmgr = GetUndoRmgr(header->urec_rmid); + if (rmgr == NULL) + { + ereport(WARNING, + (errmsg("UNDO rollback: unknown RM ID %u for record at %llu, skipping", + header->urec_rmid, + (unsigned long long) urec_ptr))); + return false; + } + + /* + * Dispatch to the RM's undo-apply callback, which does all of the + * consumer-specific work (see UndoRmgrApplyFunc). + */ + result = rmgr->rm_undo(header->urec_rmid, + header->urec_info, + header->urec_xid, + header->urec_reloid, + payload, + header->urec_payload_len, + urec_ptr); + + if (result == UNDO_APPLY_SUCCESS) + { + ereport(DEBUG2, + (errmsg("UNDO rollback: applied %s record at %llu", + rmgr->rm_name, + (unsigned long long) urec_ptr))); + return true; + } + else if (result == UNDO_APPLY_SKIPPED) + { + ereport(DEBUG2, + (errmsg("UNDO rollback: skipped %s record at %llu", + rmgr->rm_name, + (unsigned long long) urec_ptr))); + return false; + } + else + { + ereport(WARNING, + (errmsg("UNDO rollback: error applying %s record at %llu", + rmgr->rm_name, + (unsigned long long) urec_ptr))); + return false; + } +} + +/* + * ApplyUndoChainFromWAL - Walk and apply an UNDO chain from WAL + * + * Reads XLOG_UNDO_BATCH WAL records via UndoReadBatchFromWAL() and + * iterates through the serialized records within each batch. + * + * The chain is walked backward via the xl_undo_batch.chain_prev LSN + * from the most recent batch to the first. Within each batch, records + * are applied in reverse order (newest to oldest) as required by + * ARIES-style rollback semantics. This is achieved by first scanning + * forward through the serialized records to collect their start offsets, + * then iterating the collected offsets in reverse to apply each record. + */ +bool +ApplyUndoChainFromWAL(XLogRecPtr last_batch_lsn) +{ + return ApplyUndoChainFromWALBounded(last_batch_lsn, InvalidXLogRecPtr); +} + +/* + * ApplyUndoChainFromWALBounded - Apply an UNDO chain, stopping at a boundary + * + * Identical to ApplyUndoChainFromWAL() except that the chain walk halts before + * processing any batch whose LSN is at or below stop_at_lsn. This applies only + * the batches strictly newer than stop_at_lsn, which is what subtransaction + * abort needs: stop_at_lsn is the parent's saved chain head, so the parent's + * (and earlier subtransactions') batches are left intact while the aborting + * subtransaction's batches are reverted. + * + * Pass InvalidXLogRecPtr as stop_at_lsn to walk the entire chain (top-level + * rollback / recovery semantics). + */ +bool +ApplyUndoChainFromWALBounded(XLogRecPtr last_batch_lsn, XLogRecPtr stop_at_lsn) +{ + XLogRecPtr batch_lsn; + int records_applied = 0; + int records_skipped = 0; + int batches_processed = 0; + + if (!XLogRecPtrIsValid(last_batch_lsn)) + return false; + + ereport(DEBUG1, + (errmsg("applying UNDO chain from WAL starting at %X/%08X " + "(stop boundary %X/%08X)", + LSN_FORMAT_ARGS(last_batch_lsn), + LSN_FORMAT_ARGS(stop_at_lsn)))); + + batch_lsn = last_batch_lsn; + + while (XLogRecPtrIsValid(batch_lsn)) + { + UndoBatchData *batch; + char *pos; + char *end; + + /* + * Bounded walk: stop once we reach a batch that belongs to the parent + * (or an earlier subtransaction). chain_prev LSNs decrease strictly + * as we walk backward, so the first batch at or below the boundary + * marks the start of the region we must preserve. + */ + if (XLogRecPtrIsValid(stop_at_lsn) && batch_lsn <= stop_at_lsn) + break; + + INJECTION_POINT("undo-apply-before-batch", NULL); + + batch = UndoReadBatchFromWAL(batch_lsn); + if (batch == NULL) + { + ereport(WARNING, + (errmsg("UNDO rollback: could not read batch at %X/%08X, " + "stopping chain walk", + LSN_FORMAT_ARGS(batch_lsn)))); + break; + } + + batches_processed++; + + /* + * Walk through records within this batch in reverse order. + * + * ARIES requires that UNDO records within a batch be applied + * newest-first (reverse of serialization order). We first scan + * forward to collect pointers to each record start, then iterate the + * collected pointers in reverse to apply them. + */ + pos = batch->payload; + end = pos + batch->payload_len; + + { + char **record_starts; + int nrecords_in_batch = 0; + int max_records = 1024; + int i; + MemoryContext batchctx; + MemoryContext oldctx; + + /* + * record_starts must grow to hold EVERY record in the batch -- + * silently truncating rollback is a data-integrity bug (the + * dropped records are never applied, so a partially-applied + * in-place UPDATE/DELETE is left on the page). We therefore + * repalloc() to double the array whenever it fills. + * + * The caller's CurrentMemoryContext may be a BumpContext + * (executor abort path), which supports neither repalloc() nor + * pfree(). So we do all growth in a private AllocSet child + * context and delete it whole at the end of the batch -- + * context-agnostic, leak-free, and unbounded. + */ + batchctx = AllocSetContextCreate(CurrentMemoryContext, + "UNDO batch record_starts", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(batchctx); + + record_starts = (char **) palloc(max_records * sizeof(char *)); + + /* First pass: collect record start pointers by scanning forward */ + while (pos < end) + { + UndoRecordHeader hdr; + + if ((Size) (end - pos) < SizeOfUndoRecordHeader) + { + ereport(WARNING, + (errmsg("UNDO rollback: truncated record in batch at %X/%08X", + LSN_FORMAT_ARGS(batch_lsn)))); + break; + } + + memcpy(&hdr, pos, SizeOfUndoRecordHeader); + + if (hdr.urec_len < SizeOfUndoRecordHeader || + (Size) (end - pos) < hdr.urec_len || + hdr.urec_payload_len > hdr.urec_len - SizeOfUndoRecordHeader) + { + ereport(WARNING, + (errmsg("UNDO rollback: invalid record size %u in batch at %X/%08X", + hdr.urec_len, LSN_FORMAT_ARGS(batch_lsn)))); + break; + } + + /* + * Grow the array (double it) rather than truncate the + * rollback + */ + if (nrecords_in_batch >= max_records) + { + max_records *= 2; + record_starts = (char **) repalloc(record_starts, + max_records * sizeof(char *)); + } + + record_starts[nrecords_in_batch++] = pos; + pos += hdr.urec_len; + } + + /* + * Second pass: apply records in reverse order (newest first). + * Even if there was a scan error, apply whatever records we + * successfully collected. + */ + for (i = nrecords_in_batch - 1; i >= 0; i--) + { + UndoRecordHeader header; + char *payload = NULL; + + memcpy(&header, record_starts[i], SizeOfUndoRecordHeader); + + if (header.urec_payload_len > 0) + payload = record_starts[i] + SizeOfUndoRecordHeader; + + if (ApplyOneUndoRecord(&header, payload, InvalidUndoRecPtr)) + records_applied++; + else + records_skipped++; + } + + MemoryContextSwitchTo(oldctx); + MemoryContextDelete(batchctx); + } + + INJECTION_POINT("undo-apply-after-batch", NULL); + + /* Follow chain to previous batch */ + batch_lsn = batch->header.chain_prev; + UndoFreeBatchData(batch); + } + + /* Report results */ + if (records_skipped > 0) + { + ereport(WARNING, + (errmsg("UNDO rollback from WAL: %d batches, %d records applied, " + "%d skipped", + batches_processed, records_applied, records_skipped))); + } + else + { + ereport(DEBUG1, + (errmsg("UNDO rollback from WAL complete: %d batches, " + "%d records applied", + batches_processed, records_applied))); + } + + return (batches_processed > 0); +} diff --git a/src/backend/access/undo/undoinsert.c b/src/backend/access/undo/undoinsert.c new file mode 100644 index 0000000000000..1c7a728a463a7 --- /dev/null +++ b/src/backend/access/undo/undoinsert.c @@ -0,0 +1,127 @@ +/*------------------------------------------------------------------------- + * + * undoinsert.c + * UNDO record batch insertion operations + * + * This file implements batch insertion of UNDO records into the WAL + * stream. Records are accumulated in an UndoRecordSet and then + * written as a single XLOG_UNDO_BATCH WAL record. + * + * UNDO-IN-WAL ARCHITECTURE + * ------------------------ + * All UNDO record data flows through the standard WAL pipeline: + * UndoRecordSetInsert() -> XLogBeginInsert() + * -> XLogRegisterData(batch_header) + * -> XLogRegisterData(uset->buffer, uset->buffer_size) + * -> XLogInsert(RM_UNDO_ID, XLOG_UNDO_BATCH) + * + * Coalescing: The existing UndoRecordSet mechanism batches records. + * This batch becomes one WAL record. A 1000-row INSERT produces ~1 + * WAL record containing 1000 UNDO records. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undoinsert.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undo_xlog.h" +#include "access/xloginsert.h" +#include "access/xlog.h" +#include "utils/injection_point.h" + +/* + * UndoRecordSetInsert - Insert accumulated UNDO records into WAL + * + * This function writes all UNDO records in the set as a single + * XLOG_UNDO_BATCH WAL record. The batch payload is the serialized + * content of uset->buffer (concatenated UndoRecordHeader+payload). + * + * Returns the (legacy) UndoRecPtr for backward compatibility. + * The actual record location is the XLogRecPtr stored in + * uset->last_batch_lsn after this call. + */ +UndoRecPtr +UndoRecordSetInsert(UndoRecordSet *uset) +{ + xl_undo_batch xlrec; + XLogRecPtr batch_lsn; + Oid primary_reloid = InvalidOid; + + if (uset == NULL || uset->nrecords == 0) + return InvalidUndoRecPtr; + + /* + * Extract the primary relation OID from the first record in the batch as + * an optimization hint. Most batches contain records for a single + * relation. + */ + if (uset->buffer_size >= SizeOfUndoRecordHeader) + { + UndoRecordHeader *first_hdr = (UndoRecordHeader *) uset->buffer; + + primary_reloid = first_hdr->urec_reloid; + } + + /* Build the batch header */ + xlrec.xid = uset->xid; + xlrec.chain_prev = uset->last_batch_lsn; + xlrec.nrecords = (uint32) uset->nrecords; + xlrec.total_len = (uint32) uset->buffer_size; + xlrec.primary_reloid = primary_reloid; + xlrec.persistence = uset->persistence; + + /* + * Write the UNDO batch as a single WAL record. + * + * XLogRegisterData has no size limit on main data (tracked as uint64 in + * xloginsert.c), so even a 256KB batch is fine. The WAL insertion lock + * will be held for the duration of the record write, which is acceptable + * for batch sizes up to a few hundred KB. + */ + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfUndoBatch); + XLogRegisterData(uset->buffer, uset->buffer_size); + + /* + * Use the CACHED variant: UndoRecordSetInsert() always runs inside the + * caller's critical section (it emits a WAL record), and + * InjectionPointRun() may palloc (dlopen the callback library, or a + * wait-mode callback allocating to suspend the backend), which is + * forbidden in a critical section. The matching INJECTION_POINT_LOAD is + * issued pre-crit in PrepareXactUndoData(); INJECTION_POINT_CACHED is + * palloc-free. + */ + INJECTION_POINT_CACHED("undo-batch-before-wal-insert", NULL); + + (void) XLogInsert(RM_UNDO_ID, XLOG_UNDO_BATCH); + + /* + * XLogInsert() returns the end+1 position of the record, but the rollback + * path must re-read the batch by its START LSN. ProcLastRecPtr holds the + * start of the record we just inserted; use it so UndoReadBatchFromWAL + * lands on this XLOG_UNDO_BATCH instead of the following record. + */ + batch_lsn = ProcLastRecPtr; + + INJECTION_POINT_CACHED("undo-batch-after-wal-insert", NULL); + + /* Update the record set's chain pointer for subsequent batches */ + uset->last_batch_lsn = batch_lsn; + + /* + * Register the batch LSN for WAL retention tracking. Only the first call + * per transaction takes effect (UndoRegisterBatchLSN is a no-op if the + * slot is already occupied), so this records the oldest batch for this + * transaction without additional bookkeeping. + */ + UndoRegisterBatchLSN(batch_lsn); + + return (UndoRecPtr) batch_lsn; +} diff --git a/src/backend/access/undo/undolog.c b/src/backend/access/undo/undolog.c new file mode 100644 index 0000000000000..d1af41a6ae6cb --- /dev/null +++ b/src/backend/access/undo/undolog.c @@ -0,0 +1,153 @@ +/*------------------------------------------------------------------------- + * + * undolog.c + * WAL retention for UNDO records + * + * See access/undolog.h. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undolog.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/twophase.h" +#include "access/undolog.h" +#include "miscadmin.h" +#include "storage/procnumber.h" +#include "storage/shmem.h" + +pg_atomic_uint64 *UndoBatchLSNs = NULL; + +/* + * UndoLogShmemRequest + * Request the per-backend slots from the shared memory framework. + * + * Called from the UNDO subsystem's request_fn. The framework sets + * UndoBatchLSNs, in the postmaster and, under EXEC_BACKEND, in every child. + */ +void +UndoLogShmemRequest(void) +{ + ShmemRequestStruct(.name = "UNDO Log Control", + .size = mul_size(MaxBackends, sizeof(pg_atomic_uint64)), + .ptr = (void **) &UndoBatchLSNs); +} + +/* + * UndoLogShmemInit + * Initialize the slots requested by UndoLogShmemRequest(). + */ +void +UndoLogShmemInit(void) +{ + for (int i = 0; i < MaxBackends; i++) + pg_atomic_init_u64(&UndoBatchLSNs[i], + InvalidXLogRecPtr); +} + +/* + * UndoRegisterBatchLSN + * Register the first UNDO batch LSN for the current backend. + * + * Called from UndoRecordSetInsert() each time a transaction writes an UNDO + * batch. Stores the LSN in the backend's slot, which KeepLogSeg() reads + * through UndoGetOldestBatchLSN(). + * + * Only the FIRST call per transaction takes effect (we want the oldest, + * i.e., smallest, LSN). Subsequent calls for the same transaction are + * no-ops because the slot is already occupied. + */ +void +UndoRegisterBatchLSN(XLogRecPtr batch_lsn) +{ + pg_atomic_uint64 *slot; + uint64 expected; + + if (UndoBatchLSNs == NULL || !XLogRecPtrIsValid(batch_lsn)) + return; + if (MyProcNumber < 0 || MyProcNumber >= MaxBackends) + return; + + slot = &UndoBatchLSNs[MyProcNumber]; + expected = InvalidXLogRecPtr; + + /* + * Only set if the slot is currently empty. This records the first + * (oldest) batch for this transaction; later batches have larger LSNs and + * should not overwrite the stored value. + */ + (void) pg_atomic_compare_exchange_u64(slot, &expected, (uint64) batch_lsn); +} + +/* + * UndoClearBatchLSN + * Clear the per-backend UNDO batch LSN registration. + * + * Called at transaction commit or abort to release the WAL retention + * hold that was established by UndoRegisterBatchLSN(). + */ +void +UndoClearBatchLSN(void) +{ + if (UndoBatchLSNs == NULL) + return; + if (MyProcNumber < 0 || MyProcNumber >= MaxBackends) + return; + + pg_atomic_write_u64(&UndoBatchLSNs[MyProcNumber], + (uint64) InvalidXLogRecPtr); +} + +/* + * UndoGetOldestBatchLSN + * Return the oldest UNDO batch LSN that must be retained in WAL. + * + * Considers both: + * 1. Per-backend slots (in-flight transactions with UNDO data) + * 2. Prepared transactions, whose chain ROLLBACK PREPARED still reads + * + * Returns InvalidXLogRecPtr if no WAL retention is needed for UNDO. + */ +XLogRecPtr +UndoGetOldestBatchLSN(void) +{ + XLogRecPtr oldest = InvalidXLogRecPtr; + int i; + + if (UndoBatchLSNs == NULL) + return InvalidXLogRecPtr; + + /* Check per-backend slots for in-flight transactions */ + for (i = 0; i < MaxBackends; i++) + { + XLogRecPtr lsn = (XLogRecPtr) + pg_atomic_read_u64(&UndoBatchLSNs[i]); + + if (XLogRecPtrIsValid(lsn)) + { + if (!XLogRecPtrIsValid(oldest) || lsn < oldest) + oldest = lsn; + } + } + + /* + * Check prepared (2PC) transactions. A xact can sit PREPARED + * indefinitely; its UNDO-batch WAL must survive until ROLLBACK PREPARED + * reads it. The per-backend slot is cleared when the preparing backend + * exits, so it does not cover this. + */ + { + XLogRecPtr prep_oldest = TwoPhaseGetOldestUndoBatchLSN(); + + if (XLogRecPtrIsValid(prep_oldest) && + (!XLogRecPtrIsValid(oldest) || prep_oldest < oldest)) + oldest = prep_oldest; + } + + return oldest; +} diff --git a/src/backend/access/undo/undorecord.c b/src/backend/access/undo/undorecord.c new file mode 100644 index 0000000000000..6cde66ce2899b --- /dev/null +++ b/src/backend/access/undo/undorecord.c @@ -0,0 +1,360 @@ +/*------------------------------------------------------------------------- + * + * undorecord.c + * UNDO record assembly and serialization + * + * This file implements the AM-agnostic UNDO record format and provides + * functions for creating, serializing, and deserializing UNDO records. + * All AM-specific knowledge is kept out of this module; records carry + * opaque payloads whose interpretation is delegated to the owning RM. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undorecord.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undorecord.h" +#include "utils/memutils.h" + +/* + * Per-backend recycled memory context for UndoRecordSet. + * + * Instead of creating and destroying a MemoryContext for every + * UndoRecordSet, we recycle one context across operations within a + * transaction. This avoids the overhead of repeated + * AllocSetContextCreate/MemoryContextDelete for high-frequency + * operations (e.g., 1000-row INSERTs). The cached context is cleaned + * up at transaction end by UndoRecordSetResetCache(). + */ +static MemoryContext UndoRecordReusableContext = NULL; + +/* + * UndoRecordGetPayloadSize - Calculate size needed for an UNDO record + * + * This includes the fixed header plus the RM-specific payload, rounded up to + * MAXALIGN so that each record in a packed stream begins at a MAXALIGN'd + * offset. The writers build the header in place through an + * (UndoRecordHeader *) cast into the packed buffer; without this rounding a record whose length is + * not a multiple of MAXIMUM_ALIGNOF would leave the next record's header at a + * mis-aligned address, which is undefined behavior (and traps under + * -fsanitize=alignment / crashes on alignment-strict hardware). The padded + * size is stored in urec_len, so readers advance by the same stride; the true + * payload length travels separately in urec_payload_len, so the padding never + * affects payload extraction. + */ +Size +UndoRecordGetPayloadSize(Size payload_len) +{ + return MAXALIGN(SizeOfUndoRecordHeader + payload_len); +} + +/* + * UndoRecordSetCreate - Create a new UNDO record set + * + * A record set accumulates multiple UNDO records before writing them + * to the UNDO log in a batch. This improves performance by reducing + * I/O operations. + */ +UndoRecordSet * +UndoRecordSetCreate(TransactionId xid) +{ + UndoRecordSet *uset; + MemoryContext mctx; + MemoryContext parent; + + /* + * Use TopTransactionContext as the parent so the record set survives + * across SPI statement boundaries. When called from PL/pgSQL DO blocks, + * CurrentMemoryContext is the executor's per-query context + * (es_query_cxt), which is destroyed in FreeExecutorState() after each + * SPI_execute call. Since xactundo.c stores the uset pointer in the + * static XactUndo.record_set[] and reuses it across multiple statements + * within a transaction, the context must outlive any single query. + * TopTransactionContext is ideal: it survives until transaction + * commit/abort, and AtAbort cleanup will free the uset via + * UndoRecordSetFree(). + */ + parent = TopTransactionContext; + + /* + * Reuse a previously recycled memory context if available. This avoids + * the overhead of AllocSetContextCreate/MemoryContextDelete for every + * UndoRecordSet within a transaction. MemoryContextReset clears all + * allocations but keeps the context's memory blocks for reuse. + */ + if (UndoRecordReusableContext != NULL) + { + mctx = UndoRecordReusableContext; + UndoRecordReusableContext = NULL; /* take ownership */ + MemoryContextReset(mctx); + MemoryContextSetParent(mctx, parent); + } + else + { + mctx = AllocSetContextCreate(parent, + "UNDO record set", + ALLOCSET_DEFAULT_SIZES); + } + + /* + * Allocate everything in the uset's memory context using direct + * MemoryContextAlloc to avoid MemoryContextSwitchTo overhead. + */ + uset = (UndoRecordSet *) MemoryContextAllocZero(mctx, sizeof(UndoRecordSet)); + uset->xid = xid; + uset->persistence = UNDOPERSISTENCE_PERMANENT; + + /* + * Allocate initial buffer. 512 bytes holds a typical UNDO record; the + * buffer grows as needed via UndoRecordEnsureCapacity. + */ + uset->buffer_capacity = 512; + uset->buffer = (char *) MemoryContextAlloc(mctx, uset->buffer_capacity); + uset->buffer_size = 0; + + uset->last_batch_lsn = InvalidXLogRecPtr; + uset->mctx = mctx; + + return uset; +} + +/* + * UndoRecordSetFree - Free an UNDO record set + * + * Recycles the memory context for later reuse if possible, otherwise + * destroys it. We keep at most one recycled context to bound memory. + */ +void +UndoRecordSetFree(UndoRecordSet *uset) +{ + MemoryContext mctx; + + if (uset == NULL || uset->mctx == NULL) + return; + + mctx = uset->mctx; + + if (UndoRecordReusableContext == NULL) + { + /* + * Recycle this context for the next UndoRecordSetCreate call. + * + * Re-parent to TopMemoryContext so the cached context is not + * destroyed if its original parent is cleaned up before + * UndoRecordSetResetCache() runs. This can happen when the UNDO + * record set was created inside an SPI execution context (e.g., DO $$ + * ... $$ blocks): SPI_finish() deletes its procCxt/execCxt, which + * would recursively destroy this child context, leaving + * UndoRecordReusableContext as a dangling pointer. + * UndoRecordSetCreate() will re-parent it to the caller's + * CurrentMemoryContext on reuse. + */ + MemoryContextSetParent(mctx, TopMemoryContext); + UndoRecordReusableContext = mctx; + } + else + { + /* Already have one recycled context; destroy this one */ + MemoryContextDelete(mctx); + } +} + +/* + * UndoRecordEnsureCapacity - Ensure the uset buffer can hold additional bytes + * + * Grows the buffer (using the uset's memory context) if needed. + * Avoids MemoryContextSwitchTo overhead by using MemoryContextAlloc directly. + */ +static void +UndoRecordEnsureCapacity(UndoRecordSet *uset, Size additional) +{ + if (uset->buffer_size + additional > uset->buffer_capacity) + { + Size new_capacity = uset->buffer_capacity * 2; + char *newbuf; + + while (new_capacity < uset->buffer_size + additional) + new_capacity *= 2; + + newbuf = (char *) MemoryContextAlloc(uset->mctx, new_capacity); + if (uset->buffer_size > 0) + memcpy(newbuf, uset->buffer, uset->buffer_size); + pfree(uset->buffer); + uset->buffer = newbuf; + uset->buffer_capacity = new_capacity; + } +} + +/* + * UndoRecordSetReset - Reset a record set for reuse + * + * Resets the buffer position and record count without freeing the memory + * context or reallocating the buffer. This is much cheaper than + * UndoRecordSetCreate/Free (~5 cycles vs ~300 cycles) because it avoids + * MemoryContextReset/AllocSetContextCreate overhead entirely. + */ +void +UndoRecordSetReset(UndoRecordSet *uset) +{ + if (uset == NULL) + return; + + uset->buffer_size = 0; + uset->nrecords = 0; +} + +/* + * UndoRecordSetResetCache - Release the recycled memory context. + * + * Called at transaction end (commit or abort) to ensure the cached + * context does not outlive the transaction. + */ +void +UndoRecordSetResetCache(void) +{ + if (UndoRecordReusableContext != NULL) + { + MemoryContextDelete(UndoRecordReusableContext); + UndoRecordReusableContext = NULL; + } +} + +/* + * UndoRecordAddPayload - Add an UNDO record with opaque payload to the set + * + * This is the main API for adding UNDO records. The caller provides an + * RM ID, RM-specific info flags, a relation OID, and an opaque payload. + * The payload's interpretation is entirely RM-specific. + */ +void +UndoRecordAddPayload(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *payload, + Size payload_len) +{ + UndoRecordHeader *header; + Size record_size; + char *dest; + + if (uset == NULL) + elog(ERROR, "cannot add UNDO record to NULL set"); + + record_size = UndoRecordGetPayloadSize(payload_len); + + /* Expand buffer if needed (allocate in the uset's memory context) */ + UndoRecordEnsureCapacity(uset, record_size); + + /* + * Build the header directly in the buffer, avoiding a separate stack + * variable, memset, and memcpy. We zero the header in-place to avoid + * uninitialized padding bytes in the on-disk format. + */ + dest = uset->buffer + uset->buffer_size; + header = (UndoRecordHeader *) dest; + memset(header, 0, SizeOfUndoRecordHeader); + header->urec_rmid = rmid; + header->urec_flags = UNDO_INFO_XID_VALID; + if (payload_len > 0) + header->urec_flags |= UNDO_INFO_HAS_PAYLOAD; + header->urec_info = info; + header->urec_len = (uint32) record_size; + header->urec_xid = uset->xid; + header->urec_reloid = reloid; + header->urec_payload_len = (uint32) payload_len; + + /* Copy payload directly after header */ + if (payload_len > 0 && payload != NULL) + memcpy(dest + SizeOfUndoRecordHeader, payload, payload_len); + + /* + * Zero the alignment padding between the payload and the next record so + * the serialized stream is deterministic (the padding is skipped by + * readers, which advance by urec_len and read urec_payload_len bytes of + * payload, but deterministic bytes keep WAL consistency checking happy). + */ + if (record_size > SizeOfUndoRecordHeader + payload_len) + memset(dest + SizeOfUndoRecordHeader + payload_len, 0, + record_size - SizeOfUndoRecordHeader - payload_len); + + uset->buffer_size += record_size; + uset->nrecords++; +} + +/* + * UndoRecordAddPayloadParts - Add an UNDO record with scatter-gather payload + * + * Like UndoRecordAddPayload, but takes the payload as two parts that are + * concatenated directly into the uset buffer. This avoids allocating an + * intermediate payload buffer when the caller has the data in separate + * pieces (e.g., a fixed header struct + variable-length tuple data). + */ +void +UndoRecordAddPayloadParts(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *part1, + Size part1_len, + const char *part2, + Size part2_len) +{ + UndoRecordHeader *header; + Size payload_len = part1_len + part2_len; + Size record_size; + char *dest; + + if (uset == NULL) + elog(ERROR, "cannot add UNDO record to NULL set"); + + record_size = UndoRecordGetPayloadSize(payload_len); + + UndoRecordEnsureCapacity(uset, record_size); + + /* Build header directly in the buffer */ + dest = uset->buffer + uset->buffer_size; + header = (UndoRecordHeader *) dest; + memset(header, 0, SizeOfUndoRecordHeader); + header->urec_rmid = rmid; + header->urec_flags = UNDO_INFO_XID_VALID; + if (payload_len > 0) + header->urec_flags |= UNDO_INFO_HAS_PAYLOAD; + header->urec_info = info; + header->urec_len = (uint32) record_size; + header->urec_xid = uset->xid; + header->urec_reloid = reloid; + header->urec_payload_len = (uint32) payload_len; + + /* Copy payload parts directly after header */ + dest += SizeOfUndoRecordHeader; + if (part1_len > 0 && part1 != NULL) + { + memcpy(dest, part1, part1_len); + dest += part1_len; + } + if (part2_len > 0 && part2 != NULL) + { + memcpy(dest, part2, part2_len); + dest += part2_len; + } + + /* + * Zero the alignment padding between the payload and the next record (see + * UndoRecordAddPayload); dest now points just past the payload. + */ + { + char *rec_start = uset->buffer + uset->buffer_size; + + if ((Size) (dest - rec_start) < record_size) + memset(dest, 0, record_size - (Size) (dest - rec_start)); + } + + uset->buffer_size += record_size; + uset->nrecords++; +} diff --git a/src/backend/access/undo/undormgr.c b/src/backend/access/undo/undormgr.c new file mode 100644 index 0000000000000..851ef61ae8d4f --- /dev/null +++ b/src/backend/access/undo/undormgr.c @@ -0,0 +1,70 @@ +/*------------------------------------------------------------------------- + * + * undormgr.c + * UNDO resource manager registration and dispatch + * + * This module manages the registration table for UNDO resource managers. + * Each access method or subsystem that writes UNDO records registers + * its callbacks here. The generic UNDO infrastructure dispatches to + * the appropriate callback based on the urec_rmid in the record header. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undormgr.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undormgr.h" + +/* Global registration table, indexed by RM ID */ +const UndoRmgrData *UndoRmgrs[MAX_UNDO_RMGRS]; + +/* + * RegisterUndoRmgr - Register an UNDO resource manager + * + * Called by each AM/subsystem during initialization to register its + * UNDO apply and describe callbacks. + */ +void +RegisterUndoRmgr(uint8 rmid, const UndoRmgrData *rmgr) +{ + if (rmid == UNDO_RMID_INVALID) + elog(ERROR, "cannot register UNDO RM with invalid ID 0"); + + if (UndoRmgrs[rmid] != NULL) + elog(ERROR, "UNDO RM ID %u already registered as \"%s\"", + rmid, UndoRmgrs[rmid]->rm_name); + + if (rmgr->rm_undo == NULL) + elog(ERROR, "UNDO RM \"%s\" must provide an rm_undo callback", + rmgr->rm_name ? rmgr->rm_name : "(null)"); + + UndoRmgrs[rmid] = rmgr; +} + +/* + * GetUndoRmgr - Look up an UNDO resource manager by ID + * + * Returns the registration entry, or NULL if not registered. + */ +const UndoRmgrData * +GetUndoRmgr(uint8 rmid) +{ + return UndoRmgrs[rmid]; +} + +/* + * InitUndoRmgrs - Initialize the UNDO resource manager table + * + * Called during postmaster startup. Individual RMs register themselves + * via RegisterUndoRmgr() during their initialization. + */ +void +InitUndoRmgrs(void) +{ + MemSet(UndoRmgrs, 0, sizeof(UndoRmgrs)); +} diff --git a/src/backend/access/undo/xactundo.c b/src/backend/access/undo/xactundo.c new file mode 100644 index 0000000000000..3c7e22b499b15 --- /dev/null +++ b/src/backend/access/undo/xactundo.c @@ -0,0 +1,1149 @@ +/*------------------------------------------------------------------------- + * + * xactundo.c + * Management of undo record sets for transactions + * + * Undo records that need to be applied after a transaction or + * subtransaction abort should be inserted using the functions defined + * in this file; every consumer that wants undo for post-abort cleanup should + * use these interfaces. + * + * The reason for this design is that we want to pack all of the undo + * records for a single transaction into one place, regardless of the + * AM which generated them. That way, we can apply the undo actions + * which pertain to that transaction in the correct order; namely, + * backwards as compared with the order in which the records were + * generated. + * + * We may use up to three undo record sets per transaction, one per + * persistence level (permanent, unlogged, temporary). We assume that + * it's OK to apply the undo records for each persistence level + * independently of the others. This is safe since the modifications + * must necessarily touch disjoint sets of pages. + * + * All of a transaction's UNDO records at one persistence level go into the + * same UndoRecordSet in the order they were written, whatever objects they + * describe, and rollback applies them newest first. + * + * SUBTRANSACTION TRACKING: + * + * Subtransaction state is tracked using a dynamically-grown array allocated + * in TopMemoryContext. The array starts at INITIAL_SUBXACT_CAPACITY (64) + * slots and doubles when needed via repalloc(). The array persists across + * transactions within the same backend to avoid repeated allocation for + * steady-state workloads. It lives in TopMemoryContext rather than + * CurTransactionContext so that growing it in a SubXactCallback allocates + * nothing per subtransaction. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/backend/access/undo/xactundo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undo.h" +#include "access/undo_xlog.h" +#include "access/xlog.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/xact.h" +#include "access/xactundo.h" +#include "access/xlogdefs.h" +#include "access/xloginsert.h" +#include "access/table.h" +#include "catalog/pg_class.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/ipc.h" +#include "utils/injection_point.h" +#include "storage/lmgr.h" +#include "storage/proc.h" +#include "storage/shmem.h" +#include "storage/spin.h" +#include "utils/memutils.h" +#include "utils/rel.h" + +/* + * Initial capacity for the dynamically-grown subtransaction stack. + * Covers 99.9% of workloads without needing reallocation. The stack + * doubles when needed, so there is no artificial upper limit. + */ +#define INITIAL_SUBXACT_CAPACITY 64 + +/* Per-subtransaction backend-private undo state (array element). */ +typedef struct XactUndoSubTransactionState +{ + SubTransactionId nestingLevel; + UndoRecPtr start_location[NUndoPersistenceLevels]; + + /* + * Snapshot of the parent's last_batch_lsn at the time this subtransaction + * started. On subtransaction abort, after applying this subtransaction's + * UNDO chain, we restore XactUndo.last_batch_lsn to these saved values so + * the parent's subsequent abort (or further subtransactions) only covers + * the parent's own records and does not double-apply already-reversed + * batches. + */ + XLogRecPtr last_batch_lsn[NUndoPersistenceLevels]; +} XactUndoSubTransactionState; + +/* Backend-private undo state. */ +typedef struct XactUndoData +{ + bool has_undo; /* has this xact generated any undo? */ + int subxact_depth; /* 0 = top-level, 1+ = savepoints */ + int subxact_capacity; /* allocated slots in subxact_stack */ + + /* Dynamically-grown subtransaction stack (TopMemoryContext). */ + XactUndoSubTransactionState *subxact_stack; + + /* + * Per-persistence-level record sets. These are created lazily on first + * use and destroyed at transaction end. + */ + UndoRecordSet *record_set[NUndoPersistenceLevels]; + + /* Tracking for the most recent undo insertion per persistence level. */ + UndoRecPtr last_location[NUndoPersistenceLevels]; + + /* + * WAL-based UNDO chain heads. When UNDO records are routed through WAL + * via XLOG_UNDO_BATCH, this tracks the LSN of the most recent batch per + * persistence level. Used for rollback chain walking. + */ + XLogRecPtr last_batch_lsn[NUndoPersistenceLevels]; +} XactUndoData; + +static XactUndoData XactUndo; +static bool subxact_callback_registered = false; + +/* + * Per-backend copy of the current transaction's UNDO chain heads, for the + * checkpointer (see LogXactUndoInflight). Only the owning backend writes its + * slot, under the slot's spinlock; xid is invalid when it has none. + */ +typedef struct XactUndoInflightSlot +{ + slock_t mutex; + xl_undo_inflight_xact chain; +} XactUndoInflightSlot; + +static XactUndoInflightSlot *XactUndoInflight = NULL; + +static void PublishXactUndoChain(TransactionId xid); + +/* + * Compile-time guard: xl_xact_prepare.last_batch_lsn[3] must match + * NUndoPersistenceLevels. Both headers are available here; xact.h avoids + * including undodefs.h to keep its include footprint minimal. + */ +StaticAssertDecl(NUndoPersistenceLevels == 3, + "xl_xact_prepare.last_batch_lsn array size (3) must match NUndoPersistenceLevels"); + +static void ResetXactUndo(void); +static void CollapseXactUndoSubTransactions(void); +static UndoPersistenceLevel GetUndoPersistenceLevel(char relpersistence); +static void EnsureSubxactStackCapacity(void); +static void XactUndo_SubXactCallback(SubXactEvent event, SubTransactionId mySubid, + SubTransactionId parentSubid, void *arg); + +/* Convenience macro: pointer to current subtransaction state. */ +#define CURRENT_SUBXACT() (&XactUndo.subxact_stack[XactUndo.subxact_depth]) + +/* + * XactUndoShmemSize + * How much shared memory do we need for transaction undo state? + * + * One XactUndoInflightSlot per backend. + */ +Size +XactUndoShmemSize(void) +{ + return mul_size(MaxBackends, sizeof(XactUndoInflightSlot)); +} + +/* + * XactUndoShmemRequest + * Register the in-flight chain slots with the shared memory framework. + */ +void +XactUndoShmemRequest(void) +{ + ShmemRequestStruct(.name = "UNDO In-flight Chains", + .size = XactUndoShmemSize(), + .ptr = (void **) &XactUndoInflight); +} + +/* + * XactUndoShmemInit + * Initialize shared memory for transaction undo state. + */ +void +XactUndoShmemInit(void) +{ + for (int i = 0; i < MaxBackends; i++) + { + SpinLockInit(&XactUndoInflight[i].mutex); + XactUndoInflight[i].chain.xid = InvalidTransactionId; + } +} + +/* + * PublishXactUndoChain + * Make this backend's chain heads visible to LogXactUndoInflight(). + * + * xid invalid clears the slot. Called in critical sections: no palloc. + */ +static void +PublishXactUndoChain(TransactionId xid) +{ + XactUndoInflightSlot *slot; + + if (XactUndoInflight == NULL || MyProcNumber < 0 || + MyProcNumber >= MaxBackends) + return; + slot = &XactUndoInflight[MyProcNumber]; + if (!TransactionIdIsValid(xid) && !TransactionIdIsValid(slot->chain.xid)) + return; + + SpinLockAcquire(&slot->mutex); + slot->chain.xid = xid; + for (int i = 0; i < NUndoPersistenceLevels; i++) + slot->chain.last_batch_lsn[i] = XactUndo.last_batch_lsn[i]; + SpinLockRelease(&slot->mutex); +} + +/* + * LogXactUndoInflight + * WAL-log the UNDO chains of every transaction in flight. + * + * Called by the checkpointer after the checkpoint's redo pointer has been + * determined. Redo starts at that pointer and so does not see the batches a + * transaction wrote before it; this record tells recovery about them. A + * transaction that commits or aborts after we read its slot is still listed; + * its commit or abort record, which follows this one, removes the entry + * again during redo. + */ +void +LogXactUndoInflight(void) +{ + xl_undo_inflight *rec; + uint32 n = 0; + + if (XactUndoInflight == NULL) + return; + + rec = palloc(SizeOfUndoInflight + + MaxBackends * sizeof(xl_undo_inflight_xact)); + for (int i = 0; i < MaxBackends; i++) + { + XactUndoInflightSlot *slot = &XactUndoInflight[i]; + + SpinLockAcquire(&slot->mutex); + if (TransactionIdIsValid(slot->chain.xid)) + rec->xacts[n++] = slot->chain; + SpinLockRelease(&slot->mutex); + } + + /* Unimportant, so that an idle system still skips checkpoints. */ + if (n > 0) + { + rec->nxacts = n; + XLogBeginInsert(); + XLogSetRecordFlags(XLOG_MARK_UNIMPORTANT); + XLogRegisterData(rec, SizeOfUndoInflight + + n * sizeof(xl_undo_inflight_xact)); + (void) XLogInsert(RM_UNDO_ID, XLOG_UNDO_INFLIGHT); + } + pfree(rec); +} + +/* + * InitializeXactUndo + * Per-backend initialization for transaction undo. + */ +void +InitializeXactUndo(void) +{ + /* Ensure the dynamic subxact stack is allocated */ + EnsureSubxactStackCapacity(); + + ResetXactUndo(); + + /* + * Register callback to track subtransaction lifecycle. Do this lazily on + * first transaction to ensure it's registered for the backend that will + * actually use UNDO. + */ + if (!subxact_callback_registered) + { + RegisterSubXactCallback(XactUndo_SubXactCallback, NULL); + subxact_callback_registered = true; + } +} + +/* + * GetUndoPersistenceLevel + * Map relation persistence character to UndoPersistenceLevel. + */ +static UndoPersistenceLevel +GetUndoPersistenceLevel(char relpersistence) +{ + switch (relpersistence) + { + case RELPERSISTENCE_PERMANENT: + return UNDOPERSISTENCE_PERMANENT; + case RELPERSISTENCE_UNLOGGED: + return UNDOPERSISTENCE_UNLOGGED; + case RELPERSISTENCE_TEMP: + return UNDOPERSISTENCE_TEMP; + default: + elog(ERROR, "unrecognized relpersistence: %c", relpersistence); + return UNDOPERSISTENCE_PERMANENT; /* keep compiler quiet */ + } +} + +/* + * PrepareXactUndoData + * Prepare to insert a transactional undo record. + * + * Finds or creates the appropriate per-persistence-level UndoRecordSet + * for the current transaction and adds the record to it. + * + * The API is AM-agnostic: callers pass an RM ID, RM-specific info, + * a relation OID, and an opaque payload. + * + * Returns the UndoRecPtr where the record will be inserted (or + * InvalidUndoRecPtr if undo is disabled). + */ +UndoRecPtr +PrepareXactUndoData(XactUndoContext *ctx, char persistence, + uint8 rmid, uint16 info, Oid reloid, + const char *payload, Size payload_len) +{ + int nestingLevel = GetCurrentTransactionNestLevel(); + UndoPersistenceLevel plevel = GetUndoPersistenceLevel(persistence); + UndoRecordSet *uset; + XactUndoSubTransactionState *cur; + UndoRecPtr *sub_start_location; + + /* Remember that we've done something undo-related. */ + XactUndo.has_undo = true; + + /* The current (sub)transaction must have an xid; see below. */ + (void) GetCurrentTransactionId(); + + /* + * Load the UndoRecordSetInsert() injection points into this backend's + * local cache now, while we are guaranteed not to be inside a critical + * section (PrepareXactUndoData may itself palloc). UndoRecordSetInsert() + * runs inside the caller's crit section and can only fire these via the + * palloc-free INJECTION_POINT_CACHED variant. + */ + INJECTION_POINT_LOAD("undo-batch-before-wal-insert"); + INJECTION_POINT_LOAD("undo-batch-after-wal-insert"); + + /* + * If we've entered a subtransaction deeper than what's currently tracked, + * push a new entry onto the subxact_stack. This handles the case where + * PrepareXactUndoData is called for the first time in a subtransaction + * that was started before the SubXactCallback fired (e.g., if the + * callback hadn't been registered yet when the subtransaction began). + */ + cur = CURRENT_SUBXACT(); + if (nestingLevel > (int) cur->nestingLevel) + { + int i; + + XactUndo.subxact_depth++; + EnsureSubxactStackCapacity(); + + cur = CURRENT_SUBXACT(); + cur->nestingLevel = nestingLevel; + for (i = 0; i < NUndoPersistenceLevels; ++i) + { + cur->start_location[i] = InvalidUndoRecPtr; + cur->last_batch_lsn[i] = XactUndo.last_batch_lsn[i]; + } + } + + /* + * Make sure we have an UndoRecordSet of the appropriate type open for + * this persistence level. These record sets are always associated with + * the toplevel transaction, not a subtransaction, to avoid fragmentation. + * + * The set, and so every batch it writes, is stamped with the top-level + * xid even when the first record comes from a subtransaction. Crash + * recovery tracks loser chains by that xid and forgets them at the top + * level's commit or abort record; a subxid stamp would outlive the commit + * and get a committed transaction's UNDO applied. + */ + uset = XactUndo.record_set[plevel]; + if (uset == NULL) + { + uset = UndoRecordSetCreate(GetTopTransactionId()); + XactUndo.record_set[plevel] = uset; + } + + /* Remember persistence level for InsertXactUndoData. */ + ctx->plevel = plevel; + ctx->uset = uset; + + /* Add the record to the record set using generic payload API. */ + UndoRecordAddPayload(uset, rmid, info, reloid, payload, payload_len); + + /* + * If this is the first undo for this persistence level in this + * subtransaction, record the start location. The actual UndoRecPtr is not + * known until insertion, so we use a sentinel for now and the caller will + * update it after InsertXactUndoData. + */ + sub_start_location = &cur->start_location[plevel]; + if (!UndoRecPtrIsValid(*sub_start_location)) + *sub_start_location = (UndoRecPtr) 1; /* will be set properly */ + + return InvalidUndoRecPtr; /* actual ptr assigned during insert */ +} + +/* + * InsertXactUndoData + * Insert the prepared undo data into the undo log. + * + * This performs the actual write of the accumulated records. + */ +void +InsertXactUndoData(XactUndoContext *ctx) +{ + UndoRecordSet *uset = ctx->uset; + UndoRecPtr ptr; + int saved_delay = 0; + + Assert(uset != NULL); + + /* + * Hold off a checkpoint's redo-pointer phase while the batch is inserted + * and published for LogXactUndoInflight(): a batch that precedes the redo + * pointer must be visible to the checkpointer when it reads the slots. + */ + if (MyProc != NULL) + { + saved_delay = MyProc->delayChkptFlags; + MyProc->delayChkptFlags |= DELAY_CHKPT_START; + } + + ptr = UndoRecordSetInsert(uset); + if (UndoRecPtrIsValid(ptr)) + { + XactUndoSubTransactionState *cur = CURRENT_SUBXACT(); + + XactUndo.last_location[ctx->plevel] = ptr; + + /* + * Track the WAL LSN of the most recent UNDO batch for this + * persistence level. This is used during rollback to walk the UNDO + * chain backward through WAL. + * + * The pointer must be a valid LSN; the caller must have produced an + * RM_UNDO_ID XLOG_UNDO_BATCH record via UndoRecordSetInsert. We do + * not validate the rmid here because UndoValidateBatchLSN reads from + * the on-disk WAL via read_local_xlog_page, which waits on flush and + * cannot be safely called from the WAL-insertion hot path. The + * rollback path at AtAbort_XactUndo validates before applying. + */ + Assert(XLogRecPtrIsValid(uset->last_batch_lsn)); + XactUndo.last_batch_lsn[ctx->plevel] = uset->last_batch_lsn; + PublishXactUndoChain(uset->xid); + + /* Fix up subtransaction start location if needed */ + if (cur->start_location[ctx->plevel] == (UndoRecPtr) 1) + cur->start_location[ctx->plevel] = ptr; + } + else + { + XactUndoSubTransactionState *cur = CURRENT_SUBXACT(); + + /* + * The batch was empty or elided, so no record pointer exists to fix + * up the sentinel with. Leaving the (UndoRecPtr) 1 sentinel in place + * would make UndoRecPtrIsValid() treat it as a real start location + * and leak it into subxact->parent merges and rollback chain walks; + * reset it so this level records no start location. + */ + if (cur->start_location[ctx->plevel] == (UndoRecPtr) 1) + cur->start_location[ctx->plevel] = InvalidUndoRecPtr; + } + + if (MyProc != NULL) + MyProc->delayChkptFlags = saved_delay; +} + +/* + * CleanupXactUndoInsertion + * Clean up after an undo insertion cycle. + * + * Resets the record set's buffer position and record count so it can + * accumulate more records. Does NOT free the record set -- that + * happens at transaction end (AtCommit_XactUndo / AtAbort_XactUndo). + */ +void +CleanupXactUndoInsertion(XactUndoContext *ctx) +{ + if (ctx->uset != NULL) + UndoRecordSetReset(ctx->uset); +} + +/* + * GetCurrentXactLastBatchLSN + * Get the WAL LSN of the most recent UNDO batch for a persistence level. + * + * Used during transaction abort to start the WAL-based UNDO chain walk. + */ +XLogRecPtr +GetCurrentXactLastBatchLSN(UndoPersistenceLevel plevel) +{ + return XactUndo.last_batch_lsn[plevel]; +} + +/* + * AtCommit_XactUndo + * Post-commit cleanup of the undo state. + * + * On commit, undo records are no longer needed for rollback. + * Free all record sets and reset state. + * + * The records are already in WAL, made durable by the commit record's + * flush, so nothing needs writing here. + * + * NB: This code MUST NOT FAIL, since it is run as a post-commit step. + */ +void +AtCommit_XactUndo(void) +{ + int i; + + + if (!XactUndo.has_undo) + return; + + /* + * Free all per-persistence-level record sets. + * + * We can safely call UndoRecordSetFree() during commit because we're in + * CurTransactionContext, not BumpContext (which is only used during + * abort). The record sets are allocated in CurTransactionContext and will + * be freed when that context is destroyed at transaction end. + */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (XactUndo.record_set[i] != NULL) + { + UndoRecordSetFree(XactUndo.record_set[i]); + XactUndo.record_set[i] = NULL; + } + } + + /* Release WAL retention hold acquired in UndoRecordSetInsert(). */ + UndoClearBatchLSN(); + + ResetXactUndo(); +} + +/* + * AtPrepare_XactUndo + * End-of-PREPARE cleanup of the backend-private undo state. + * + * PREPARE TRANSACTION hands ownership of this transaction's cluster-wide UNDO + * to the 2PC state: the permanent chain-head LSN was durably saved in + * xl_xact_prepare (twophase.c gxact->undo_batch_lsn), its WAL is pinned while + * the xact stays prepared, and ROLLBACK PREPARED applies that LSN + * synchronously via ApplyUndoChainFromWAL(). + * + * The in-memory UndoRecordSets, however, live in TopTransactionContext, which + * PrepareTransaction() destroys (AtCommit_Memory) right after this. We must + * therefore drop the backend-private references to them here, or the dangling + * XactUndo.record_set[] pointers get freed at backend exit (AtProcExit_XactUndo + * -> UndoRecordSetFree on already-freed memory) and crash. We only NULL the + * pointers (via ResetXactUndo); the record sets themselves are freed by + * TopTransactionContext destruction. + * + * Unlike AtCommit_XactUndo, we do NOT call UndoClearBatchLSN(): the WAL + * retention hold must persist for as long as the transaction stays prepared. + */ +void +AtPrepare_XactUndo(void) +{ + /* + * Drop the backend-private record-set references (memory freed with + * TopTransactionContext); keep the WAL retention hold. + */ + ResetXactUndo(); +} + +/* + * AtAbort_XactUndo + * Post-abort cleanup of the undo state. + * + * Apply the transaction's UNDO chain and clean up the record sets. The + * batches are already in WAL, so a crash during the rollback leaves them for + * the recovery undo phase. + */ +void +AtAbort_XactUndo(void) +{ + int i; + bool lsn_safely_held = false; /* true once inline UNDO completed */ + + + /* Always clean up the recycled context; see AtCommit_XactUndo. */ + UndoRecordSetResetCache(); + + if (!XactUndo.has_undo) + return; + + INJECTION_POINT("undo-xact-abort-before-apply", NULL); + + /* Collapse all subtransaction state. */ + CollapseXactUndoSubTransactions(); + + /* + * UNDO application strategy: apply the transaction's cluster-wide UNDO + * chain synchronously in this backend, here at abort. + * + * The BumpContext issue (pfree crashes during abort) is avoided by: - + * Creating a temporary AllocSetContext for inline UNDO application - + * ApplyUndoChainFromWAL already avoids pfree on its allocations - + * Switching back to the abort context afterward + */ + { + XLogRecPtr perm_lsn = + XactUndo.last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; + + if (XLogRecPtrIsValid(perm_lsn)) + { + /* + * The UNDO batch records were inserted into the WAL buffers + * during this transaction but may not yet be flushed to disk. The + * inline apply path reads them back via the local XLog reader, + * which would otherwise busy-wait (pg_usleep) for the walwriter + * to flush past the batch LSN -- adding multi-second latency to + * every rollback. Flush WAL up to the end of the last record this + * backend wrote so the batch is immediately readable. We must not + * flush to GetXLogInsertRecPtr(): under concurrency that global + * insert position can point into the middle of a record another + * backend has reserved but not finished copying, so XLogFlush + * would fail with "xlog flush request is not satisfied". + * XactLastRecEnd is always a valid record boundary and, since + * this backend wrote the UNDO batch, is guaranteed to be at or + * past the batch end. This mirrors the commit-path flush in + * RecordTransactionCommit. + */ + XLogFlush(XactLastRecEnd); + + { + /* + * Apply UNDO inline. Use a dedicated AllocSetContext to + * avoid BumpContext pfree issues. + */ + MemoryContext undo_ctx; + MemoryContext old_ctx; + int saved_trans_state; + bool undo_applied = false; + bool valid_batch; + + undo_ctx = AllocSetContextCreate(TopMemoryContext, + "Inline UNDO Apply", + ALLOCSET_DEFAULT_SIZES); + old_ctx = MemoryContextSwitchTo(undo_ctx); + + /* + * Validate the batch LSN points to an actual UNDO record + * before attempting inline application. Stale LSNs from + * chain_prev tracking anomalies can point to non-UNDO WAL + * records, which would cause "not an UNDO batch" warnings. + */ + valid_batch = UndoValidateBatchLSN(perm_lsn); +#ifdef USE_INJECTION_POINTS + /* Force the invalid-batch early exit for testing. */ + if (IS_INJECTION_POINT_ATTACHED("inline-undo-invalid-batch")) + valid_batch = false; +#endif + if (!valid_batch) + { + elog(DEBUG1, "inline UNDO: last_batch_lsn %X/%08X is not " + "a valid UNDO batch", + LSN_FORMAT_ARGS(perm_lsn)); + undo_applied = false; + MemoryContextSwitchTo(old_ctx); + MemoryContextDelete(undo_ctx); + goto inline_undo_done; + } + + /* + * AbortTransaction() has already advanced the transaction + * state to TRANS_ABORT, but the relcache, locks, and resource + * owner are all still live. UNDO appliers open relations, + * which asserts IsTransactionState(); temporarily present + * TRANS_INPROGRESS for the duration of the inline apply and + * always restore the real state afterward. + */ + saved_trans_state = EnterInlineUndoApplyState(); + PG_TRY(); + { + undo_applied = ApplyUndoChainFromWAL(perm_lsn); + } + PG_CATCH(); + { + /* + * If inline UNDO throws an error, release any content + * locks the failed chain left held before unwinding. + */ + BufferLockReleaseAll(); + LeaveInlineUndoApplyState(saved_trans_state); + FlushErrorState(); + undo_applied = false; + } + PG_END_TRY(); + LeaveInlineUndoApplyState(saved_trans_state); + + MemoryContextSwitchTo(old_ctx); + MemoryContextDelete(undo_ctx); + + inline_undo_done: + if (!undo_applied) + { + /* + * Inline UNDO failed (WAL recycled, wrong record type, or + * chain walk aborted). Retain the UNDO WAL retention + * slot so the WAL segment holding our UNDO data is not + * recycled; the slot is cleared at backend exit. + * + * The error that is aborting us may have been raised + * while this backend held buffer content locks, which + * resource-owner cleanup releases only later. The + * appliers lock pages, possibly those very ones, so + * release the stragglers first. LWLockReleaseAll does + * not cover them. + */ + BufferLockReleaseAll(); + elog(WARNING, "inline UNDO failed for xid %u", + GetCurrentTransactionId()); + } + else + { + lsn_safely_held = true; /* UNDO fully applied, WAL can be + * recycled */ + ereport(DEBUG2, + (errmsg("inline UNDO applied for xid %u", + GetCurrentTransactionId()))); + } + } + } + } + + INJECTION_POINT("undo-xact-abort-after-apply", NULL); + + /* Free all per-persistence-level record sets. */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (XactUndo.record_set[i] != NULL) + { + UndoRecordSetFree(XactUndo.record_set[i]); + XactUndo.record_set[i] = NULL; + } + } + + /* + * Release the WAL retention hold if inline UNDO completed: nothing needs + * the UNDO records any more. + * + * Otherwise the slot keeps the hold, but only until this backend's next + * transaction commits or aborts, or the backend exits; both clear it. + * Nothing re-applies the chain in the meantime. The abort record is + * already written, so crash recovery does not treat the transaction as a + * loser either. + */ + if (lsn_safely_held) + UndoClearBatchLSN(); + else + elog(DEBUG1, "retaining UNDO WAL retention slot (inline UNDO failed)"); + + ResetXactUndo(); +} + +/* + * AtSubCommit_XactUndo + * Subtransaction commit: merge sub undo state into parent. + */ +void +AtSubCommit_XactUndo(int level) +{ + XactUndoSubTransactionState *cur; + XactUndoSubTransactionState *parent; + int i; + + + if (XactUndo.subxact_depth <= 0) + return; + + cur = CURRENT_SUBXACT(); + if ((int) cur->nestingLevel != level) + return; + + parent = &XactUndo.subxact_stack[XactUndo.subxact_depth - 1]; + + /* + * Merge start locations into parent. + * + * Invariant: all UNDO records for this transaction, regardless of nesting + * level, are stored in a single chain per persistence level (one + * UndoRecordSet). start_location tracks the earliest record the + * subtransaction generated. Since records are strictly append-only, the + * parent's start location is always earlier than the subtransaction's if + * it exists. We only update the parent's start when it is not yet set + * (the parent wrote no UNDO before this subtransaction). + */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (UndoRecPtrIsValid(cur->start_location[i]) && + !UndoRecPtrIsValid(parent->start_location[i])) + { + parent->start_location[i] = cur->start_location[i]; + } + } + + XactUndo.subxact_depth--; +} + +/* + * AtSubAbort_XactUndo + * Subtransaction abort: apply undo for this sub-level, clean up. + * + * We apply the batches the subtransaction wrote synchronously, before + * control returns to the caller; the parent's batches are left alone. + */ +void +AtSubAbort_XactUndo(int level) +{ + XactUndoSubTransactionState *cur; + XactUndoSubTransactionState *parent; + int i; + + + if (XactUndo.subxact_depth <= 0) + return; + + cur = CURRENT_SUBXACT(); + if ((int) cur->nestingLevel != level) + return; + + parent = &XactUndo.subxact_stack[XactUndo.subxact_depth - 1]; + + /* + * For each persistence level where this subtransaction wrote UNDO, apply + * the batches newer than the parent's chain head; the parent's records + * are preserved. + */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + UndoRecPtr sub_start = cur->start_location[i]; + + if (!UndoRecPtrIsValid(sub_start)) + continue; + + /* + * Eagerly roll back the cluster-wide UNDO this subtransaction wrote. + * + * If the subtransaction advanced XactUndo.last_batch_lsn[i] beyond + * the value saved at subtransaction start, it wrote UNDO batches that + * must now be reversed. We must apply them here rather than leave + * them to the top-level abort: a subtransaction can abort while its + * parent goes on to COMMIT, in which case the top level never aborts + * and the UNDO would never run, leaking the aborted subtransaction's + * writes as if committed. + * + * Apply only the batches strictly newer than the parent's saved head + * (cur->last_batch_lsn[i]) via the bounded chain walk, so the + * parent's and earlier subtransactions' batches are preserved. + * Afterward restore last_batch_lsn[i] to the parent's value so a + * later parent abort walks only the parent's batches and does not + * double-apply ours. + * + * BumpContext safety: subtransaction abort may run under a + * BumpContext that does not support pfree(), so apply inside a + * dedicated AllocSetContext and present an in-progress transaction + * state for the relation opens, mirroring the inline path in + * AtAbort_XactUndo(). + */ + if (XLogRecPtrIsValid(XactUndo.last_batch_lsn[i]) && + XactUndo.last_batch_lsn[i] != cur->last_batch_lsn[i]) + { + XLogRecPtr sub_head = XactUndo.last_batch_lsn[i]; + XLogRecPtr parent_head = cur->last_batch_lsn[i]; + + /* + * Invariant: the current head must be strictly newer (larger) + * than the parent's saved head. If this fires, a sibling + * subtransaction improperly modified last_batch_lsn after its own + * abort, which would cause us to skip or double-apply batches. + */ + Assert(!XLogRecPtrIsValid(parent_head) || + sub_head > parent_head); + + /* + * Flush WAL so the subtransaction's batches are readable by the + * local XLog reader without busy-waiting on the walwriter. See + * the matching flush in AtAbort_XactUndo() for why XactLastRecEnd + * is the correct, always-valid record boundary to flush to. + */ + XLogFlush(XactLastRecEnd); + + if (UndoValidateBatchLSN(sub_head)) + { + MemoryContext undo_ctx; + MemoryContext old_ctx; + int saved_trans_state; + + undo_ctx = AllocSetContextCreate(TopMemoryContext, + "Subxact UNDO Apply", + ALLOCSET_DEFAULT_SIZES); + old_ctx = MemoryContextSwitchTo(undo_ctx); + + /* + * Release buffer content locks the failed subtransaction may + * still hold -- an error raised mid-split leaves both halves + * locked -- before the appliers lock pages again. See + * AtAbort_XactUndo. + */ + BufferLockReleaseAll(); + saved_trans_state = EnterInlineUndoApplyState(); + PG_TRY(); + { + ApplyUndoChainFromWALBounded(sub_head, parent_head); + } + PG_CATCH(); + { + /* + * Release any content locks the failed subxact UNDO chain + * left held before unwinding. + */ + BufferLockReleaseAll(); + LeaveInlineUndoApplyState(saved_trans_state); + FlushErrorState(); + } + PG_END_TRY(); + LeaveInlineUndoApplyState(saved_trans_state); + + MemoryContextSwitchTo(old_ctx); + MemoryContextDelete(undo_ctx); + } + else + elog(DEBUG1, "subxact UNDO: head %X/%08X is not a valid UNDO " + "batch, skipping eager rollback", + LSN_FORMAT_ARGS(sub_head)); + + XactUndo.last_batch_lsn[i] = parent_head; + PublishXactUndoChain(GetTopTransactionIdIfAny()); + } + + /* + * Reset the last_location to what it was before this subtransaction, + * so that if the parent transaction continues and then aborts, only + * the parent's records are applied (the subtransaction's records have + * already been applied). + */ + if (UndoRecPtrIsValid(parent->start_location[i])) + XactUndo.last_location[i] = parent->start_location[i]; + } + + XactUndo.subxact_depth--; +} + +/* + * AtProcExit_XactUndo + * Process exit cleanup for transaction undo. + */ +void +AtProcExit_XactUndo(void) +{ + int i; + + /* Free any lingering record sets. */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (XactUndo.record_set[i] != NULL) + { + UndoRecordSetFree(XactUndo.record_set[i]); + XactUndo.record_set[i] = NULL; + } + } + + /* Release any WAL retention hold (in case process exits mid-transaction). */ + UndoClearBatchLSN(); + + ResetXactUndo(); +} + +/* + * XactUndo_SubXactCallback + * Subtransaction callback to manage UNDO subtransaction state. + * + * This ensures the UNDO subsystem properly tracks all subtransactions, + * including those created by ROLLBACK TO SAVEPOINT. + * + * The subtransaction state is stored in a dynamically-grown array in + * TopMemoryContext. In the common case (depth < capacity), no allocation + * occurs. Growth via repalloc is safe here because it operates on a + * TopMemoryContext allocation, not a per-subtransaction context. + */ +static void +XactUndo_SubXactCallback(SubXactEvent event, SubTransactionId mySubid, + SubTransactionId parentSubid, void *arg) +{ + int i; + + /* These parameters are mandated by the callback signature. */ + (void) parentSubid; + (void) arg; + + switch (event) + { + case SUBXACT_EVENT_START_SUB: + + + /* + * A new subtransaction is starting. Push an entry onto the + * dynamically-grown stack, extending it if needed. + */ + XactUndo.subxact_depth++; + EnsureSubxactStackCapacity(); + { + XactUndoSubTransactionState *s = CURRENT_SUBXACT(); + + s->nestingLevel = mySubid; + for (i = 0; i < NUndoPersistenceLevels; ++i) + { + s->start_location[i] = InvalidUndoRecPtr; + /* Save parent's last_batch_lsn for restore on abort */ + s->last_batch_lsn[i] = XactUndo.last_batch_lsn[i]; + } + } + break; + + case SUBXACT_EVENT_COMMIT_SUB: + + /* + * Subtransaction is committing. Merge its UNDO state into parent. + */ + AtSubCommit_XactUndo(mySubid); + break; + + case SUBXACT_EVENT_ABORT_SUB: + + /* + * Subtransaction is aborting. Apply UNDO and clean up. + */ + AtSubAbort_XactUndo(mySubid); + break; + + case SUBXACT_EVENT_PRE_COMMIT_SUB: + /* Nothing to do at pre-commit */ + break; + } +} + +/* + * EnsureSubxactStackCapacity + * Ensure the subxact_stack has room for the current depth. + * + * If the stack hasn't been allocated yet, allocate it with the initial + * capacity. If we've exceeded the current capacity, double it. + * The stack lives in TopMemoryContext so it persists across transactions + * within the same backend. + */ +static void +EnsureSubxactStackCapacity(void) +{ + if (XactUndo.subxact_stack == NULL) + { + /* First-time allocation */ + XactUndo.subxact_capacity = INITIAL_SUBXACT_CAPACITY; + XactUndo.subxact_stack = (XactUndoSubTransactionState *) + MemoryContextAllocZero(TopMemoryContext, + XactUndo.subxact_capacity * + sizeof(XactUndoSubTransactionState)); + } + else if (XactUndo.subxact_depth >= XactUndo.subxact_capacity) + { + int new_capacity = XactUndo.subxact_capacity * 2; + + XactUndo.subxact_stack = (XactUndoSubTransactionState *) + repalloc(XactUndo.subxact_stack, + new_capacity * sizeof(XactUndoSubTransactionState)); + /* Zero the newly-allocated portion */ + memset(&XactUndo.subxact_stack[XactUndo.subxact_capacity], 0, + (new_capacity - XactUndo.subxact_capacity) * + sizeof(XactUndoSubTransactionState)); + XactUndo.subxact_capacity = new_capacity; + } +} + +/* + * ResetXactUndo + * Reset all backend-private undo state for the next transaction. + */ +static void +ResetXactUndo(void) +{ + int i; + + XactUndo.has_undo = false; + XactUndo.subxact_depth = 0; + + /* + * The subxact_stack allocation persists across transactions (it's in + * TopMemoryContext). We just reset the depth and initialize slot 0. + */ + if (XactUndo.subxact_stack != NULL) + { + XactUndo.subxact_stack[0].nestingLevel = 1; + for (i = 0; i < NUndoPersistenceLevels; i++) + { + XactUndo.subxact_stack[0].start_location[i] = InvalidUndoRecPtr; + XactUndo.subxact_stack[0].last_batch_lsn[i] = InvalidXLogRecPtr; + } + } + + for (i = 0; i < NUndoPersistenceLevels; i++) + { + XactUndo.record_set[i] = NULL; + XactUndo.last_location[i] = InvalidUndoRecPtr; + XactUndo.last_batch_lsn[i] = InvalidXLogRecPtr; + } + + /* + * The chain is now the commit's, the abort's or the 2PC state's; none of + * those needs the in-flight record. + */ + PublishXactUndoChain(InvalidTransactionId); +} + +/* + * CollapseXactUndoSubTransactions + * Collapse all subtransaction state into the top level. + */ +static void +CollapseXactUndoSubTransactions(void) +{ + while (XactUndo.subxact_depth > 0) + { + /* + * Merge current level into parent by calling AtSubCommit_XactUndo + * with the current level's nestingLevel. + */ + AtSubCommit_XactUndo( + XactUndo.subxact_stack[XactUndo.subxact_depth].nestingLevel); + } +} diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index b6700929459f8..f01593bf38151 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -974,6 +974,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, break; default: (void) heap_reloptions(relkind, reloptions, true); + break; } if (stmt->ofTypename) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 4dbc25c6c5054..3dbd1245d4090 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -5928,6 +5928,79 @@ UnlockBuffers(void) } } +/* + * BufferLockReleaseAll -- release any buffer content locks still held by this + * backend. + * + * Normally, buffer content locks are released as part of resource-owner + * cleanup (see ResOwnerReleaseBuffer, which calls BufferLockUnlock on any + * entry whose data.lockmode != BUFFER_LOCK_UNLOCK). However, that cleanup + * runs in ResourceOwnerRelease(RESOURCE_RELEASE_LOCKS), which is *after* + * AtAbort_XactUndo() in AbortTransaction(). Inline UNDO application may + * itself take buffer content locks, and BufferLockAcquire asserts that no + * lock is already held. If a statement errors out while holding a share + * or exclusive content lock, that assert would trip on any re-lock during + * UNDO application. + * + * This helper releases such stragglers early, before inline UNDO runs. + * It mirrors the still-locked branch of ResOwnerReleaseBuffer (matching + * the HOLD_INTERRUPTS/BufferLockUnlock discipline). Callers must be in + * an error-recovery path -- releasing content locks under a live statement + * would violate correctness. + * + * The subsequent ResourceOwnerRelease pass at RESOURCE_RELEASE_LOCKS still + * runs, but ResOwnerReleaseBuffer only calls BufferLockUnlock when + * data.lockmode != BUFFER_LOCK_UNLOCK, so already-released buffers are + * skipped -- no double-unlock. + */ +void +BufferLockReleaseAll(void) +{ + PrivateRefCountEntry *res; + int i; + + /* Walk the small array first (fast path, covers the usual case). */ + for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) + { + Buffer buffer = PrivateRefCountArrayKeys[i]; + + if (buffer == InvalidBuffer) + continue; + + res = &PrivateRefCountArray[i]; + if (res->data.lockmode == BUFFER_LOCK_UNLOCK) + continue; + + /* Local buffers do not have content locks. */ + if (BufferIsLocal(buffer)) + continue; + + HOLD_INTERRUPTS(); /* matched by RESUME_INTERRUPTS in + * BufferLockUnlock */ + BufferLockUnlock(buffer, GetBufferDescriptor(buffer - 1)); + } + + /* Then the overflow hash, if any. */ + if (PrivateRefCountOverflowed) + { + refcount_iterator iter; + + refcount_start_iterate(PrivateRefCountHash, &iter); + while ((res = refcount_iterate(PrivateRefCountHash, &iter)) != NULL) + { + Buffer buffer = res->buffer; + + if (res->data.lockmode == BUFFER_LOCK_UNLOCK) + continue; + if (BufferIsLocal(buffer)) + continue; + + HOLD_INTERRUPTS(); + BufferLockUnlock(buffer, GetBufferDescriptor(buffer - 1)); + } + } +} + /* * Acquire the buffer content lock in the specified mode * diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 1512d2df196e6..613984a301ce2 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1187,6 +1187,7 @@ ProcessUtilitySlow(ParseState *pstate, validnsps, true, false); + (void) heap_reloptions(RELKIND_TOASTVALUE, toast_options, true); diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 8b6ea195ecade..7a0522ae859d8 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -21,6 +21,7 @@ #include "access/genam.h" #include "access/heapam.h" +#include "access/undo.h" #include "access/htup_details.h" #include "access/session.h" #include "access/tableam.h" @@ -857,6 +858,9 @@ InitPostgres(const char *in_dbname, Oid dboid, InitCatalogCache(); InitPlanCache(); + /* Initialize per-backend undo subsystem state */ + InitializeUndo(); + /* Initialize portal manager */ EnablePortalManager(); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index a3f38f3b88679..811f6869bb8a9 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -17530,6 +17530,7 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) if (nonemptyReloptions(tbinfo->reloptions)) { addcomma = true; + /* Emits all heap reloptions */ appendReloptionsArrayAH(q, tbinfo->reloptions, "", fout); } if (nonemptyReloptions(tbinfo->toast_reloptions)) diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c index 931ab8b979e23..8570f17916fc3 100644 --- a/src/bin/pg_waldump/rmgrdesc.c +++ b/src/bin/pg_waldump/rmgrdesc.c @@ -20,6 +20,7 @@ #include "access/nbtxlog.h" #include "access/rmgr.h" #include "access/spgxlog.h" +#include "access/undo_xlog.h" #include "access/xact.h" #include "access/xlog_internal.h" #include "catalog/storage_xlog.h" diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index 7b33efc62999f..fbeb19ee4a4e8 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -96,7 +96,8 @@ ReplicationOrigin Generic LogicalMessage -XLOG2$/, +XLOG2 +Undo$/, 'rmgr list'); diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 99b51296e8018..ea91747ae85ab 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -544,4 +544,5 @@ heap_execute_freeze_tuple(HeapTupleHeader tuple, HeapTupleFreeze *frz) tuple->t_infomask2 = frz->t_infomask2; } + #endif /* HEAPAM_H */ diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index ae32ef16d67b6..046ba45cb2532 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -48,3 +48,4 @@ PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask, NULL) PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL, logicalmsg_decode) PG_RMGR(RM_XLOG2_ID, "XLOG2", xlog2_redo, xlog2_desc, xlog2_identify, NULL, NULL, NULL, xlog2_decode) +PG_RMGR(RM_UNDO_ID, "Undo", undo_redo, undo_desc, undo_identify, NULL, NULL, NULL, NULL) diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h index 1d2ff42c9b72f..3714ea62b3385 100644 --- a/src/include/access/twophase.h +++ b/src/include/access/twophase.h @@ -48,6 +48,7 @@ extern GlobalTransaction MarkAsPreparing(FullTransactionId fxid, const char *gid extern void StartPrepare(GlobalTransaction gxact); extern void EndPrepare(GlobalTransaction gxact); extern bool StandbyTransactionIdIsPrepared(TransactionId xid); +extern bool RecoveryTransactionIdIsPrepared(TransactionId xid); extern TransactionId PrescanPreparedTransactions(TransactionId **xids_p, int *nxids_p); @@ -72,4 +73,6 @@ extern bool LookupGXactBySubid(Oid subid); extern TransactionId TwoPhaseGetOldestXidInCommit(void); +extern XLogRecPtr TwoPhaseGetOldestUndoBatchLSN(void); + #endif /* TWOPHASE_H */ diff --git a/src/include/access/undo.h b/src/include/access/undo.h new file mode 100644 index 0000000000000..84007d10e06c0 --- /dev/null +++ b/src/include/access/undo.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * undo.h + * Common undo layer interface + * + * See access/undo/README for the pieces of the UNDO engine. This header + * provides the entry points for shared memory set-up and per-backend + * initialization across them. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undo.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDO_H +#define UNDO_H + +#include "access/undodefs.h" +#include "utils/palloc.h" + +/* + * Shared memory initialization, called through UndoShmemCallbacks (see + * storage/subsystemlist.h). + */ +extern void UndoShmemInit(void); + +/* Per-backend initialization */ +extern void InitializeUndo(void); + +/* Memory context for undo-related allocations */ +extern MemoryContext UndoContext; + +#endif /* UNDO_H */ diff --git a/src/include/access/undo_xlog.h b/src/include/access/undo_xlog.h new file mode 100644 index 0000000000000..aceef55fdbbe6 --- /dev/null +++ b/src/include/access/undo_xlog.h @@ -0,0 +1,169 @@ +/*------------------------------------------------------------------------- + * + * undo_xlog.h + * UNDO resource manager WAL record definitions + * + * This file contains the WAL record format definitions for UNDO log + * operations. These records are logged by the RM_UNDO_ID resource manager. + * + * Record types: + * XLOG_UNDO_BATCH - A batch of a transaction's UNDO records + * XLOG_UNDO_INFLIGHT - UNDO chains in flight at a checkpoint + * XLOG_UNDO_ROLLBACK - A crash loser rolled back by recovery + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undo_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDO_XLOG_H +#define UNDO_XLOG_H + +#include "access/transam.h" +#include "access/xlogdefs.h" +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* + * UndoRecPtr type definition. We use undodefs.h which is lightweight + * and can be included in both frontend and backend code. If undodefs.h + * has already been included (via undolog.h or directly), this is a no-op. + */ +#include "access/undodefs.h" + +/* + * WAL record types for UNDO operations + * + * These are the info codes for UNDO WAL records. The low 4 bits are used + * for operation type, leaving the upper 4 bits for flags. + */ +#define XLOG_UNDO_BATCH 0x00 /* Batched UNDO records in WAL */ +#define XLOG_UNDO_INFLIGHT 0x10 /* UNDO chains in flight at a + * checkpoint */ +#define XLOG_UNDO_ROLLBACK 0x20 /* A crash loser rolled back by + * recovery */ + +/* + * xl_undo_rollback - A crash loser that the end of recovery rolled back + * + * A transaction that a crash leaves in progress has no abort record. At the + * end of crash recovery, once WAL can be written again, the startup process + * applies each such loser's UNDO records and then writes one of these, + * carrying the records that applied, in the order it applied them (newest + * first). Redo applies them again, the same way, so that a standby, and a + * later recovery that replays this record, end up where the primary did, + * without reading the loser's UNDO chain from WAL. Every UNDO resource + * manager's apply function must therefore leave an object alone that no + * longer shows the record's effect. + * + * The records follow the header as a packed stream of UndoRecordHeader and + * payload, each urec_len bytes long, as in an XLOG_UNDO_BATCH record. + */ +typedef struct xl_undo_rollback +{ + TransactionId xid; /* the loser */ + uint32 nrecords; /* UNDO records that follow */ + /* followed by the UNDO records */ +} xl_undo_rollback; + +#define SizeOfUndoRollback sizeof(xl_undo_rollback) + +/* The records must start MAXALIGN'd, as they do in a batch. */ +StaticAssertDecl(SizeOfUndoRollback % MAXIMUM_ALIGNOF == 0, + "xl_undo_rollback must keep the UNDO records after it aligned"); + +/* + * xl_undo_batch - WAL record for batched UNDO data (XLOG_UNDO_BATCH) + * + * All UNDO records for a batch are serialized into a single WAL record. + * The batch payload contains concatenated UndoRecordHeader+payload pairs + * in their exact serialized format. + * + * The chain_prev field links this batch to the previous batch for the + * same transaction. During rollback, the UNDO chain is walked backward + * by reading WAL records at successive chain_prev LSNs. + * + * Coalescing: The existing UndoRecordSet mechanism batches records + * (flush at 256KB or 1000 records). This batch becomes one WAL record. + * A 1000-row INSERT produces ~1 WAL record containing 1000 UNDO records. + */ +typedef struct xl_undo_batch +{ + TransactionId xid; /* Owning transaction */ + XLogRecPtr chain_prev; /* LSN of previous batch for this xact + * (InvalidXLogRecPtr if first batch) */ + uint32 nrecords; /* Number of UNDO records in batch */ + uint32 total_len; /* Total bytes of serialized UNDO data */ + Oid primary_reloid; /* Relation OID (optimization for + * single-relation batches) */ + UndoPersistenceLevel persistence; /* Persistence level of this batch */ + /* Followed by total_len bytes of serialized UndoRecordHeader+payload */ +} xl_undo_batch; + +#define SizeOfUndoBatch (offsetof(xl_undo_batch, persistence) + sizeof(UndoPersistenceLevel)) + +/* + * xl_undo_inflight - UNDO chains of transactions in flight at a checkpoint + * + * Written by every online checkpoint after its redo pointer. Crash recovery + * starts redo there and so replays none of the batches written before it; a + * transaction whose batches all precede the redo pointer is known to + * recovery only through this record. + */ +typedef struct xl_undo_inflight_xact +{ + TransactionId xid; /* top-level transaction */ + XLogRecPtr last_batch_lsn[NUndoPersistenceLevels]; +} xl_undo_inflight_xact; + +typedef struct xl_undo_inflight +{ + uint32 nxacts; + xl_undo_inflight_xact xacts[FLEXIBLE_ARRAY_MEMBER]; +} xl_undo_inflight; + +#define SizeOfUndoInflight offsetof(xl_undo_inflight, xacts) + +/* Function declarations for WAL operations */ +extern void undo_redo(XLogReaderState *record); +extern void undo_desc(StringInfo buf, XLogReaderState *record); +extern const char *undo_identify(uint8 info); + +/* + * UNDO batch reading from WAL for rollback and recovery. + * + * UndoReadBatchFromWAL reads a single XLOG_UNDO_BATCH record at the + * given LSN and returns the header plus a pointer to the payload data. + * The caller must pfree the returned data when done. + */ +typedef struct UndoBatchData +{ + xl_undo_batch header; /* Batch header */ + char *payload; /* Serialized UNDO records (palloc'd) */ + Size payload_len; /* Length of payload */ +} UndoBatchData; + +extern bool UndoValidateBatchLSN(XLogRecPtr batch_lsn); +extern UndoBatchData *UndoReadBatchFromWAL(XLogRecPtr batch_lsn); +extern void UndoFreeBatchData(UndoBatchData *batch); + +/* + * Recovery UNDO phase support. + * + * During WAL redo, XLOG_UNDO_BATCH and XLOG_UNDO_INFLIGHT records are + * tracked, so that at the end of recovery the transactions a crash left in + * progress can be found and rolled back (UndoRecoveryRollBack). + */ +extern void UndoRecoveryTrackBatch(TransactionId xid, XLogRecPtr batch_lsn, + XLogRecPtr chain_prev, + UndoPersistenceLevel persistence); +extern void UndoRecoveryRemoveXid(TransactionId xid); +extern void UndoRecoveryTrackInflight(const xl_undo_inflight_xact *x); +extern void UndoRecoveryRollBack(void); + +/* True while recovery applies UNDO records outside redo; see there. */ +extern PGDLLIMPORT bool UndoRecoveryRollingBack; + +#endif /* UNDO_XLOG_H */ diff --git a/src/include/access/undodefs.h b/src/include/access/undodefs.h new file mode 100644 index 0000000000000..c64192f01fb6d --- /dev/null +++ b/src/include/access/undodefs.h @@ -0,0 +1,47 @@ +/*------------------------------------------------------------------------- + * + * undodefs.h + * + * Basic definitions for PostgreSQL undo layer. These are separated into + * their own header file to avoid including more things than necessary + * into widely-used headers like xact.h. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undodefs.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDODEFS_H +#define UNDODEFS_H + +/* The type used to identify an undo log and position within it. */ +typedef uint64 UndoRecPtr; + +/* Special value for undo record pointer which indicates that it is invalid. */ +#define InvalidUndoRecPtr ((UndoRecPtr) 0) + +/* + * UndoRecPtrIsValid + * True iff undoRecPtr is valid. + */ +#define UndoRecPtrIsValid(undoRecPtr) \ + ((bool) ((UndoRecPtr) (undoRecPtr) != InvalidUndoRecPtr)) + +/* Persistence levels as small integers that can be used as array indexes. */ +typedef enum +{ + UNDOPERSISTENCE_PERMANENT = 0, + UNDOPERSISTENCE_UNLOGGED = 1, + UNDOPERSISTENCE_TEMP = 2 +} UndoPersistenceLevel; + +/* Number of supported persistence levels for undo. */ +#define NUndoPersistenceLevels 3 + +/* Opaque types. */ +struct UndoRecordSet; +typedef struct UndoRecordSet UndoRecordSet; + +#endif diff --git a/src/include/access/undolog.h b/src/include/access/undolog.h new file mode 100644 index 0000000000000..519739acdace1 --- /dev/null +++ b/src/include/access/undolog.h @@ -0,0 +1,41 @@ +/*------------------------------------------------------------------------- + * + * undolog.h + * WAL retention for UNDO records + * + * UNDO records live in the WAL stream as XLOG_UNDO_BATCH records, so the + * WAL holding a transaction's UNDO must not be recycled while something may + * still read it: the transaction's own abort, ROLLBACK PREPARED, or crash + * recovery. Each backend publishes the LSN of its current transaction's + * first UNDO batch in a shared slot, and KeepLogSeg() keeps WAL from the + * oldest such LSN onward. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undolog.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDOLOG_H +#define UNDOLOG_H + +#include "access/xlogdefs.h" +#include "port/atomics.h" + +/* + * The per-backend first-batch LSN slots, MaxBackends of them, indexed by + * MyProcNumber. A slot holds the LSN of the first XLOG_UNDO_BATCH record of + * the backend's current transaction, or InvalidXLogRecPtr. Only the owning + * backend writes its slot. + */ +extern PGDLLIMPORT pg_atomic_uint64 *UndoBatchLSNs; + +extern void UndoLogShmemRequest(void); +extern void UndoLogShmemInit(void); + +extern void UndoRegisterBatchLSN(XLogRecPtr batch_lsn); +extern void UndoClearBatchLSN(void); +extern XLogRecPtr UndoGetOldestBatchLSN(void); + +#endif /* UNDOLOG_H */ diff --git a/src/include/access/undorecord.h b/src/include/access/undorecord.h new file mode 100644 index 0000000000000..19337f9291389 --- /dev/null +++ b/src/include/access/undorecord.h @@ -0,0 +1,166 @@ +/*------------------------------------------------------------------------- + * + * undorecord.h + * UNDO record format and insertion API + * + * This file defines the generic UNDO record format that can be used by + * any access method or subsystem. UNDO records are AM-agnostic: each + * record carries an RM ID (urec_rmid) that identifies the resource + * manager responsible for interpreting and applying the record. + * + * Design principles: + * - Physical: UNDO stores opaque payload data for direct restore + * - Generic: Usable by any AM or subsystem + * - Compact: Variable-length format to minimize space + * - Chained: Batches form backward chains via xl_undo_batch.chain_prev + * - Batch-oriented: API encourages batching for performance + * - AM-agnostic: No AM-specific types in the generic header or API + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undorecord.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDORECORD_H +#define UNDORECORD_H + +#include "access/undodefs.h" +#include "access/undolog.h" +#include "access/xlogdefs.h" +#include "storage/block.h" +#include "storage/itemptr.h" + +/* + * UNDO record info flags + * + * These flags provide additional metadata about the UNDO record. + * The lower byte is reserved for generic flags; the upper byte is + * available for RM-specific use. + */ +#define UNDO_INFO_HAS_PAYLOAD 0x01 /* Record contains opaque payload */ +#define UNDO_INFO_XID_VALID 0x08 /* urec_xid is valid */ + +/* + * UndoRecordHeader - Fixed header for all UNDO records + * + * Every UNDO record starts with this header, followed by an optional + * opaque payload whose interpretation is RM-specific. + * + * The urec_rmid field identifies which resource manager owns this record. + * The urec_info field carries RM-specific subtype/flags (e.g., an in-place + * update AM uses it to distinguish INSERT vs DELETE vs UPDATE). + * + * Size: 20 bytes; each record is padded to MAXALIGN (see undorecord.c). + */ +typedef struct UndoRecordHeader +{ + uint8 urec_rmid; /* UNDO RM ID */ + uint8 urec_flags; /* Generic flags (UNDO_INFO_*) */ + uint16 urec_info; /* RM-specific subtype and flags */ + uint32 urec_len; /* Total length including header + payload */ + + TransactionId urec_xid; /* Transaction that created this */ + Oid urec_reloid; /* Relation OID (InvalidOid if N/A) */ + + /* + * Payload length: size of the RM-specific opaque data that follows the + * header. Interpretation is entirely RM-specific. + */ + uint32 urec_payload_len; + + /* Followed by variable-length RM-specific payload */ +} UndoRecordHeader; + +#define SizeOfUndoRecordHeader (offsetof(UndoRecordHeader, urec_payload_len) + sizeof(uint32)) + +/* + * Access macros for payload data following the header + * + * The payload immediately follows the fixed header in the serialized + * record. Its interpretation is entirely RM-specific. + */ +#define UndoRecGetPayload(header) \ + ((char *)(header) + SizeOfUndoRecordHeader) + +/* + * UndoRecordSet - Batch container for UNDO records + * + * This structure accumulates multiple UNDO records before writing them + * to the UNDO log in a single operation. This improves performance by + * reducing the number of I/O operations and lock acquisitions. + * + * The records are serialized into a contiguous buffer that grows + * dynamically. + */ +typedef struct UndoRecordSet +{ + TransactionId xid; /* Transaction ID for all records */ + UndoPersistenceLevel persistence; /* Persistence level of this set */ + + int nrecords; /* Number of records in set */ + + /* + * Dynamic buffer for serialized records. Grows as needed; no fixed + * maximum. This replaces the old fixed-capacity max_records array. + */ + char *buffer; /* Serialized record buffer */ + Size buffer_size; /* Current buffer size */ + Size buffer_capacity; /* Allocated buffer capacity */ + + /* + * WAL-based UNDO chain tracking. When UNDO records are written to WAL + * via XLOG_UNDO_BATCH, last_batch_lsn tracks the LSN of the most recent + * batch for this record set. This is used as the chain_prev link when + * the next batch is written. + */ + XLogRecPtr last_batch_lsn; /* LSN of last XLOG_UNDO_BATCH record */ + + MemoryContext mctx; /* Memory context for allocations */ +} UndoRecordSet; + +/* + * Public API for UNDO record management + */ + +/* Create/destroy/reset UNDO record sets */ +extern UndoRecordSet *UndoRecordSetCreate(TransactionId xid); +extern void UndoRecordSetFree(UndoRecordSet *uset); +extern void UndoRecordSetReset(UndoRecordSet *uset); +extern void UndoRecordSetResetCache(void); + +/* Add records to a set - generic payload API */ +extern void UndoRecordAddPayload(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *payload, + Size payload_len); + +/* Add records with scatter-gather payload (avoids intermediate buffer) */ +extern void UndoRecordAddPayloadParts(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *part1, + Size part1_len, + const char *part2, + Size part2_len); + +/* Insert the accumulated records into UNDO log */ +extern UndoRecPtr UndoRecordSetInsert(UndoRecordSet *uset); + +/* Utility functions for record manipulation */ +extern Size UndoRecordGetPayloadSize(Size payload_len); + +/* Statistics and debugging */ + +/* Records currently accumulated in the set (0 after a flush/reset). */ + +/* UNDO application during rollback */ +extern bool ApplyUndoChainFromWAL(XLogRecPtr last_batch_lsn); +extern bool ApplyUndoChainFromWALBounded(XLogRecPtr last_batch_lsn, + XLogRecPtr stop_at_lsn); + +#endif /* UNDORECORD_H */ diff --git a/src/include/access/undormgr.h b/src/include/access/undormgr.h new file mode 100644 index 0000000000000..f03f5e0c8dd7b --- /dev/null +++ b/src/include/access/undormgr.h @@ -0,0 +1,117 @@ +/*------------------------------------------------------------------------- + * + * undormgr.h + * UNDO resource manager dispatch definitions + * + * This module provides a dispatch mechanism for UNDO record application, + * analogous to the WAL resource manager (rmgr) system. Each access method + * or subsystem that writes UNDO records registers an UndoRmgrData entry + * with callbacks for applying UNDO records and describing them for debugging. + * + * The generic UNDO infrastructure (undoapply.c) dispatches to the appropriate + * RM callback based on the urec_rmid field in the UNDO record header. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undormgr.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDORMGR_H +#define UNDORMGR_H + +#include "postgres_ext.h" +#include "access/undodefs.h" +#include "access/xlogdefs.h" +#include "lib/stringinfo.h" + +/* + * UNDO Resource Manager IDs + * + * Each AM or subsystem that writes UNDO records is assigned a unique ID. + * This ID is stored in the urec_rmid field of every UNDO record header, + * enabling the generic UNDO infrastructure to dispatch to the correct + * apply callback during rollback. + * + * The core defines only UNDO_RMID_INVALID. Each consumer defines its own + * UNDO_RMID_* constant in its own header (the same commit that registers it + * in access/undormgrlist.h), so this core header names no specific AM or + * subsystem. The ID number space is a shared resource: IDs must be unique + * and stable across releases (they are stamped into WAL-durable UNDO + * records), so a new consumer takes the next free value in + * [1, MAX_UNDO_RMGRS) and never reuses one. + */ +#define UNDO_RMID_INVALID 0 + +#define MAX_UNDO_RMGRS 256 + +/* + * UndoApplyResult - Return value from undo apply callbacks + */ +typedef enum UndoApplyResult +{ + UNDO_APPLY_SUCCESS = 0, /* Successfully applied */ + UNDO_APPLY_SKIPPED, /* Skipped (e.g., relation dropped) */ + UNDO_APPLY_ERROR /* Error during application */ +} UndoApplyResult; + +/* + * UndoRmgrData - Resource manager registration entry + * + * Each UNDO RM provides: + * rm_name: Human-readable name for debugging/logging + * rm_undo: Apply one UNDO record (rollback callback) + * rm_desc: Describe an UNDO record for debugging output + * + * The rm_undo callback receives: + * - rmid: The RM ID (for verification) + * - info: RM-specific subtype/flags from urec_info + * - xid: Transaction being rolled back + * - reloid: Target relation OID (may be InvalidOid for non-relation ops) + * - payload: RM-specific opaque payload data + * - payload_len: Length of payload + * - urec_ptr: Position of this record in UNDO log, if known + * + * The callback is responsible for all of the work, including releasing any + * locks or buffers it takes. It may be called more than once for the same + * record: by an online abort, by the undo phase at the end of crash + * recovery, and by redo of the XLOG_UNDO_ROLLBACK record that logs the undo + * phase's work, on the primary and on standbys. So it must leave alone an + * object that no longer shows the record's effect, and it must not write + * WAL, since redo cannot. + */ +typedef UndoApplyResult (*UndoRmgrApplyFunc) (uint8 rmid, + uint16 info, + TransactionId xid, + Oid reloid, + const char *payload, + Size payload_len, + UndoRecPtr urec_ptr); + +typedef void (*UndoRmgrDescFunc) (StringInfo buf, + uint8 rmid, + uint16 info, + const char *payload, + Size payload_len); + +typedef struct UndoRmgrData +{ + const char *rm_name; /* Human-readable name */ + UndoRmgrApplyFunc rm_undo; /* Apply callback */ + UndoRmgrDescFunc rm_desc; /* Describe callback */ +} UndoRmgrData; + +/* Global registration table */ +extern const UndoRmgrData *UndoRmgrs[MAX_UNDO_RMGRS]; + +/* Registration function (called during _PG_init or startup) */ +extern void RegisterUndoRmgr(uint8 rmid, const UndoRmgrData *rmgr); + +/* Lookup function */ +extern const UndoRmgrData *GetUndoRmgr(uint8 rmid); + +/* Initialization */ +extern void InitUndoRmgrs(void); + +#endif /* UNDORMGR_H */ diff --git a/src/include/access/undormgrlist.h b/src/include/access/undormgrlist.h new file mode 100644 index 0000000000000..4ab60e2141881 --- /dev/null +++ b/src/include/access/undormgrlist.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------- + * + * undormgrlist.h + * + * List of *UndoRmgrInit() initialization calls for built-in UNDO resource + * managers. Kept in its own source file, mirroring the pattern used by + * storage/subsystemlist.h for ShmemCallbacks registration, so that the + * generic UNDO core (undo.c) can register every built-in resource manager + * without knowing any of their names at compile time: undo.c only expands + * this list through a macro it controls, and the extern declarations are + * generated the same way (see access/undormgrs.h). + * + * Each access method or subsystem that writes UNDO records adds itself here + * when it is compiled in. This file is edited by whichever commit + * introduces a new UNDO-writing consumer; undo.c itself is never touched to + * add a new consumer. + * + * UNDO_RMGR_INIT is defined by the caller depending on how the list is used. + * + * No built-in resource managers exist yet at this point in the UNDO + * subsystem's history; this file starts empty and grows as each consumer + * (an index AM, a table AM, or a subsystem that writes UNDO records) is + * compiled in and adds itself here. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undormgrlist.h + * + *------------------------------------------------------------------------- + */ + +/* there is deliberately not an #ifndef UNDORMGRLIST_H here */ diff --git a/src/include/access/undormgrs.h b/src/include/access/undormgrs.h new file mode 100644 index 0000000000000..2a399f78dfab2 --- /dev/null +++ b/src/include/access/undormgrs.h @@ -0,0 +1,28 @@ +/*------------------------------------------------------------------------- + * + * undormgrs.h + * Provide extern declarations for all the built-in UNDO resource + * manager *Init() functions. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undormgrs.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDORMGRS_H +#define UNDORMGRS_H + +/* + * Extern declarations of all the built-in *UndoRmgrInit() functions. + * + * The actual list is in undormgrlist.h, so that the same list can be used + * for other purposes (e.g. undo.c's RegisterUndoRmgrs()). + */ +#define UNDO_RMGR_INIT(initfunc) \ + extern void initfunc(void); +#include "access/undormgrlist.h" +#undef UNDO_RMGR_INIT + +#endif /* UNDORMGRS_H */ diff --git a/src/include/access/xact.h b/src/include/access/xact.h index a8cbdf247c866..3abc534fa20a1 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -138,6 +138,24 @@ typedef enum typedef void (*XactCallback) (XactEvent event, void *arg); +/* + * PendingPhysOps{Do,PostPrepare,AtSubCommit,AtSubAbort}_hook: mirror the + * matching core smgr{DoPendingDeletes,PostPrepare,AtSubCommit,AtSubAbort} + * calls in xact.c. A subsystem that manages pending structural filesystem + * operations needs the SAME commit/abort-sequencing position + * as smgr's own pending-deletes housekeeping -- e.g. + * PendingPhysOpsDo_hook(true) is placed immediately after + * smgrDoPendingDeletes(true), which core's own comment there documents as + * "best done after releasing relcache and buffer pins... this ordering is + * definitely critical during abort" -- a physical-file-cleanup ordering + * requirement no ordinary XactCallback/SubXactCallback event exposes. NULL + * is a valid no-op for a build with no such subsystem compiled in. + */ +extern void (*PendingPhysOpsDo_hook) (bool isCommit); +extern void (*PendingPhysOpsPostPrepare_hook) (void); +extern void (*PendingPhysOpsAtSubCommit_hook) (void); +extern void (*PendingPhysOpsAtSubAbort_hook) (void); + typedef enum { SUBXACT_EVENT_START_SUB, @@ -368,8 +386,24 @@ typedef struct xl_xact_prepare uint16 gidlen; /* length of the GID - GID follows the header */ XLogRecPtr origin_lsn; /* lsn of this record at origin node */ TimestampTz origin_timestamp; /* time of prepare at origin node */ + + /* + * UNDO chain head LSN per persistence level (3 == NUndoPersistenceLevels + * from undodefs.h; hardcoded here to keep xact.h free of UNDO headers). + * If NUndoPersistenceLevels changes, this array must be updated and both + * XLOG_PAGE_MAGIC (xlog_internal.h) and TWOPHASE_MAGIC (twophase.c) must + * be bumped. See StaticAssertDecl in xactundo.c for compile-time guard. + */ + XLogRecPtr last_batch_lsn[3]; } xl_xact_prepare; +/* + * xl_xact_prepare is written into WAL as part of XLOG_XACT_PREPARE records, + * and into 2PC state files via TwoPhaseFileHeader. Any layout change + * requires bumping both XLOG_PAGE_MAGIC (xlog_internal.h) and TWOPHASE_MAGIC + * (twophase.c). + */ + /* * Commit/Abort records in the above form are a bit verbose to parse, so * there's a deconstructed versions generated by ParseCommit/AbortRecord() for @@ -439,6 +473,8 @@ typedef struct xl_xact_parsed_abort */ extern bool IsTransactionState(void); extern bool IsAbortedTransactionBlockState(void); +extern int EnterInlineUndoApplyState(void); +extern void LeaveInlineUndoApplyState(int saved); extern TransactionId GetTopTransactionId(void); extern TransactionId GetTopTransactionIdIfAny(void); extern TransactionId GetCurrentTransactionId(void); diff --git a/src/include/access/xactundo.h b/src/include/access/xactundo.h new file mode 100644 index 0000000000000..d624e63af5785 --- /dev/null +++ b/src/include/access/xactundo.h @@ -0,0 +1,88 @@ +/*------------------------------------------------------------------------- + * + * xactundo.h + * Transaction-level undo management + * + * This module manages per-transaction undo record sets. It maintains + * up to NUndoPersistenceLevels (3) record sets per transaction -- one + * for each persistence level (permanent, unlogged, temporary), so that + * records for different persistence levels are kept separate. + * + * Code that wants to write transactional undo should interface with + * these functions rather than manipulating UndoRecordSet directly. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/xactundo.h + * + *------------------------------------------------------------------------- + */ +#ifndef XACTUNDO_H +#define XACTUNDO_H + +#include "access/undodefs.h" +#include "access/undorecord.h" +#include "access/xlogdefs.h" + +/* + * XactUndoContext - Context for a single undo insertion within a transaction. + * + * Created by PrepareXactUndoData(), consumed by InsertXactUndoData() + * and cleaned up by CleanupXactUndoInsertion(). The plevel tracks which + * persistence-level record set this insertion belongs to. + */ +typedef struct XactUndoContext +{ + UndoPersistenceLevel plevel; + UndoRecordSet *uset; /* borrowed reference, do not free */ +} XactUndoContext; + +/* Shared memory initialization */ +extern Size XactUndoShmemSize(void); +extern void XactUndoShmemRequest(void); +extern void XactUndoShmemInit(void); + +/* Per-backend initialization */ +extern void InitializeXactUndo(void); + +/* + * Undo insertion API for any AM or subsystem. + * + * PrepareXactUndoData: Find or create the appropriate per-persistence-level + * UndoRecordSet for the current transaction and prepare it for a new + * record. Returns the UndoRecPtr where the record will be written. + * + * Parameters are AM-agnostic: the caller provides an RM ID, RM-specific + * info flags, a relation OID, and an opaque payload. + * + * InsertXactUndoData: Actually write the record data into the undo log. + * + * CleanupXactUndoInsertion: Release any resources held by the context. + */ +extern UndoRecPtr PrepareXactUndoData(XactUndoContext *ctx, + char persistence, + uint8 rmid, + uint16 info, + Oid reloid, + const char *payload, + Size payload_len); +extern void InsertXactUndoData(XactUndoContext *ctx); +extern void CleanupXactUndoInsertion(XactUndoContext *ctx); + +/* Transaction lifecycle hooks */ +extern void AtCommit_XactUndo(void); +extern void AtAbort_XactUndo(void); +extern void AtPrepare_XactUndo(void); +extern void AtSubCommit_XactUndo(int level); +extern void AtSubAbort_XactUndo(int level); +extern void AtProcExit_XactUndo(void); + + +/* Checkpoint: WAL-log the UNDO chains of in-flight transactions */ +extern void LogXactUndoInflight(void); + +/* Undo chain traversal for rollback */ +extern XLogRecPtr GetCurrentXactLastBatchLSN(UndoPersistenceLevel plevel); + +#endif /* XACTUNDO_H */ diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index bf609c3c703d9..949bfd496c81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -32,7 +32,7 @@ /* * Each page of XLOG file has a header like this: */ -#define XLOG_PAGE_MAGIC 0xD122 /* can be used as WAL version indicator */ +#define XLOG_PAGE_MAGIC 0xD12E /* can be used as WAL version indicator */ typedef struct XLogPageHeaderData { diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index b8e9b7f109312..b40c825013c37 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -322,6 +322,7 @@ extern bool BufferBeginSetHintBits(Buffer buffer); extern void BufferFinishSetHintBits(Buffer buffer, bool mark_dirty, bool buffer_std); extern void UnlockBuffers(void); +extern void BufferLockReleaseAll(void); extern void UnlockBuffer(Buffer buffer); extern void LockBufferInternal(Buffer buffer, BufferLockMode mode); diff --git a/src/include/storage/subsystemlist.h b/src/include/storage/subsystemlist.h index 9ad619080be22..9b995f9bad799 100644 --- a/src/include/storage/subsystemlist.h +++ b/src/include/storage/subsystemlist.h @@ -88,3 +88,6 @@ PG_SHMEM_SUBSYSTEM(DataChecksumsShmemCallbacks) /* AIO subsystem. This delegates to the method-specific callbacks */ PG_SHMEM_SUBSYSTEM(AioShmemCallbacks) + +/* UNDO subsystem */ +PG_SHMEM_SUBSYSTEM(UndoShmemCallbacks) diff --git a/src/tools/pginclude/headerscheck b/src/tools/pginclude/headerscheck index 785d6f867ad09..29c63abfb2ba8 100755 --- a/src/tools/pginclude/headerscheck +++ b/src/tools/pginclude/headerscheck @@ -129,6 +129,7 @@ do # These files are not meant to be included standalone, because # they contain lists that might have multiple use-cases. test "$f" = src/include/access/rmgrlist.h && continue + test "$f" = src/include/access/undormgrlist.h && continue test "$f" = src/include/parser/kwlist.h && continue test "$f" = src/include/postmaster/proctypelist.h && continue test "$f" = src/include/regex/regerrs.h && continue diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f10eb46926676..bdaea778ee5bb 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -618,6 +618,7 @@ DBState DCHCacheEntry DEADLOCK_INFO DECountItem +DeferredUndoXact DH DIR DNSServiceErrorType @@ -3312,6 +3313,16 @@ UINT ULARGE_INTEGER ULONG ULONG_PTR +UndoApplyResult +UndoBatchData +UndoPersistenceLevel +UndoRecordHeader +UndoRecordSet +UndoRecoveryEntry +UndoRecPtr +UndoRmgrApplyFunc +UndoRmgrData +UndoRmgrDescFunc UV UVersionInfo UnicodeNormalizationForm @@ -3492,6 +3503,14 @@ X509_NAME_ENTRY X509_STORE X509_STORE_CTX X86FeatureId +XactUndoContext +XactUndoData +XactUndoInflightSlot +XactUndoSubTransactionState +xl_undo_batch +xl_undo_inflight +xl_undo_inflight_xact +xl_undo_rollback XLTW_Oper XLogCtlData XLogCtlInsert From 0fec96965fabb0d240997fb2ce1e8dc837caaa31 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 14 Sep 2026 13:03:40 +0000 Subject: [PATCH 4/5] Add transactional filesystem operations (FILEOPS) backed by UNDO Make the structural filesystem mutations the server performs outside the buffer manager crash-safe and rollback-safe via a WAL resource manager (RM_FILEOPS_ID). Forward operations are WAL-logged and, inside a transaction, register a pending-op / UNDO record so an abort or crash recovery recursively reverts them. FILEOPS records its UNDO in the common WAL stream, so crash recovery reverts a loser transaction's structural changes through the cluster-wide UNDO engine's recovery pass; the AtPrepare path is handled so a prepared transaction's pending file operations survive 2PC. This commit adds the FILEOPS access method and its UNDO plumbing but does not yet route any in-tree DDL/storage caller through it (that is the next commit): the FileOps* API, fileops_redo, the fileops UNDO resource manager (FileopsUndoRmgrInit, registered via undormgrlist.h), the pg_xattr portability wrapper, rmgrdesc + pg_waldump support, the test_fileops module, SQL regression coverage, and recovery TAP tests for crash and 2PC. Author: Greg Burd Reviewed-by: shihao zhong Reviewed-by: Manuel Reyes Bravo Discussion: https://postgr.es/m/7BNP_kRdfRAWyq4swht7lEjtTjEyIPgbDLkR7PXvp9qgNPfXb_fbg4k8ceAd6j-4genQldy9CY9j9KthZUI0SducvaQsxSDoa51e0GdEtQo=@burd.me Changes since v2: - Report FILEOPS UNDO apply failures as failures [f3bd4a3cc7] Tests: - Do not set logical_revert_naptime in 059 and 061 [3072609035] - Show that a crash right after commit loses deferred ops [b34b2463bb] Changes since v7: - recovery/060_fileops_standby_divergence is now 064: upstream added t/060_multixact_crashed_updater.pl - test_fileops gains 023 and 024, the tests for the two UNDO engine fixes above (in the previous commit) - recovery/072_perbackend_crash_undo is now 065_fileops_crash_undo; it never involved per-backend UNDO - Comments and tests no longer describe ROLLBACK PREPARED as handing the chain to the ATM and an asynchronous revert worker; in this series FinishPreparedTransaction() applies it synchronously - recovery/062 (Test 12) and 065 skip their file-mode checks on Windows, which has no POSIX modes; v8 carried these guards in 0005 - recovery/065 crashes the server before it closes the client. It closed the client first, so the backend could roll the transaction back online, and when that abort reached disk before the crash, recovery redid the changes Changes since v9 (the v8 fixes that the cut to three commits had lost): - An ordinary filesystem error is an ERROR again, not a PANIC. Each immediate operation makes its system call first and WAL-logs it only once it has succeeded; v9 ran the call inside the critical section after the WAL flush, so CREATE TABLESPACE on a directory the server cannot chmod took the server down, as in Shihao's v2 review (test_fileops/004). The window this leaves between the call and its WAL record is documented in fileops.c and the README - The abort record carries the transaction's FILEOPS inverses (XACT_XINFO_HAS_FILEOPS), and abort redo applies them. An online ROLLBACK reverted the files in the backend only, so a physical standby kept a change the primary rolled back, and so did the primary after a crash once the abort record was on disk (test_fileops/008). ROLLBACK PREPARED carries them too, read from the 2PC state, and keeps upstream's DELAY_CHKPT_START set until they are durable - FileOpsRename() renames at once and is undone by renaming back, instead of being deferred to commit (test_fileops/018) - An inverse reverts only an object that still shows the operation's effect, so a name reused since is left alone; a rollback's inverses are made durable once per path; the abort data a transaction may accumulate is capped (FILEOPS_MAX_ABORT_BYTES), so the abort record cannot fail - FileOpsMkdirTree() creates a directory that its UNDO removes with everything in it (used by the next commit for CREATE DATABASE) Changes since v10: - FileOpsCreate() opens with O_EXCL. It wrote the create's UNDO even when the file existed, so a rollback unlinked a committed file, online, on a standby and after a crash (Manu; test_fileops/025) - test_fileops/026 crashes after a ROLLBACK or ROLLBACK PREPARED has flushed its abort record and before it applies the inverses, with a CHECKPOINT requested in between. It fails without the checkpoint hold-off in either path (Manu) - test_fileops/028 stops a backend between inserting an UNDO batch and publishing its chain head (injection point undo-batch-after-wal-insert), then checkpoints and crashes. It fails without the hold-off in InsertXactUndoData() (Manu, whose repro drove gdb) - test_fileops/024 has two failing inverses in one loser, so it checks that the undo phase counts failures rather than flags one (Manu) - test_fileops/027: a crash loser's create and mkdir are gone on a streaming standby, not only on the primary (Manu) - The undo phase makes each FILEOPS inverse durable before it logs the loser's rollback. v8 did this at the end of the chain walk, which v10 no longer had. The inode checks that redo skips are skipped in the undo phase too, which now runs after InRecovery is cleared - test_fileops/029: a crash loser whose UNDO batch a standby removed at a restartpoint is rolled back on the standby. 030: the standby keeps a change the primary could not roll back, and a second crash leaves a committed reuse of the loser's name alone - typedefs.list gains FileOpsSyncEntry and FileopsUndoWrite; pgindent - Register RM_FILEOPS_ID unconditionally, and drop the USE_FILEOPS macro that only guarded it; sort the new rmgr includes - Remove FileOpsCancelPendingDelete(), FileOpsSync() and FileOpsRmdirRecursive(), which nothing calls - recovery/061-065 are now 062-066: upstream added t/061_recovery_target_prefetch.pl --- src/backend/access/rmgrdesc/Makefile | 1 + src/backend/access/rmgrdesc/fileopsdesc.c | 220 ++ src/backend/access/rmgrdesc/meson.build | 1 + src/backend/access/rmgrdesc/xactdesc.c | 8 + src/backend/access/transam/rmgr.c | 1 + src/backend/access/transam/twophase.c | 89 +- src/backend/access/transam/twophase_rmgr.c | 20 +- src/backend/access/transam/xact.c | 68 +- src/backend/storage/file/Makefile | 2 + src/backend/storage/file/README | 272 ++ src/backend/storage/file/fileops.c | 2471 +++++++++++++++++ src/backend/storage/file/fileops_undo.c | 1065 +++++++ src/backend/storage/file/meson.build | 2 + src/bin/pg_waldump/rmgrdesc.c | 1 + src/bin/pg_waldump/t/001_basic.pl | 3 +- src/include/access/fileops_xlog.h | 40 + src/include/access/rmgrlist.h | 1 + src/include/access/twophase_rmgr.h | 9 +- src/include/access/undormgrlist.h | 3 + src/include/access/xact.h | 8 +- src/include/port/pg_xattr.h | 40 + src/include/storage/fileops.h | 519 ++++ src/port/Makefile | 1 + src/port/meson.build | 1 + src/port/pg_xattr.c | 217 ++ src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_fileops/Makefile | 19 + .../test_fileops/expected/test_fileops.out | 413 +++ .../test_fileops/expected/test_fileops_1.out | 332 +++ .../test_fileops/expected/test_fileops_2.out | 18 + src/test/modules/test_fileops/meson.build | 54 + .../modules/test_fileops/sql/test_fileops.sql | 229 ++ .../t/001_fileops_apply_failure.pl | 38 + .../test_fileops/t/004_fileops_fs_errors.pl | 107 + .../t/005_fileops_checkpoint_loser.pl | 38 + .../t/007_fileops_commit_crash.pl | 77 + .../t/008_fileops_standby_abort.pl | 89 + .../test_fileops/t/018_fileops_rename.pl | 347 +++ .../t/023_fileops_inline_abort_ctx.pl | 60 + .../t/024_fileops_recovery_apply_failure.pl | 64 + .../t/025_fileops_create_existing.pl | 56 + .../t/026_fileops_abort_checkpoint_order.pl | 77 + .../t/027_fileops_crash_loser_standby.pl | 64 + .../t/028_fileops_insert_holdoff.pl | 67 + .../t/029_fileops_crash_loser_recycled_wal.pl | 53 + .../t/030_fileops_rollback_record.pl | 74 + .../test_fileops/test_fileops--1.0.sql | 99 + src/test/modules/test_fileops/test_fileops.c | 471 ++++ .../modules/test_fileops/test_fileops.control | 4 + src/test/recovery/Makefile | 1 + src/test/recovery/meson.build | 5 + .../recovery/t/062_fileops_2pc_deferred.pl | 172 ++ src/test/recovery/t/063_fileops_recovery.pl | 423 +++ src/test/recovery/t/064_fileops_2pc.pl | 149 + .../t/065_fileops_standby_divergence.pl | 124 + src/test/recovery/t/066_fileops_crash_undo.pl | 137 + src/tools/pgindent/typedefs.list | 28 + 58 files changed, 8931 insertions(+), 23 deletions(-) create mode 100644 src/backend/access/rmgrdesc/fileopsdesc.c create mode 100644 src/backend/storage/file/README create mode 100644 src/backend/storage/file/fileops.c create mode 100644 src/backend/storage/file/fileops_undo.c create mode 100644 src/include/access/fileops_xlog.h create mode 100644 src/include/port/pg_xattr.h create mode 100644 src/include/storage/fileops.h create mode 100644 src/port/pg_xattr.c create mode 100644 src/test/modules/test_fileops/Makefile create mode 100644 src/test/modules/test_fileops/expected/test_fileops.out create mode 100644 src/test/modules/test_fileops/expected/test_fileops_1.out create mode 100644 src/test/modules/test_fileops/expected/test_fileops_2.out create mode 100644 src/test/modules/test_fileops/meson.build create mode 100644 src/test/modules/test_fileops/sql/test_fileops.sql create mode 100644 src/test/modules/test_fileops/t/001_fileops_apply_failure.pl create mode 100644 src/test/modules/test_fileops/t/004_fileops_fs_errors.pl create mode 100644 src/test/modules/test_fileops/t/005_fileops_checkpoint_loser.pl create mode 100644 src/test/modules/test_fileops/t/007_fileops_commit_crash.pl create mode 100644 src/test/modules/test_fileops/t/008_fileops_standby_abort.pl create mode 100644 src/test/modules/test_fileops/t/018_fileops_rename.pl create mode 100644 src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl create mode 100644 src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl create mode 100644 src/test/modules/test_fileops/t/025_fileops_create_existing.pl create mode 100644 src/test/modules/test_fileops/t/026_fileops_abort_checkpoint_order.pl create mode 100644 src/test/modules/test_fileops/t/027_fileops_crash_loser_standby.pl create mode 100644 src/test/modules/test_fileops/t/028_fileops_insert_holdoff.pl create mode 100644 src/test/modules/test_fileops/t/029_fileops_crash_loser_recycled_wal.pl create mode 100644 src/test/modules/test_fileops/t/030_fileops_rollback_record.pl create mode 100644 src/test/modules/test_fileops/test_fileops--1.0.sql create mode 100644 src/test/modules/test_fileops/test_fileops.c create mode 100644 src/test/modules/test_fileops/test_fileops.control create mode 100644 src/test/recovery/t/062_fileops_2pc_deferred.pl create mode 100644 src/test/recovery/t/063_fileops_recovery.pl create mode 100644 src/test/recovery/t/064_fileops_2pc.pl create mode 100644 src/test/recovery/t/065_fileops_standby_divergence.pl create mode 100644 src/test/recovery/t/066_fileops_crash_undo.pl diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile index bf6709e738d99..5b0d8922ea272 100644 --- a/src/backend/access/rmgrdesc/Makefile +++ b/src/backend/access/rmgrdesc/Makefile @@ -13,6 +13,7 @@ OBJS = \ clogdesc.o \ committsdesc.o \ dbasedesc.o \ + fileopsdesc.o \ genericdesc.o \ gindesc.o \ gistdesc.o \ diff --git a/src/backend/access/rmgrdesc/fileopsdesc.c b/src/backend/access/rmgrdesc/fileopsdesc.c new file mode 100644 index 0000000000000..ba4cb111c372e --- /dev/null +++ b/src/backend/access/rmgrdesc/fileopsdesc.c @@ -0,0 +1,220 @@ +/*------------------------------------------------------------------------- + * + * fileopsdesc.c + * rmgr descriptor routines for storage/file/fileops.c + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/rmgrdesc/fileopsdesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "storage/fileops.h" + +void +fileops_desc(StringInfo buf, XLogReaderState *record) +{ + char *data = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_FILEOPS_CREATE: + { + xl_fileops_create *xlrec = (xl_fileops_create *) data; + const char *path = data + SizeOfFileOpsCreate; + + appendStringInfo(buf, "create \"%s\" flags 0x%x mode 0%o", + path, xlrec->flags, xlrec->mode); + } + break; + + case XLOG_FILEOPS_WRITE: + { + xl_fileops_write *xlrec = (xl_fileops_write *) data; + const char *path = data + SizeOfFileOpsWrite; + + appendStringInfo(buf, "write \"%s\" offset %lld len %u", + path, (long long) xlrec->offset, xlrec->len); + } + break; + + case XLOG_FILEOPS_RENAME: + { + xl_fileops_rename *xlrec = (xl_fileops_rename *) data; + const char *oldpath = data + SizeOfFileOpsRename; + const char *newpath = oldpath + xlrec->oldpath_len; + + appendStringInfo(buf, "rename \"%s\" to \"%s\"", + oldpath, newpath); + } + break; + + case XLOG_FILEOPS_DELETE: + { + xl_fileops_delete *xlrec = (xl_fileops_delete *) data; + const char *path = data + SizeOfFileOpsDelete; + + appendStringInfo(buf, "delete \"%s\" at_%s", + path, + xlrec->at_commit ? "commit" : "abort"); + } + break; + + case XLOG_FILEOPS_SYMLINK: + { + xl_fileops_symlink *xlrec = (xl_fileops_symlink *) data; + const char *target = data + SizeOfFileOpsSymlink; + const char *linkpath = target + xlrec->target_len; + + appendStringInfo(buf, "symlink \"%s\" -> \"%s\"", + linkpath, target); + } + break; + + case XLOG_FILEOPS_LINK: + { + xl_fileops_link *xlrec = (xl_fileops_link *) data; + const char *oldpath = data + SizeOfFileOpsLink; + const char *newpath = oldpath + xlrec->oldpath_len; + + appendStringInfo(buf, "link \"%s\" -> \"%s\"", + newpath, oldpath); + } + break; + + case XLOG_FILEOPS_MKDIR: + { + xl_fileops_mkdir *xlrec = (xl_fileops_mkdir *) data; + const char *path = data + SizeOfFileOpsMkdir; + + appendStringInfo(buf, "mkdir \"%s\" mode 0%o%s", + path, (unsigned int) xlrec->mode, + (xlrec->flags & FILEOPS_MKDIR_TREE) ? " tree" : ""); + } + break; + + case XLOG_FILEOPS_RMDIR: + { + xl_fileops_rmdir *xlrec = (xl_fileops_rmdir *) data; + const char *path = data + SizeOfFileOpsRmdir; + + appendStringInfo(buf, "rmdir \"%s\" at_%s", + path, + xlrec->at_commit ? "commit" : "abort"); + } + break; + + case XLOG_FILEOPS_CHMOD: + { + xl_fileops_chmod *xlrec = (xl_fileops_chmod *) data; + const char *path = data + SizeOfFileOpsChmod; + + appendStringInfo(buf, "chmod \"%s\" mode 0%o", + path, (unsigned int) xlrec->mode); + } + break; + + case XLOG_FILEOPS_CHOWN: + { + xl_fileops_chown *xlrec = (xl_fileops_chown *) data; + const char *path = data + SizeOfFileOpsChown; + + appendStringInfo(buf, "chown \"%s\" uid %d gid %d", + path, (int) xlrec->uid, (int) xlrec->gid); + } + break; + + case XLOG_FILEOPS_TRUNCATE: + { + xl_fileops_truncate *xlrec = (xl_fileops_truncate *) data; + const char *path = data + SizeOfFileOpsTruncate; + + appendStringInfo(buf, "truncate \"%s\" to %lld bytes", + path, (long long) xlrec->length); + } + break; + + case XLOG_FILEOPS_SETXATTR: + { + xl_fileops_setxattr *xlrec = (xl_fileops_setxattr *) data; + const char *path = data + SizeOfFileOpsSetxattr; + const char *name = path + xlrec->path_len; + + appendStringInfo(buf, "setxattr \"%s\" name \"%s\" len %u", + path, name, xlrec->value_len); + } + break; + + case XLOG_FILEOPS_REMOVEXATTR: + { + xl_fileops_removexattr *xlrec = + (xl_fileops_removexattr *) data; + const char *path = data + SizeOfFileOpsRemovexattr; + const char *name = path + xlrec->path_len; + + appendStringInfo(buf, "removexattr \"%s\" name \"%s\"", + path, name); + } + break; + + default: + appendStringInfo(buf, "unknown fileops op code %u", info); + break; + } +} + +const char * +fileops_identify(uint8 info) +{ + const char *id = NULL; + + switch (info & ~XLR_INFO_MASK) + { + case XLOG_FILEOPS_CREATE: + id = "CREATE"; + break; + case XLOG_FILEOPS_DELETE: + id = "DELETE"; + break; + case XLOG_FILEOPS_RENAME: + id = "RENAME"; + break; + case XLOG_FILEOPS_WRITE: + id = "WRITE"; + break; + case XLOG_FILEOPS_TRUNCATE: + id = "TRUNCATE"; + break; + case XLOG_FILEOPS_CHMOD: + id = "CHMOD"; + break; + case XLOG_FILEOPS_CHOWN: + id = "CHOWN"; + break; + case XLOG_FILEOPS_MKDIR: + id = "MKDIR"; + break; + case XLOG_FILEOPS_RMDIR: + id = "RMDIR"; + break; + case XLOG_FILEOPS_SYMLINK: + id = "SYMLINK"; + break; + case XLOG_FILEOPS_LINK: + id = "LINK"; + break; + case XLOG_FILEOPS_SETXATTR: + id = "SETXATTR"; + break; + case XLOG_FILEOPS_REMOVEXATTR: + id = "REMOVEXATTR"; + break; + } + + return id; +} diff --git a/src/backend/access/rmgrdesc/meson.build b/src/backend/access/rmgrdesc/meson.build index d0dc4cb229a18..91c6dd7da26e9 100644 --- a/src/backend/access/rmgrdesc/meson.build +++ b/src/backend/access/rmgrdesc/meson.build @@ -6,6 +6,7 @@ rmgr_desc_sources = files( 'clogdesc.c', 'committsdesc.c', 'dbasedesc.c', + 'fileopsdesc.c', 'genericdesc.c', 'gindesc.c', 'gistdesc.c', diff --git a/src/backend/access/rmgrdesc/xactdesc.c b/src/backend/access/rmgrdesc/xactdesc.c index 96aa6e17bd50c..f416e2d09f1e2 100644 --- a/src/backend/access/rmgrdesc/xactdesc.c +++ b/src/backend/access/rmgrdesc/xactdesc.c @@ -230,6 +230,12 @@ ParseAbortRecord(uint8 info, xl_xact_abort *xlrec, xl_xact_parsed_abort *parsed) data += sizeof(xl_xact_origin); } + + if (parsed->xinfo & XACT_XINFO_HAS_FILEOPS) + { + memcpy(&parsed->fileops_len, data, sizeof(uint32)); + parsed->fileops = data + sizeof(uint32); + } } /* @@ -391,6 +397,8 @@ xact_desc_abort(StringInfo buf, uint8 info, xl_xact_abort *xlrec, ReplOriginId o } xact_desc_stats(buf, "", parsed.nstats, parsed.stats); + if (parsed.fileops_len) + appendStringInfo(buf, "; fileops undo: %u bytes", parsed.fileops_len); } static void diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c index aa427b3a3b09d..22ef4170d25e7 100644 --- a/src/backend/access/transam/rmgr.c +++ b/src/backend/access/transam/rmgr.c @@ -39,6 +39,7 @@ #include "replication/decode.h" #include "replication/message.h" #include "replication/origin.h" +#include "storage/fileops.h" #include "storage/standby.h" #include "utils/relmapper.h" /* IWYU pragma: end_keep */ diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 6b1b466d859b5..088b6a1584053 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -100,6 +100,7 @@ #include "replication/origin.h" #include "replication/syncrep.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/ipc.h" #include "storage/md.h" #include "storage/predicate.h" @@ -150,6 +151,15 @@ int max_prepared_xacts = 0; * twophase.h */ +/* + * FILEOPS 2PC prepare hook (fileops.c). Declared locally by prototype to + * avoid pulling storage/fileops.h into this file. Serializes this + * transaction's pending file operations into the 2PC state file so + * COMMIT/ROLLBACK PREPARED -- possibly from another backend after a crash -- + * can perform the deferred commit-time ops or the abort-time filesystem undo. + */ +extern void FileOpsAtPrepare(void); + typedef struct GlobalTransactionData { GlobalTransaction next; /* list link for free list */ @@ -241,7 +251,9 @@ static void RecordTransactionAbortPrepared(TransactionId xid, RelFileLocator *rels, int nstats, xl_xact_stats_item *stats, - const char *gid); + const char *gid, + const char *fileops, + uint32 fileops_len); static void ProcessRecords(char *bufptr, FullTransactionId fxid, const TwoPhaseCallback callbacks[]); static void RemoveGXact(GlobalTransaction gxact); @@ -996,11 +1008,17 @@ TwoPhaseFilePath(char *path, FullTransactionId fxid) * * TWOPHASE_MAGIC must be bumped whenever xl_xact_prepare changes layout. * The struct gained last_batch_lsn[NUndoPersistenceLevels] (24 bytes) for - * UNDO chain tracking across 2PC boundaries, requiring this bump from + * UNDO chain tracking across 2PC boundaries, requiring a bump from * 0x57F94534 to 0x57F94535 to prevent old servers from silently misreading * the variable-length arrays that follow the fixed header at the wrong offsets. + * + * Bumped again to 0x57F94536: FILEOPS now registers TWOPHASE_RM_FILEOPS_ID + * records so deferred commit-time file operations (RENAME/DELETE/RMDIR/RMTREE) + * and abort-time filesystem undo survive PREPARE. This adds a new 2PC record + * type (not a header field), but a bump still keeps a mixed-version cluster + * from replaying a state file whose record stream it does not understand. */ -#define TWOPHASE_MAGIC 0x57F94535 /* format identifier */ +#define TWOPHASE_MAGIC 0x57F94536 /* format identifier */ typedef xl_xact_prepare TwoPhaseFileHeader; @@ -1179,6 +1197,14 @@ EndPrepare(GlobalTransaction gxact) StateFileChunk *record; bool replorigin; + /* + * FILEOPS: persist this transaction's pending file operations into the + * 2PC state file, so COMMIT PREPARED can run them after a crash. Must + * run before the END sentinel while the record chain is still open. A + * no-op if this transaction registered no file operations. + */ + FileOpsAtPrepare(); + /* Add the end sentinel to the list of 2PC records */ RegisterTwoPhaseRecord(TWOPHASE_RM_END_ID, 0, NULL, 0); @@ -1652,12 +1678,38 @@ FinishPreparedTransaction(const char *gid, bool isCommit) hdr->initfileinval, gid); else { + char *fileops = NULL; + uint32 fileops_len = 0; + char *scan = bufptr; + + /* + * The abort decision and the FILEOPS inverses must be in the same WAL + * record, so a standby and crash recovery apply the inverses whenever + * they replay the abort. PREPARE saved them as a 2PC record. + */ + for (;;) + { + TwoPhaseRecordOnDisk *record = (TwoPhaseRecordOnDisk *) scan; + + if (record->rmid == TWOPHASE_RM_END_ID) + break; + scan += MAXALIGN(sizeof(TwoPhaseRecordOnDisk)); + if (record->rmid == TWOPHASE_RM_FILEOPS_ID && + record->info == FILEOPS_TWOPHASE_INVERSES) + { + fileops = scan; + fileops_len = record->len; + break; + } + scan += MAXALIGN(record->len); + } + RecordTransactionAbortPrepared(xid, hdr->nsubxacts, children, hdr->nabortrels, abortrels, hdr->nabortstats, abortstats, - gid); + gid, fileops, fileops_len); /* * ROLLBACK PREPARED: apply any cluster-wide UNDO chain synchronously, @@ -2574,7 +2626,9 @@ RecordTransactionAbortPrepared(TransactionId xid, RelFileLocator *rels, int nstats, xl_xact_stats_item *stats, - const char *gid) + const char *gid, + const char *fileops, + uint32 fileops_len) { XLogRecPtr recptr; bool replorigin; @@ -2619,7 +2673,7 @@ RecordTransactionAbortPrepared(TransactionId xid, nrels, rels, nstats, stats, MyXactFlags | XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK, - xid, gid); + xid, gid, fileops, fileops_len); if (replorigin) /* Move LSNs forward for this replication origin */ @@ -2632,11 +2686,30 @@ RecordTransactionAbortPrepared(TransactionId xid, /* Mark the transaction aborted in clog */ TransactionIdAbortTree(xid, nchildren, children); - /* Checkpoint can proceed now */ - MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; + /* + * Checkpoint can proceed now, unless FILEOPS inverses still have to be + * applied: a checkpoint must not move the redo pointer past this abort + * record until they are durable, so keep it delayed until then. + */ + if (!fileops_len) + MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; END_CRIT_SECTION(); + if (fileops_len) + { + PG_TRY(); + { + INJECTION_POINT("fileops-abort-after-flush", NULL); + FileOpsApplyAbort(fileops, fileops_len); + } + PG_FINALLY(); + { + MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; + } + PG_END_TRY(); + } + /* * Wait for synchronous replication, if required. * diff --git a/src/backend/access/transam/twophase_rmgr.c b/src/backend/access/transam/twophase_rmgr.c index fae254c6e2364..687e77f7e6743 100644 --- a/src/backend/access/transam/twophase_rmgr.c +++ b/src/backend/access/transam/twophase_rmgr.c @@ -19,6 +19,14 @@ #include "pgstat.h" #include "storage/lock.h" #include "storage/predicate.h" +#include "storage/fileops.h" + +/* + * FILEOPS 2PC callbacks (fileops.c). Declared locally by prototype to avoid + * pulling storage/fileops.h in here; the signature is the generic + * TwoPhaseCallback. Postcommit performs the deferred commit-time file ops; + * postabort performs the abort-time filesystem undo. + */ const TwoPhaseCallback twophase_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -27,7 +35,8 @@ const TwoPhaseCallback twophase_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_recover, /* Lock */ NULL, /* pgstat */ multixact_twophase_recover, /* MultiXact */ - predicatelock_twophase_recover /* PredicateLock */ + predicatelock_twophase_recover, /* PredicateLock */ + NULL /* FILEOPS (post-commit/abort only) */ }; const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -36,7 +45,8 @@ const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_postcommit, /* Lock */ pgstat_twophase_postcommit, /* pgstat */ multixact_twophase_postcommit, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + fileops_twophase_postcommit /* FILEOPS */ }; const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -45,7 +55,8 @@ const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_postabort, /* Lock */ pgstat_twophase_postabort, /* pgstat */ multixact_twophase_postabort, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + fileops_twophase_postabort /* FILEOPS */ }; const TwoPhaseCallback twophase_standby_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -54,5 +65,6 @@ const TwoPhaseCallback twophase_standby_recover_callbacks[TWOPHASE_RM_MAX_ID + 1 lock_twophase_standby_recover, /* Lock */ NULL, /* pgstat */ NULL, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + NULL /* FILEOPS */ }; diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 0ad6c376c4542..26f8589a50de8 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -59,6 +59,7 @@ #include "storage/aio_subsys.h" #include "storage/condition_variable.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/lmgr.h" #include "storage/md.h" #include "storage/predicate.h" @@ -1859,6 +1860,8 @@ RecordTransactionAbort(bool isSubXact) TransactionId *children; TimestampTz xact_time; bool replorigin; + char *fileops; + uint32 fileops_len; /* * If we haven't been assigned an XID, nobody will care whether we aborted @@ -1877,9 +1880,9 @@ RecordTransactionAbort(bool isSubXact) /* * We have a valid XID, so we should write an ABORT record for it. * - * We do not flush XLOG to disk here, since the default assumption after a - * crash would be that we aborted, anyway. For the same reason, we don't - * need to worry about interlocking against checkpoint start. + * An ordinary abort need not flush WAL: a crash treats it as a loser. + * FILEOPS inverses, however, must follow a durable abort decision, with + * checkpoint start delayed until their filesystem effects are synced. */ /* @@ -1901,6 +1904,10 @@ RecordTransactionAbort(bool isSubXact) nchildren = xactGetCommittedChildren(&children); ndroppedstats = pgstat_get_transactional_drops(false, &droppedstats); + FileOpsGetAbortData(&fileops, &fileops_len); + if (fileops_len) + FileOpsBeginAbort(); + /* XXX do we really need a critical section here? */ START_CRIT_SECTION(); @@ -1917,7 +1924,7 @@ RecordTransactionAbort(bool isSubXact) nrels, rels, ndroppedstats, droppedstats, MyXactFlags, InvalidTransactionId, - NULL); + NULL, fileops, fileops_len); if (replorigin) /* Move LSNs forward for this replication origin */ @@ -1946,8 +1953,26 @@ RecordTransactionAbort(bool isSubXact) */ TransactionIdAbortTree(xid, nchildren, children); + if (fileops_len) + XLogFlush(XactLastRecEnd); END_CRIT_SECTION(); + if (fileops_len) + { + PG_TRY(); + { + INJECTION_POINT("fileops-abort-after-flush", NULL); + FileOpsApplyAbort(fileops, fileops_len); + } + PG_FINALLY(); + { + FileOpsEndAbort(); + } + PG_END_TRY(); + } + if (fileops) + pfree(fileops); + /* Compute latestXid while we have the child XIDs handy */ latestXid = TransactionIdLatest(xid, nchildren, children); @@ -3040,8 +3065,21 @@ AbortTransaction(void) s->parallelModeLevel = 0; s->parallelChildXact = false; /* should be false already */ - /* Clean up transaction undo state (free per-persistence record sets) */ - AtAbort_XactUndo(); + /* + * Apply the transaction's UNDO, except FILEOPS: its inverses ride in the + * abort record (RecordTransactionAbort), so that a standby and crash + * recovery apply them too. The generic chain is for crash losers. + */ + FileOpsSkipUndo = true; + PG_TRY(); + { + AtAbort_XactUndo(); + } + PG_FINALLY(); + { + FileOpsSkipUndo = false; + } + PG_END_TRY(); /* * do abort processing @@ -6164,7 +6202,8 @@ XactLogAbortRecord(TimestampTz abort_time, int nrels, RelFileLocator *rels, int ndroppedstats, xl_xact_stats_item *droppedstats, int xactflags, TransactionId twophase_xid, - const char *twophase_gid) + const char *twophase_gid, + const char *fileops, uint32 fileops_len) { xl_xact_abort xlrec; xl_xact_xinfo xl_xinfo; @@ -6179,7 +6218,7 @@ XactLogAbortRecord(TimestampTz abort_time, Assert(CritSectionCount > 0); - xl_xinfo.xinfo = 0; + xl_xinfo.xinfo = fileops_len ? XACT_XINFO_HAS_FILEOPS : 0; /* decide between a plain and 2pc abort */ if (!TransactionIdIsValid(twophase_xid)) @@ -6292,6 +6331,12 @@ XactLogAbortRecord(TimestampTz abort_time, if (xl_xinfo.xinfo & XACT_XINFO_HAS_ORIGIN) XLogRegisterData(&xl_origin, sizeof(xl_xact_origin)); + if (fileops_len) + { + XLogRegisterData(&fileops_len, sizeof(fileops_len)); + XLogRegisterData(fileops, fileops_len); + } + /* Include the replication origin */ XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN); @@ -6513,6 +6558,13 @@ xact_redo_abort(xl_xact_parsed_abort *parsed, TransactionId xid, false /* backward */ , false /* WAL */ ); } + if (parsed->fileops_len) + { + XLogFlush(lsn); + INJECTION_POINT("fileops-abort-after-flush", NULL); + FileOpsApplyAbort(parsed->fileops, parsed->fileops_len); + } + /* Make sure files supposed to be dropped are dropped */ if (parsed->nrels > 0) { diff --git a/src/backend/storage/file/Makefile b/src/backend/storage/file/Makefile index 660ac51807e79..8bcfafec7cf7c 100644 --- a/src/backend/storage/file/Makefile +++ b/src/backend/storage/file/Makefile @@ -16,6 +16,8 @@ OBJS = \ buffile.o \ copydir.o \ fd.o \ + fileops.o \ + fileops_undo.o \ fileset.o \ reinit.o \ sharedfileset.o diff --git a/src/backend/storage/file/README b/src/backend/storage/file/README new file mode 100644 index 0000000000000..ece2768ac0af9 --- /dev/null +++ b/src/backend/storage/file/README @@ -0,0 +1,272 @@ +src/backend/storage/file/README + +Transactional File Operations (FILEOPS) +======================================= + +This directory's fileops.c implements FILEOPS: a set of filesystem +operations that participate in PostgreSQL transactions. An ordinary +unlink() or rename() issued in the middle of a transaction is not covered by +WAL and is not reversed if the transaction aborts or the server crashes. +FILEOPS closes that gap for the filesystem mutations the server performs +outside the buffer manager. + +FILEOPS is NOT used for relation files. Creating and dropping tables, +indexes, TRUNCATE, and CREATE/DROP DATABASE|TABLESPACE at the catalog/smgr +level go through smgr and catalog/storage.c, which have zero FileOps calls. +The only in-core callers of the FileOps* API are: + + - dbcommands.c (CREATE/ALTER/DROP DATABASE: directory + PG_VERSION, + ALTER DATABASE ... SET TABLESPACE) + - tablespace.c (CREATE/DROP TABLESPACE: the per-tablespace directory + and symlink; the LOCATION paths are absolute by design) + - copydir.c (the destination tree of a directory copy) + + +When each operation takes effect, and what undoes it +---------------------------------------------------- + +The rule: everything that CREATES or CHANGES an object takes effect at once +and is undone if the transaction aborts; DELETING an object is DEFERRED to +transaction end, so nothing is ever destroyed that a rollback would need. + + operation takes effect undone on abort by (*) + --------------------- ---------------- ------------------------------ + FileOpsCreate now unlink the file + FileOpsMkdir now rmdir the directory + FileOpsMkdirTree now remove the whole tree + FileOpsChmod / Chown now restore the old mode / owner + FileOpsTruncate now restore length and removed bytes + FileOpsWrite now restore the overwritten bytes + FileOpsSymlink / Link now unlink the link + FileOpsRename now rename back + FileOpsSetXattr / now restore the old value + FileOpsRemoveXattr + FileOpsDelete at commit (**) nothing to undo: not done yet + FileOpsRmdir at commit (**) nothing to undo: not done yet + + (*) at wal_level >= replica, by the operation's UNDO record: on ROLLBACK, + ROLLBACK TO SAVEPOINT and ROLLBACK PREPARED (applied with the abort + record, which carries the inverses), on a standby replaying that + abort record, and by the startup undo phase for a transaction a crash + left in flight. At wal_level = minimal, by an abort-time pending op + on ROLLBACK and ROLLBACK TO SAVEPOINT only (see below). + (**) or at abort, with at_commit = false. + +Each undo reverts only an object that still shows the operation's effect +(fileops_undo_apply() in fileops_undo.c), so a name reused in the meantime is +left alone. Undo runs newest first, so an operation on a renamed file's new +name is undone before the rename. + + +Immediate operations +-------------------- + +These run when the API is called, so the file or directory can be used +within the same transaction, and each registers an abort-time undo action +that reverses it if the transaction rolls back: + + FileOpsCreate -- create() now, an ERROR if the file exists; + UNDO/pending-delete on abort + FileOpsMkdir -- mkdir() now; UNDO rmdir on abort + FileOpsMkdirTree -- mkdir() now; UNDO removes the whole tree on abort + FileOpsChmod -- chmod() now; UNDO restore old mode on abort + FileOpsChown -- chown() now; UNDO restore old uid/gid on abort + FileOpsTruncate -- ftruncate() now; UNDO restore old length on abort + FileOpsSymlink -- symlink() now; UNDO unlink on abort + FileOpsLink -- link() now; UNDO unlink on abort + FileOpsRename -- rename() now; UNDO rename back on abort + FileOpsWrite -- pwrite() now (WAL-logged, bounded before-image UNDO) + FileOpsSetXattr / FileOpsRemoveXattr -- now; UNDO restore old value on abort + +FileOpsRename(oldpath, newpath) renames a file or directory. It never +replaces newpath: an existing newpath is an ERROR (ERRCODE_DUPLICATE_FILE), +as are a missing oldpath and a newpath on another filesystem, and each +changes nothing. Undoing a replacement would need the replaced object kept +somewhere until the transaction ends, and no caller needs one. The rename +is reversed only while newpath exists and oldpath is free. Like the undo of +FileOpsCreate and FileOpsMkdir, that test looks at the paths alone: the +device and inode of the object are not stable across a restore, a promoted +standby, or redo that makes the object again, so ROLLBACK PREPARED on a +promoted standby could not use them. +Unlike the other immediate operations it writes and flushes its WAL and UNDO +records BEFORE the rename() call, so a crash cannot lose the old name with +nothing on record: a crash before the call is harmless, since redo makes the +rename and the loser's UNDO reverses it. + + +Deferred operations +------------------- + +The deleting operations only record a PendingFileOp, and the filesystem is +touched at transaction end: + + FileOpsDelete -- deferred; at_commit chooses commit vs abort + FileOpsRmdir -- deferred; at_commit chooses commit vs abort + +FileOpsDoPendingOps(isCommit) runs the pending list at transaction end: the +at_commit=true ops fire on commit, the at_commit=false ops fire on abort. + + +When FILEOPS is durable: only at wal_level >= replica +----------------------------------------------------- + +FILEOPS writes WAL and UNDO records ONLY when XLogIsNeeded() is true, i.e. +wal_level >= replica. At wal_level = minimal it emits no WAL and no UNDO: + + - There is NO crash safety at minimal. The immediate constructive ops + mutate the filesystem with no WAL record to redo, and the only reversal + for a crash-mid-transaction is the in-memory pendingFileOps list, which + does not survive a crash. + + - There is NO 2PC safety at minimal. With nothing serialized, a PREPARE'd + transaction's file operations cannot be reproduced after a crash. + +Every FILEOPS + crash-recovery + 2PC guarantee below assumes wal_level >= +replica, which is what any deployment relying on those guarantees must set. + + +Revert and recovery model (NO fileops_cleanup) +---------------------------------------------- + +There is NO fileops_cleanup() function. The RM_FILEOPS_ID WAL resource +manager registers only rm_redo/rm_desc/rm_identify; its rm_startup and +rm_cleanup slots are NULL (see access/rmgrlist.h). Reversal rides two +mechanisms, both on the COMMON WAL stream: + + 1. Live-transaction abort (in-memory). ROLLBACK runs + FileOpsDoPendingOps(false), executing each constructive op's registered + abort-time undo (delete the created file, restore the old + mode/length/xattr, rmdir the created directory, ...). No WAL needed; + the in-process list has the state. + + 2. Crash / async revert (common-WAL UNDO records). Each constructive op + also prepares an UNDO record (PrepareXactUndoData(UNDO_RMID_FILEOPS,...)) + and inserts it (InsertXactUndoData) in the same critical section as its + forward WAL record. On crash, a loser transaction's UNDO chain is + applied by the cluster-wide UNDO engine's undo phase at the end of + recovery (UndoRecoveryRollBack), which logs the records it applied in + an XLOG_UNDO_ROLLBACK record for standbys and later recoveries to + replay; the apply side is fileops_undo_apply() in fileops_undo.c, + registered through the UNDO rmgr registry (FileopsUndoRmgrInit), NOT + through a WAL rm_cleanup callback. + +Deferred ops emit their forward WAL record at commit-EXECUTION time, inside +FileOpsDoPendingOps() when the op actually happens on the primary, so a +physical standby reproduces it by redo. Their redo handlers are idempotent +(delete of an already-absent file, rmdir of an already-absent directory both +count as success). Only the commit path is logged; an at_commit=false +delete/rmdir run during abort emits no WAL, because a standby must never +apply an aborted transaction's effect. + +Redo replays each record onto the filesystem as it is now, which after a +crash may already include later operations, and so it cannot tell a name +reused later from the original. This is a known limit of FILEOPS redo: +making FILEOPS fully crash-atomic is future work (a resource manager that +logs enough to replay against the state the record was made in). Rename +redo converges on the primary's state as far as the paths allow: + + - oldpath absent: the rename was made already; nothing to do. + - newpath absent: rename oldpath to newpath. + - both present: oldpath was made again by the replay of an earlier record + (the CREATE, MKDIR or LINK that made it since the checkpoint), or by a + later operation, and newpath is the later state. Remove oldpath (a + directory only if it is empty) and keep newpath. If oldpath is a + non-empty directory, leave both, with a WARNING. + +Renaming over newpath instead would lose data. After a -> b, then b -> c, +then a new b, with a older than the checkpoint, redo of b -> c finds the new +b and c both present, and would put the new b in place of c. + +One deliberate exception: + + SYMLINK redo is an intentional no-op. FileOpsSymlink is used only for + tablespace symlinks; standby/crash-recovery replay of those is owned by + RM_TBLSPC, which applies the recovery-time layout remap. + + +Two-phase commit +---------------- + +PREPARE TRANSACTION serializes the transaction's pending FileOps into the +2PC state file so they survive to COMMIT/ROLLBACK PREPARED, including from a +different backend after a crash between PREPARE and the final decision: + + - FileOpsAtPrepare(), called from EndPrepare() between StartPrepare() and + the END sentinel, writes one TWOPHASE_RM_FILEOPS_ID record per pending + at_commit=true op into the durable 2PC state. + + - The FILEOPS 2PC post-commit callback performs those deferred commit-time + ops (delete/rmdir), reusing the same per-op executor as + FileOpsDoPendingOps(). COMMIT PREPARED thus applies them; a crash before + it re-drives them from the durable 2PC state. + + - The immediate ops' inverses are serialized separately, as one + FILEOPS_TWOPHASE_INVERSES record. ROLLBACK PREPARED carries them in its + abort record and applies them before it releases the prepared + transaction's locks, as an ordinary abort does; COMMIT PREPARED ignores + them. The FILEOPS post-abort callback is therefore effectively a no-op. + + - PostPrepare_FileOps() then discards this backend's in-memory list; the + durable 2PC state (for commit-time ops) and the UNDO chain (for abort-time + reversal), not this backend, drive the eventual work. + +Before this, PostPrepare_FileOps() freed the pending list without recording +it, so deferred commit-time ops were lost across PREPARE. Adding the +TWOPHASE_RM_FILEOPS_ID record type bumped TWOPHASE_MAGIC (twophase.c). + + +How to use the API +------------------ + +The public API is declared in src/include/storage/fileops.h. Call these in +place of the raw libc/syscall equivalents whenever the operation must be +transactional. + + register_delete (FileOpsCreate). When true, the newly created file is + registered for deletion if the transaction aborts. + + at_commit (FileOpsDelete, FileOpsRmdir). + at_commit = true removes on commit (dropping state the transaction is + logically deleting); at_commit = false removes only on abort. + +FileOpsMkdirTree creates a directory that the transaction then fills by +other means (smgr for CREATE DATABASE, copydir() for FILE_COPY and ALTER +DATABASE SET TABLESPACE). Its UNDO record and its abort-time pending op +remove the whole tree, whatever ended up in it, where FileOpsMkdir's only +rmdir() an empty directory. For a database directory the UNDO first drops +that database's shared buffers and pending fsync requests. + +The API hides platform differences: PG_O_DIRECT for unbuffered I/O, +pg_fsync() for the correct per-platform flush, and fsync_parent_path() for +directory durability on Unix. + + +Transaction lifecycle integration +---------------------------------- + +The deferred operations are driven by AM-neutral hooks that xact.c calls +(wired to the FileOps* functions in fileops_undo.c): + + FileOpsDoPendingOps(isCommit) -- executes the pending list at end of xact. + AtSubCommit_FileOps() -- reparents a subxact's pending ops. + AtSubAbort_FileOps() -- runs a subxact's abort-time ops. + FileOpsAtPrepare() -- serializes pending ops into 2PC state. + PostPrepare_FileOps() -- discards the in-memory list after PREPARE. + + +WAL replay +---------- + +fileops_redo() dispatches each XLOG_FILEOPS_* record to the matching action +during recovery; fileops_desc()/fileops_identify() render records for +pg_waldump. Because the redo handler is itself the recovery action, code +paths that run under WAL replay (the "redo" arms in dbcommands.c, +tablespace.c, and copydir.c) call the raw syscalls directly rather than +re-entering the FileOps* API. + +Tests: src/test/recovery/t/063_fileops_recovery.pl (crash recovery), +064_fileops_2pc.pl (constructive-op 2PC via chmod), +065_fileops_standby_divergence.pl (physical-standby convergence), +062_fileops_2pc_deferred.pl (deferred commit-time ops through PREPARE + +crash), and the test_fileops module's regression test and TAP tests +(018_fileops_rename.pl covers FileOpsRename). diff --git a/src/backend/storage/file/fileops.c b/src/backend/storage/file/fileops.c new file mode 100644 index 0000000000000..045557dccf072 --- /dev/null +++ b/src/backend/storage/file/fileops.c @@ -0,0 +1,2471 @@ +/*------------------------------------------------------------------------- + * + * fileops.c + * Transactional file operations with WAL logging + * + * This module provides transactional filesystem operations that integrate + * with PostgreSQL's WAL and transaction management. It is NOT used for + * relation files (smgr/catalog handle those); the only in-core callers are + * dbcommands.c, tablespace.c, and copydir.c. + * + * Two op classes: + * - IMMEDIATE ops (create, mkdir, chmod, chown, truncate, symlink, link, + * rename, write, setxattr) take effect at once, and each registers an + * abort-time undo action (a common-WAL UNDO record plus, where relevant, + * a pending op) that reverses it on rollback. + * - DESTRUCTIVE ops (delete, rmdir, rmtree) are DEFERRED: a PendingFileOp + * is recorded now and the filesystem is touched at transaction end by + * FileOpsDoPendingOps(). + * + * WAL and UNDO are written only when XLogIsNeeded() (wal_level >= replica); + * at wal_level = minimal there is no crash or 2PC safety. + * + * An immediate op performs its system call FIRST, outside any critical + * section, and WAL-logs itself only once the call has succeeded: the forward + * WAL record and its UNDO record are inserted and flushed together in a short + * critical section (fileops_log()). So an ordinary filesystem error + * (permission denied, no space, ...) is a plain ERROR, as it is for the same + * call without FILEOPS, and a failed op leaves nothing in the WAL for redo or + * a standby to replay. + * + * This is NOT atomic. Between the system call and the WAL flush there is a + * window in which the change is on disk with no record of it: a crash there + * leaves the change in place, unknown to redo, to a standby and to rollback, + * just as the bare system call would. Nothing reverts it. The + * opposite order (log, then act inside a critical section) closes that window + * only by turning every filesystem error into a PANIC. + * + * FileOpsRename() is the exception: there the window would lose the old name + * for good, so it logs first and renames after, outside the critical section. + * A failed rename() is still a plain ERROR, because its inverse renames back + * only while the new name exists and the old one does not (see there). + * + * Reverting checks the object at the recorded path before touching it (see + * fileops_undo.c), but a path, its type and, where recorded, its inode are + * evidence that the object is the one this transaction made, not proof: + * something else can have replaced it in between. + * + * The deferred-operations pattern works as follows: + * 1. A destructive API call records a PendingFileOp entry in a linked list + * 2. At commit/abort time, FileOpsDoPendingOps() executes or discards + * the pending operations based on transaction outcome (a commit-time + * destructive op emits its WAL record at execution time) + * 3. At PREPARE, FileOpsAtPrepare() serializes the pending ops into the + * 2PC state file so COMMIT/ROLLBACK PREPARED can perform them + * + * Subtransaction support: + * - At subtransaction commit, entries are reassigned to the parent level + * - At subtransaction abort, abort-time actions execute immediately + * + * Platform-specific handling: + * - O_DIRECT: Uses PG_O_DIRECT abstraction (Linux native O_DIRECT, + * macOS F_NOCACHE via fcntl, Windows FILE_FLAG_NO_BUFFERING) + * - fsync: Uses pg_fsync() which selects the appropriate mechanism + * (Linux fdatasync, macOS F_FULLFSYNC, Windows FlushFileBuffers, + * BSD fsync) + * - Directory sync: Uses fsync_fname()/fsync_parent_path() which + * handle directory fsync on Unix platforms (not needed on Windows) + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/file/fileops.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#ifdef HAVE_SYS_FCNTL_H +#include +#endif + +/* + * ENODATA is Linux-specific; FreeBSD and other BSDs don't define it. + * When removing extended attributes, ENODATA means "attribute does not + * exist" -- equivalent to ENOATTR on BSDs. + */ +#ifndef ENODATA +#ifdef ENOATTR +#define ENODATA ENOATTR +#else +#define ENODATA ENOENT +#endif +#endif + +#include "access/fileops_xlog.h" +#include "access/rmgr.h" +#include "access/twophase_rmgr.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/xact.h" +#include "access/xactundo.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "catalog/pg_class.h" +#include "miscadmin.h" +#include "port/pg_xattr.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "storage/proc.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" + + +/* Head of the pending file operations linked list */ +static PendingFileOp *pendingFileOps = NULL; +static Size inverseBytes = 0; + +static void FileOpsExecOne(PendingFileOp *pending, bool isCommit, + XLogRecPtr *max_lsn); + +/* + * fileops_fsync_parent -- fsync the parent directory of a file path + * + * This ensures that directory entry changes (create, delete, rename) + * are durable. On Windows, directory fsync is not needed because NTFS + * journals directory entries; fsync_fname_ext() handles this by being + * a no-op for directories on Windows. + */ +static void +fileops_fsync_parent(const char *fname, int elevel) +{ + char parentpath[MAXPGPATH]; + char *sep; + + strlcpy(parentpath, fname, MAXPGPATH); + + sep = strrchr(parentpath, '/'); + if (sep != NULL) + { + /* Got a path component, fsync the directory portion */ + if (sep == parentpath) + parentpath[1] = '\0'; /* root directory */ + else + *sep = '\0'; + + fsync_fname_ext(parentpath, true, true, elevel); + } +} + +/* + * fileops_pathlen - Length of a path including its terminating NUL + * + * WAL and UNDO records store path lengths in a uint16 and the UNDO payload + * buffers are sized for MAXPGPATH, so refuse a longer path rather than + * truncate it (or overrun a buffer). Called before the system call, so a + * refused path changes nothing. + */ +static int +fileops_pathlen(const char *path) +{ + size_t len = strlen(path) + 1; + + /* Reserve room before any mutation; abort itself must never exceed WAL. */ + if (inverseBytes + 4 * MAXPGPATH + 2 * FILEOPS_MAX_DATA_BYTES > + FILEOPS_MAX_ABORT_BYTES) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("too much transactional file operation undo data"))); + + if (len > MAXPGPATH) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("file operation path length %zu exceeds maximum %d", + len, MAXPGPATH))); + return (int) len; +} + +/* + * fileops_log - WAL-log an immediate operation that has already succeeded + * + * The caller has made the filesystem change, prepared its UNDO record (if + * any, undo_ctx != NULL) with PrepareXactUndoData(), and registered the + * forward record's data after XLogBeginInsert(). Insert both and flush them + * through the end of the UNDO batch, so that a crash leaves either both or + * neither. Only this is a critical section; see the file header comment. + */ +static void +fileops_log(uint8 info, XactUndoContext *undo_ctx) +{ + START_CRIT_SECTION(); + (void) XLogInsert(RM_FILEOPS_ID, info); + if (undo_ctx != NULL) + { + InsertXactUndoData(undo_ctx); + CleanupXactUndoInsertion(undo_ctx); + } + XLogFlush(XactLastRecEnd); + END_CRIT_SECTION(); +} + +/* + * AddPendingFileOp - Add a new pending file operation to the list + * + * All fields are deep-copied into TopMemoryContext to survive + * until transaction end, following the PendingRelDelete pattern. + */ +static void +AddPendingFileOp(PendingFileOpType type, const char *path, + const char *newpath, off_t length, bool at_commit) +{ + PendingFileOp *pending; + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(TopMemoryContext); + + pending = (PendingFileOp *) palloc(sizeof(PendingFileOp)); + pending->type = type; + pending->path = pstrdup(path); + pending->newpath = newpath ? pstrdup(newpath) : NULL; + pending->length = length; + pending->data = NULL; + pending->data_len = 0; + pending->at_commit = at_commit; + pending->nestLevel = GetCurrentTransactionNestLevel(); + pending->next = pendingFileOps; + pendingFileOps = pending; + + MemoryContextSwitchTo(oldcxt); +} + +/* + * AddPendingFileOpWithData - Add a pending file operation with extra data + * + * Like AddPendingFileOp but also stores arbitrary data (e.g., original + * xattr value for restore on abort). + */ +static void +AddPendingFileOpWithData(PendingFileOpType type, const char *path, + const char *newpath, off_t length, + const void *data, size_t data_len, + bool at_commit) +{ + PendingFileOp *pending; + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(TopMemoryContext); + + pending = (PendingFileOp *) palloc(sizeof(PendingFileOp)); + pending->type = type; + pending->path = pstrdup(path); + pending->newpath = newpath ? pstrdup(newpath) : NULL; + pending->length = length; + if (data && data_len > 0) + { + pending->data = palloc(data_len); + memcpy(pending->data, data, data_len); + pending->data_len = data_len; + } + else + { + pending->data = NULL; + pending->data_len = 0; + } + pending->at_commit = at_commit; + pending->nestLevel = GetCurrentTransactionNestLevel(); + pending->next = pendingFileOps; + pendingFileOps = pending; + + MemoryContextSwitchTo(oldcxt); +} + +/* Keep the same inverse for online abort, abort redo, and crash losers. */ +static void +fileops_prepare_undo(XactUndoContext *ctx, char persistence, uint8 rmid, + uint16 info, Oid reloid, const char *payload, Size len) +{ + AddPendingFileOpWithData(PENDING_FILEOP_INVERSE, "", NULL, + info, payload, len, false); + inverseBytes += len + 2 * sizeof(uint32); + FileOpsReserveInverseSyncs(); + PrepareXactUndoData(ctx, persistence, rmid, info, reloid, payload, len); +} + +/* Serialize in execution (newest-first) order, without alignment assumptions. */ +void +FileOpsGetAbortData(char **data, uint32 *len) +{ + PendingFileOp *p; + StringInfoData buf; + int level = GetCurrentTransactionNestLevel(); + + *data = NULL; + *len = 0; + if (inverseBytes == 0) + return; + initStringInfo(&buf); + for (p = pendingFileOps; p; p = p->next) + { + uint32 info = p->length; + uint32 size = p->data_len; + + if (p->type != PENDING_FILEOP_INVERSE || p->nestLevel < level) + continue; + appendBinaryStringInfo(&buf, &info, sizeof(info)); + appendBinaryStringInfo(&buf, &size, sizeof(size)); + appendBinaryStringInfo(&buf, p->data, size); + } + *data = buf.data; + *len = buf.len; +} + +/* + * CreateCheckPoint waits for this flag after choosing its redo pointer. Even + * though inverses ride in abort WAL, a checkpoint must not omit that record + * until the corresponding filesystem changes have been fsynced. A database + * directory tree can make this interval long; there is no retry loop here. + */ +void +FileOpsBeginAbort(void) +{ + MyProc->delayChkptFlags |= DELAY_CHKPT_START; +} + +void +FileOpsEndAbort(void) +{ + MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; +} + +/* Apply the self-contained abort payload, also used during WAL replay. */ +void +FileOpsApplyAbort(const char *data, uint32 len) +{ + while (len > 0) + { + uint32 info; + uint32 size; + + if (len < 2 * sizeof(uint32)) + elog(PANIC, "invalid FILEOPS abort payload"); + memcpy(&info, data, sizeof(info)); + memcpy(&size, data + sizeof(info), sizeof(size)); + data += 2 * sizeof(uint32); + len -= 2 * sizeof(uint32); + if (size > len) + elog(PANIC, "invalid FILEOPS inverse length"); + FileOpsApplyInverse(info, data, size); + data += size; + len -= size; + } + FileOpsSyncInverses(PANIC); +} + +/* + * FreePendingFileOp - Free a pending file operation entry + */ +static void +FreePendingFileOp(PendingFileOp *pending) +{ + if (pending->type == PENDING_FILEOP_INVERSE) + inverseBytes -= pending->data_len + 2 * sizeof(uint32); + if (pending->path) + pfree(pending->path); + if (pending->newpath) + pfree(pending->newpath); + if (pending->data) + pfree(pending->data); + pfree(pending); +} + +/* + * FileOpsCreate - Create a file within a transaction + * + * Creates the file immediately (so it can be used within the transaction) + * and logs the creation to WAL. If register_delete is true, the file will + * be deleted if the transaction aborts. + * + * A path that already exists is an ERROR (ERRCODE_DUPLICATE_FILE), as for + * FileOpsMkdir(), FileOpsLink() and FileOpsRename(). The UNDO record + * unlinks the path, so the path must name a file this call created: opening + * an existing file here would let a rollback remove a committed one. + * + * Platform handling: + * - Linux/FreeBSD: O_DIRECT passed directly to open() + * - macOS: F_NOCACHE fcntl applied after open() + * - Windows: FILE_FLAG_NO_BUFFERING (handled by port layer) + * + * Returns the file descriptor; failing to create the file is an ERROR. + */ +int +FileOpsCreate(const char *path, int flags, mode_t mode, bool register_delete) +{ + int fd; + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + fd = OpenTransientFilePerm(path, flags | O_CREAT | O_EXCL, mode); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", path))); + + if (register_delete && !XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_DELETE, path, NULL, 0, false); + + if (enableFsync) + { + if (pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", path))); + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + + if (XLogIsNeeded()) + { + xl_fileops_create xlrec; + char undo_payload[sizeof(FileopsUndoCreate) + MAXPGPATH]; + FileopsUndoCreate *hdr = (FileopsUndoCreate *) undo_payload; + XactUndoContext undo_ctx; + + xlrec.flags = flags; + xlrec.mode = mode; + xlrec.register_delete = register_delete; + + hdr->path_len = (uint16) pathlen; + memcpy(undo_payload + sizeof(FileopsUndoCreate), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CREATE, + InvalidOid, undo_payload, + sizeof(FileopsUndoCreate) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsCreate); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_CREATE, &undo_ctx); + + INJECTION_POINT("fileops-create-after-wal-flush", NULL); + } + + return fd; +} + +/* + * FileOpsDelete - Schedule a file deletion within a transaction + * + * The file is not deleted immediately. Instead, the deletion is deferred + * to transaction commit (if at_commit is true) or abort (if false). + * On Unix: unlink() with parent dir fsync. + * On Windows: pgunlink() with retry on EACCES. + */ +void +FileOpsDelete(const char *path, bool at_commit) +{ + Assert(!IsInParallelMode()); + + /* + * no WAL is emitted at registration time. The deferred delete is + * WAL-logged in FileOpsDoPendingOps() at commit-execution time, when it + * actually happens on the primary, so a physical standby reproduces it + * via redo. Registration-time logging would be a no-op on redo (the + * standby never runs FileOpsDoPendingOps()) and, for at_commit=false + * delete-on-abort registrations, would wrongly apply an aborted + * transaction's effect on the standby. + */ + AddPendingFileOp(PENDING_FILEOP_DELETE, path, NULL, 0, at_commit); +} + +/* + * FileOpsRename - Rename a file or directory within a transaction + * + * Immediate execution: when this returns, newpath exists and oldpath does + * not, and the rename is reversed if the transaction or subtransaction + * aborts (rename back, while newpath exists and oldpath is free). Unlike + * rename(2), it never replaces newpath: rolling back a + * replacement would need the replaced object kept somewhere, and no caller + * needs one. An existing newpath, a missing oldpath and a newpath on another + * filesystem are each an ERROR that changes nothing. The check that newpath + * is free is not atomic with the rename; callers serialize the use of a path, + * as they must for any FILEOPS operation. + * + * Unlike the other immediate operations, the WAL and UNDO records are + * written and flushed BEFORE the system call. The other order would let a + * crash between rename() and the flush lose the old name for good, with + * nothing on record to undo it. In this order a crash before rename() is + * harmless: redo performs the rename and the loser's UNDO reverses it. If + * rename() fails after the WAL, it is an ERROR: the abort's inverse finds + * oldpath still in place and leaves it, and a standby that replayed the + * forward record reverses it when it replays the abort record. + * + * At wal_level = minimal nothing is logged, and an abort-time pending op + * renames the object back. + * + * On Windows, pgrename() retries MoveFileEx() while the file is in use. + * + * Returns 0; any failure is an ERROR. + */ +int +FileOpsRename(const char *oldpath, const char *newpath) +{ + int oldpathlen = fileops_pathlen(oldpath); + int newpathlen = fileops_pathlen(newpath); + struct stat st; + struct stat dst; + char parent[MAXPGPATH]; + + Assert(!IsInParallelMode()); + + if (lstat(oldpath, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", oldpath))); + if (lstat(newpath, &dst) == 0) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_FILE), + errmsg("could not rename file \"%s\" to \"%s\": file already exists", + oldpath, newpath))); + if (errno != ENOENT) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", newpath))); + strlcpy(parent, newpath, sizeof(parent)); + get_parent_directory(parent); + if (parent[0] == '\0') + strlcpy(parent, ".", sizeof(parent)); + if (stat(parent, &dst) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat directory \"%s\": %m", parent))); + if (dst.st_dev != st.st_dev) + { + errno = EXDEV; + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("could not rename file \"%s\" to \"%s\": %m", + oldpath, newpath))); + } + + if (XLogIsNeeded()) + { + xl_fileops_rename xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoRename) + 2 * MAXPGPATH]; + FileopsUndoRename *hdr = (FileopsUndoRename *) payload; + + xlrec.oldpath_len = (uint16) oldpathlen; + + hdr->oldpath_len = (uint16) oldpathlen; + hdr->newpath_len = (uint16) newpathlen; + memcpy(payload + sizeof(FileopsUndoRename), oldpath, oldpathlen); + memcpy(payload + sizeof(FileopsUndoRename) + oldpathlen, newpath, + newpathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_RENAME, + InvalidOid, payload, + sizeof(FileopsUndoRename) + oldpathlen + + newpathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRename); + XLogRegisterData(oldpath, oldpathlen); + XLogRegisterData(newpath, newpathlen); + fileops_log(XLOG_FILEOPS_RENAME, &undo_ctx); + + INJECTION_POINT("fileops-rename-after-wal-flush", NULL); + } + else + { + /* Registered first: it does nothing if the rename does not happen. */ + AddPendingFileOp(PENDING_FILEOP_RENAME, newpath, oldpath, 0, false); + } + + if (rename(oldpath, newpath) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", + oldpath, newpath))); + + if (enableFsync) + { + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + fileops_fsync_parent(oldpath, data_sync_elevel(ERROR)); + } + + return 0; +} + +/* Capture a bounded range before changing it, including shrink tails. */ +static char * +fileops_before_image(int fd, const char *path, off_t offset, Size len) +{ + char *data; + + if (len > FILEOPS_MAX_DATA_BYTES) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("file operation before-image exceeds %d bytes", + FILEOPS_MAX_DATA_BYTES))); + data = palloc(len ? len : 1); + if (len && pg_pread(fd, data, len, offset) != len) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read before-image of file \"%s\": %m", path))); + return data; +} + +/* + * FileOpsWrite - Write data to a file at a specific offset + * + * Immediate execution, WAL-logged for crash recovery replay. + * Uses pwrite() on POSIX. On Windows: SetFilePointerEx + WriteFile. + * + * Bounded before-images restore overwritten bytes and the original length. + * + * Returns 0; a failed write is an ERROR. + */ +int +FileOpsWrite(const char *path, off_t offset, const void *data, uint32 len) +{ + int fd; + ssize_t nbytes; + int pathlen = fileops_pathlen(path); + struct stat st; + char *before; + uint32 before_len; + char *payload; + Size payload_len; + FileopsUndoWrite *hdr; + XactUndoContext undo_ctx; + + Assert(!IsInParallelMode()); + + fd = OpenTransientFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for writing: %m", path))); + + if (offset < 0 || len > FILEOPS_MAX_DATA_BYTES || + offset > PG_INT64_MAX - len) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("invalid or excessive transactional file write range"))); + if (fstat(fd, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", path))); + before_len = offset < st.st_size ? Min((off_t) len, st.st_size - offset) : 0; + before = fileops_before_image(fd, path, offset, before_len); + payload_len = sizeof(FileopsUndoWrite) + pathlen + before_len + len; + payload = palloc0(payload_len); + hdr = (FileopsUndoWrite *) payload; + hdr->orig_length = st.st_size; + hdr->new_length = Max(st.st_size, offset + len); + hdr->offset = offset; + hdr->before_len = before_len; + hdr->after_len = len; + hdr->path_len = pathlen; + memcpy(payload + sizeof(*hdr), path, pathlen); + memcpy(payload + sizeof(*hdr) + pathlen, before, before_len); + memcpy(payload + sizeof(*hdr) + pathlen + before_len, data, len); + + /* Keep an inverse even if pwrite changes only part of the range. */ + if (XLogIsNeeded()) + { + (void) GetCurrentTransactionId(); + AddPendingFileOpWithData(PENDING_FILEOP_INVERSE, "", NULL, + FILEOPS_UNDO_WRITE, payload, payload_len, false); + inverseBytes += payload_len + 2 * sizeof(uint32); + } + + nbytes = pg_pwrite(fd, data, + IS_INJECTION_POINT_ATTACHED("fileops-write-short") ? len / 2 : len, + offset); + if (nbytes != (ssize_t) len) + { + int save_errno = errno; + + /* Match the actual after-image, not the requested complete write. */ + if (XLogIsNeeded()) + { + FileopsUndoWrite *inverse = (FileopsUndoWrite *) pendingFileOps->data; + + inverse->after_len = nbytes > 0 ? nbytes : 0; + inverse->new_length = nbytes > 0 ? + Max(st.st_size, offset + nbytes) : st.st_size; + pendingFileOps->data_len -= len - inverse->after_len; + } + CloseTransientFile(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write %u bytes to file \"%s\" at offset %lld: %m", + len, path, (long long) offset))); + } + + if (enableFsync && pg_fsync(fd) != 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after write: %m", path))); + } + + if (CloseTransientFile(fd) != 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + + if (XLogIsNeeded()) + { + xl_fileops_write xlrec; + + PrepareXactUndoData(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_WRITE, + InvalidOid, payload, payload_len); + xlrec.offset = offset; + xlrec.len = len; + xlrec.path_len = (uint16) pathlen; + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsWrite); + XLogRegisterData(path, pathlen); + XLogRegisterData(data, len); + fileops_log(XLOG_FILEOPS_WRITE, &undo_ctx); + } + + pfree(before); + pfree(payload); + return 0; +} + +/* + * FileOpsTruncate - Truncate a file within a transaction + * + * Executed immediately and WAL-logged. Uses ftruncate() on POSIX, + * SetEndOfFile() on Windows. File is fsynced after truncation. The + * original size is restored if the transaction aborts. + */ +void +FileOpsTruncate(const char *path, off_t length) +{ + int fd; + struct stat st; + off_t orig_length = 0; + int pathlen = fileops_pathlen(path); + char *before; + uint32 before_len; + + Assert(!IsInParallelMode()); + + /* Capture original length for UNDO before truncation */ + if (stat(path, &st) == 0) + orig_length = st.st_size; + + fd = OpenTransientFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for truncation: %m", path))); + + if (length < 0) + ereport(ERROR, (errmsg("negative file truncation length"))); + before_len = orig_length > length ? + Min(orig_length - length, (off_t) FILEOPS_MAX_DATA_BYTES + 1) : 0; + before = fileops_before_image(fd, path, length, before_len); + + if (ftruncate(fd, length) < 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not truncate file \"%s\" to %lld bytes: %m", + path, (long long) length))); + } + + /* + * At wal_level = minimal there is no UNDO record, so register an + * abort-time pending op that restores the original size. + */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_TRUNCATE, path, NULL, orig_length, false); + + if (enableFsync && pg_fsync(fd) != 0) + { + int save_errno = errno; + + CloseTransientFile(fd); + errno = save_errno; + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after truncation: %m", + path))); + } + + if (CloseTransientFile(fd) != 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + + if (XLogIsNeeded()) + { + xl_fileops_truncate xlrec; + XactUndoContext undo_ctx; + char *payload = palloc0(sizeof(FileopsUndoTruncate) + pathlen + before_len); + FileopsUndoTruncate *hdr = (FileopsUndoTruncate *) payload; + + xlrec.length = length; + + hdr->before_len = before_len; + hdr->orig_length = orig_length; + hdr->new_length = length; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoTruncate), path, pathlen); + memcpy(payload + sizeof(FileopsUndoTruncate) + pathlen, before, before_len); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_TRUNCATE, + InvalidOid, payload, + sizeof(FileopsUndoTruncate) + pathlen + before_len); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsTruncate); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_TRUNCATE, &undo_ctx); + pfree(payload); + } + pfree(before); +} + +/* + * FileOpsChmod - Change file permissions within a transaction + * + * Immediate execution, WAL-logged. + * On POSIX: chmod(). On Windows: _chmod() with limited mode bits + * (only _S_IREAD/_S_IWRITE; no group/other). Logs WARNING for + * unsupported mode bits on Windows. + * + * Returns 0 on success, -1 on failure with errno left set, so the caller + * can tell ENOENT from other errors and report it as it would a plain + * chmod() failure (tablespace.c does). Nothing is logged or registered + * when it fails, because nothing changed. + */ +int +FileOpsChmod(const char *path, mode_t mode) +{ + struct stat st; + mode_t orig_mode; + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + /* Capture original mode for UNDO before chmod */ + if (stat(path, &st) != 0) + return -1; /* leave errno set by stat() for the caller */ + orig_mode = st.st_mode & 07777; + + if (chmod(path, mode) < 0) + return -1; /* leave errno set by chmod() for the caller */ + + /* + * Register abort-time pending op to restore original mode on rollback. + * Store mode in the length field (mode_t fits in off_t). + */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_CHMOD, path, NULL, (off_t) orig_mode, false); + + if (XLogIsNeeded()) + { + xl_fileops_chmod xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoChmod) + MAXPGPATH]; + FileopsUndoChmod *hdr = (FileopsUndoChmod *) payload; + + xlrec.mode = mode; + xlrec.orig_mode = orig_mode; + + hdr->orig_mode = orig_mode; + hdr->new_mode = mode & 07777; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoChmod), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CHMOD, + InvalidOid, payload, + sizeof(FileopsUndoChmod) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsChmod); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_CHMOD, &undo_ctx); + + INJECTION_POINT("fileops-chmod-after-wal-flush", NULL); + } + + return 0; +} + +/* + * FileOpsChown - Change file ownership within a transaction + * + * Immediate execution, WAL-logged. + * On POSIX: chown(). On Windows: no-op with WARNING (Windows uses + * ACLs for ownership, not uid/gid). + * + * Returns 0; a failed chown() is an ERROR. + */ +int +FileOpsChown(const char *path, uid_t uid, gid_t gid) +{ + struct stat st; + uid_t orig_uid = (uid_t) -1; + gid_t orig_gid = (gid_t) -1; + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + /* Capture original ownership for UNDO before chown */ + if (stat(path, &st) == 0) + { + orig_uid = st.st_uid; + orig_gid = st.st_gid; + } + +#ifndef WIN32 + if (chown(path, uid, gid) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not chown file \"%s\" to %d:%d: %m", + path, (int) uid, (int) gid))); +#else + ereport(WARNING, + (errmsg("chown is not supported on Windows, skipping for \"%s\"", + path))); + return 0; /* nothing changed, so nothing to log */ +#endif + + if (XLogIsNeeded()) + { + xl_fileops_chown xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoChown) + MAXPGPATH]; + FileopsUndoChown *hdr = (FileopsUndoChown *) payload; + + xlrec.uid = uid; + xlrec.gid = gid; + xlrec.orig_uid = orig_uid; + xlrec.orig_gid = orig_gid; + + hdr->orig_uid = orig_uid; + hdr->orig_gid = orig_gid; + /* chown() leaves an id of -1 unchanged */ + hdr->new_uid = (uid == (uid_t) -1) ? orig_uid : uid; + hdr->new_gid = (gid == (gid_t) -1) ? orig_gid : gid; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoChown), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_CHOWN, + InvalidOid, payload, + sizeof(FileopsUndoChown) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsChown); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_CHOWN, &undo_ctx); + } + + return 0; +} + +/* + * fileops_mkdir - Create a directory within a transaction + * + * Shared by FileOpsMkdir() and FileOpsMkdirTree(); see there. + */ +static int +fileops_mkdir(const char *path, mode_t mode, uint16 flags, Oid dboid) +{ + int pathlen = fileops_pathlen(path); + + Assert(!IsInParallelMode()); + + if (MakePGDirectory(path) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", path))); + + /* Remove the directory (or tree) if the transaction aborts */ + if (!XLogIsNeeded()) + AddPendingFileOp((flags & FILEOPS_MKDIR_TREE) ? + PENDING_FILEOP_RMTREE : PENDING_FILEOP_RMDIR, + path, NULL, 0, false); + + if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + + if (XLogIsNeeded()) + { + xl_fileops_mkdir xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoMkdir) + MAXPGPATH]; + FileopsUndoMkdir *hdr = (FileopsUndoMkdir *) payload; + struct stat st; + + xlrec.mode = mode; + xlrec.flags = flags; + + if (stat(path, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat directory \"%s\": %m", path))); + hdr->dev = st.st_dev; + hdr->ino = st.st_ino; + hdr->dboid = dboid; + hdr->flags = flags; + hdr->path_len = (uint16) pathlen; + memcpy(payload + sizeof(FileopsUndoMkdir), path, pathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_MKDIR, + InvalidOid, payload, + sizeof(FileopsUndoMkdir) + pathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsMkdir); + XLogRegisterData(path, pathlen); + fileops_log(XLOG_FILEOPS_MKDIR, &undo_ctx); + } + + return 0; +} + +/* + * FileOpsMkdir - Create a directory within a transaction + * + * Immediate execution; the directory is removed (rmdir) if the transaction + * aborts, so it must then be empty. + * Uses MakePGDirectory() pattern (mkdir with pg_dir_create_mode). + * On Windows: _mkdir() (no mode parameter, permissions from parent). + * + * Returns 0; a failed mkdir() is an ERROR. + */ +int +FileOpsMkdir(const char *path, mode_t mode) +{ + return fileops_mkdir(path, mode, 0, InvalidOid); +} + +/* + * FileOpsMkdirTree - Create a directory the transaction will fill + * + * Like FileOpsMkdir(), but if the transaction aborts the whole tree is + * removed, including files created in it by other means (smgr, copydir()). + * For a database directory pass its OID as dboid: before the tree is removed + * by the UNDO record, that database's shared buffers and pending fsync + * requests are dropped, as they would be by DROP DATABASE. + */ +int +FileOpsMkdirTree(const char *path, mode_t mode, Oid dboid) +{ + return fileops_mkdir(path, mode, FILEOPS_MKDIR_TREE, dboid); +} + +/* + * FileOpsRmdir - Remove a directory within a transaction + * + * Deferred to commit time (like DELETE). Uses rmdir() on all platforms. + * On Windows: _rmdir(). + */ +void +FileOpsRmdir(const char *path, bool at_commit) +{ + Assert(!IsInParallelMode()); + + /* + * WAL is emitted at commit-execution time in FileOpsDoPendingOps(), not + * at registration, so the standby reproduces the rmdir via redo. + */ + AddPendingFileOp(PENDING_FILEOP_RMDIR, path, NULL, 0, at_commit); +} + +/* + * FileOpsSymlink - Create a symbolic link within a transaction + * + * Immediate execution. Registers delete-on-abort for cleanup. + * On POSIX: symlink(). On Windows: pgsymlink() which creates NTFS + * junction points via DeviceIoControl(). Note: junction points only + * work for directories on Windows. + * + * Returns 0; a failed symlink() is an ERROR. + */ +int +FileOpsSymlink(const char *target, const char *linkpath) +{ + int targetlen = fileops_pathlen(target); + int linkpathlen = fileops_pathlen(linkpath); + + Assert(!IsInParallelMode()); + + if (symlink(target, linkpath) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create symbolic link \"%s\" -> \"%s\": %m", + linkpath, target))); + + /* Register delete-on-abort to clean up the symlink on rollback */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_DELETE, linkpath, NULL, 0, false); + + if (enableFsync) + fileops_fsync_parent(linkpath, data_sync_elevel(ERROR)); + + if (XLogIsNeeded()) + { + xl_fileops_symlink xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoSymlink) + 2 * MAXPGPATH]; + FileopsUndoSymlink *hdr = (FileopsUndoSymlink *) payload; + + xlrec.target_len = (uint16) targetlen; + + /* the target lets the UNDO recognize the link it would remove */ + hdr->linkpath_len = (uint16) linkpathlen; + hdr->target_len = (uint16) targetlen; + memcpy(payload + sizeof(FileopsUndoSymlink), linkpath, linkpathlen); + memcpy(payload + sizeof(FileopsUndoSymlink) + linkpathlen, target, + targetlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_SYMLINK, + InvalidOid, payload, + sizeof(FileopsUndoSymlink) + linkpathlen + + targetlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsSymlink); + XLogRegisterData(target, targetlen); + XLogRegisterData(linkpath, linkpathlen); + fileops_log(XLOG_FILEOPS_SYMLINK, &undo_ctx); + } + + return 0; +} + +/* + * FileOpsLink - Create a hard link within a transaction + * + * Immediate execution. Registers delete-on-abort for cleanup. + * On POSIX: link(). On Windows: CreateHardLinkA() (NTFS only). + * + * Returns 0; a failed link() is an ERROR. + */ +int +FileOpsLink(const char *oldpath, const char *newpath) +{ + int oldpathlen = fileops_pathlen(oldpath); + int newpathlen = fileops_pathlen(newpath); + struct stat st; + + Assert(!IsInParallelMode()); + + /* + * The new link will be this inode; the UNDO removes newpath only while it + * still is. + */ + if (stat(oldpath, &st) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", oldpath))); + + if (link(oldpath, newpath) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create hard link \"%s\" -> \"%s\": %m", + newpath, oldpath))); + + /* Register delete-on-abort to clean up the link on rollback */ + if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_DELETE, newpath, NULL, 0, false); + + if (enableFsync) + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + + if (XLogIsNeeded()) + { + xl_fileops_link xlrec; + XactUndoContext undo_ctx; + char payload[sizeof(FileopsUndoLink) + MAXPGPATH]; + FileopsUndoLink *hdr = (FileopsUndoLink *) payload; + + xlrec.oldpath_len = (uint16) oldpathlen; + + hdr->dev = st.st_dev; + hdr->ino = st.st_ino; + hdr->newpath_len = (uint16) newpathlen; + memcpy(payload + sizeof(FileopsUndoLink), newpath, newpathlen); + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_LINK, + InvalidOid, payload, + sizeof(FileopsUndoLink) + newpathlen); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsLink); + XLogRegisterData(oldpath, oldpathlen); + XLogRegisterData(newpath, newpathlen); + fileops_log(XLOG_FILEOPS_LINK, &undo_ctx); + } + + return 0; +} + +/* + * FileOpsSetXattr - Set an extended attribute on a file + * + * Immediate execution, WAL-logged. + * Uses pg_setxattr() portability layer which handles: + * - Linux: setxattr() + * - macOS: setxattr() with extra options parameter + * - FreeBSD: extattr_set_file() + * - Windows: NTFS Alternate Data Streams + * - Other: returns ENOTSUP + * + * Returns 0 on success. Returns -1 with errno set if the filesystem does + * not support extended attributes (ENOTSUP/EOPNOTSUPP) or refuses this one + * (EPERM/EACCES); any other failure is an ERROR. Nothing is logged or + * registered when it fails, because nothing changed. + */ +int +FileOpsSetXattr(const char *path, const char *name, + const void *value, size_t len) +{ + bool had_value = false; + char *orig_value = NULL; + ssize_t orig_value_len = 0; + int pathlen = fileops_pathlen(path); + ssize_t vlen; + + Assert(!IsInParallelMode()); + + /* + * Capture existing xattr value before overwriting, for the abort-time + * pending op and the UNDO record. If the attribute doesn't exist, + * had_value stays false. + */ + vlen = pg_getxattr(path, name, NULL, 0); + if (vlen > 0) + { + orig_value = (char *) palloc(vlen); + orig_value_len = pg_getxattr(path, name, orig_value, vlen); + if (orig_value_len >= 0) + had_value = true; + else + { + pfree(orig_value); + orig_value = NULL; + orig_value_len = 0; + } + } + else if (vlen == 0) + { + /* Attribute exists but has zero-length value */ + had_value = true; + } + + if (pg_setxattr(path, name, value, len) < 0) + { + int save_errno = errno; + + if (orig_value) + pfree(orig_value); + errno = save_errno; + if (errno == ENOTSUP || errno == EOPNOTSUPP || + errno == EPERM || errno == EACCES) + return -1; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set extended attribute \"%s\" on \"%s\": %m", + name, path))); + } + + /* + * Register abort-time pending op to restore/remove xattr on rollback. If + * had_value: restore original value. If !had_value: remove the xattr. + */ + if (!XLogIsNeeded() && had_value) + AddPendingFileOpWithData(PENDING_FILEOP_SETXATTR, path, name, + 0, orig_value, (size_t) orig_value_len, false); + else if (!XLogIsNeeded()) + AddPendingFileOp(PENDING_FILEOP_REMOVEXATTR, path, name, 0, false); + + if (XLogIsNeeded()) + { + xl_fileops_setxattr xlrec; + int namelen = (int) strlen(name) + 1; + XactUndoContext undo_ctx; + Size payload_len; + char *payload; + FileopsUndoSetxattr *hdr; + + xlrec.path_len = (uint16) pathlen; + xlrec.name_len = (uint16) namelen; + xlrec.value_len = (uint32) len; + xlrec.had_value = had_value; + xlrec.orig_value_len = (uint32) (had_value ? orig_value_len : 0); + + payload_len = sizeof(FileopsUndoSetxattr) + pathlen + namelen + + (had_value ? orig_value_len : 0) + len; + payload = (char *) palloc(payload_len); + hdr = (FileopsUndoSetxattr *) payload; + + hdr->path_len = (uint16) pathlen; + hdr->name_len = (uint16) namelen; + hdr->orig_value_len = (uint32) (had_value ? orig_value_len : 0); + hdr->new_value_len = (uint32) len; + hdr->had_value = had_value; + + memcpy(payload + sizeof(FileopsUndoSetxattr), path, pathlen); + memcpy(payload + sizeof(FileopsUndoSetxattr) + pathlen, name, namelen); + if (had_value && orig_value_len > 0) + memcpy(payload + sizeof(FileopsUndoSetxattr) + pathlen + namelen, + orig_value, orig_value_len); + if (len > 0) + memcpy(payload + sizeof(FileopsUndoSetxattr) + pathlen + namelen + + hdr->orig_value_len, value, len); + + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_SETXATTR, + InvalidOid, payload, payload_len); + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsSetxattr); + XLogRegisterData(path, pathlen); + XLogRegisterData(name, namelen); + XLogRegisterData(value, (uint32) len); + if (had_value && orig_value_len > 0) + XLogRegisterData(orig_value, (uint32) orig_value_len); + fileops_log(XLOG_FILEOPS_SETXATTR, &undo_ctx); + pfree(payload); + } + + if (orig_value) + pfree(orig_value); + + return 0; +} + +/* + * FileOpsRemoveXattr - Remove an extended attribute from a file + * + * Immediate execution, WAL-logged. + * Uses pg_removexattr() portability layer. Removing an attribute that does + * not exist (PG_ENOATTR) succeeds. + * + * Returns 0 on success, -1 if extended attributes are not supported on + * this filesystem/platform (errno left as ENOTSUP/EOPNOTSUPP); any other + * failure is an ERROR. Nothing is logged or registered when it fails. + */ +int +FileOpsRemoveXattr(const char *path, const char *name) +{ + char *saved_value = NULL; + ssize_t saved_value_len = 0; + int pathlen = fileops_pathlen(path); + ssize_t vlen; + + Assert(!IsInParallelMode()); + + /* + * Capture existing xattr value before removal, to restore it on rollback. + * A NULL saved_value means there was none. + */ + vlen = pg_getxattr(path, name, NULL, 0); + if (vlen > 0) + { + saved_value = (char *) palloc(vlen); + saved_value_len = pg_getxattr(path, name, saved_value, vlen); + if (saved_value_len < 0) + { + pfree(saved_value); + saved_value = NULL; + saved_value_len = 0; + } + } + else if (vlen == 0) + { + /* Attribute exists but has zero-length value */ + saved_value_len = 0; + saved_value = (char *) palloc(1); /* non-NULL marker */ + } + + if (pg_removexattr(path, name) < 0 && errno != PG_ENOATTR) + { + int save_errno = errno; + + if (saved_value) + pfree(saved_value); + errno = save_errno; + if (errno == ENOTSUP || errno == EOPNOTSUPP) + return -1; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\": %m", + name, path))); + } + + /* + * Register abort-time pending op to restore the removed xattr on + * rollback. Use SETXATTR type: on abort, set the xattr back to its saved + * value. + */ + if (!XLogIsNeeded() && saved_value != NULL) + AddPendingFileOpWithData(PENDING_FILEOP_SETXATTR, path, name, + 0, saved_value, (size_t) saved_value_len, false); + + if (XLogIsNeeded()) + { + xl_fileops_removexattr xlrec; + int namelen = (int) strlen(name) + 1; + XactUndoContext undo_ctx; + char *payload = NULL; + bool had_value = (saved_value != NULL); + + xlrec.path_len = (uint16) pathlen; + xlrec.name_len = (uint16) namelen; + xlrec.had_value = had_value; + xlrec.orig_value_len = (uint32) (had_value ? saved_value_len : 0); + + if (had_value) + { + Size payload_len; + FileopsUndoRemovexattr *hdr; + + payload_len = sizeof(FileopsUndoRemovexattr) + pathlen + namelen + + saved_value_len; + payload = (char *) palloc(payload_len); + hdr = (FileopsUndoRemovexattr *) payload; + + hdr->path_len = (uint16) pathlen; + hdr->name_len = (uint16) namelen; + hdr->value_len = (uint32) saved_value_len; + + memcpy(payload + sizeof(FileopsUndoRemovexattr), path, pathlen); + memcpy(payload + sizeof(FileopsUndoRemovexattr) + pathlen, + name, namelen); + if (saved_value_len > 0) + memcpy(payload + sizeof(FileopsUndoRemovexattr) + pathlen + namelen, + saved_value, saved_value_len); + + fileops_prepare_undo(&undo_ctx, RELPERSISTENCE_PERMANENT, + UNDO_RMID_FILEOPS, FILEOPS_UNDO_REMOVEXATTR, + InvalidOid, payload, payload_len); + } + + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRemovexattr); + XLogRegisterData(path, pathlen); + XLogRegisterData(name, namelen); + if (had_value && saved_value_len > 0) + XLogRegisterData(saved_value, (uint32) saved_value_len); + fileops_log(XLOG_FILEOPS_REMOVEXATTR, had_value ? &undo_ctx : NULL); + if (payload) + pfree(payload); + } + + if (saved_value) + pfree(saved_value); + + return 0; +} + +/* + * FileOpsDoPendingOps - Execute pending file operations at transaction end + * + * At commit, operations with at_commit=true are executed. + * At abort, operations with at_commit=false are executed. + * + * This is called from xact.c at transaction commit/abort, analogous + * to smgrDoPendingDeletes(). + */ +void +FileOpsDoPendingOps(bool isCommit) +{ + int nestLevel = GetCurrentTransactionNestLevel(); + PendingFileOp *pending; + PendingFileOp *prev; + PendingFileOp *next; + PendingFileOp *exec_head = NULL; + PendingFileOp *exec_tail = NULL; + XLogRecPtr max_lsn = InvalidXLogRecPtr; + + /* + * Pass 1: detach the entries belonging to this (or an inner) nesting + * level from the pending list. Those matching the transaction outcome + * are queued for execution; the rest are discarded. Outer-level entries + * stay on the list for a later call. The queue is built in pending-list + * order, which is newest-first because AddPendingFileOp prepends. + */ + prev = NULL; + for (pending = pendingFileOps; pending != NULL; pending = next) + { + next = pending->next; + + if (pending->nestLevel < nestLevel) + { + /* outer-level entries should not be processed yet */ + prev = pending; + continue; + } + + /* unlink from list first, so we don't retry on failure */ + if (prev) + prev->next = next; + else + pendingFileOps = next; + + if (pending->at_commit == isCommit) + { + /* + * queue for execution, preserving pending-list (newest-first) + * order + */ + pending->next = NULL; + if (exec_tail != NULL) + exec_tail->next = pending; + else + exec_head = pending; + exec_tail = pending; + } + else + FreePendingFileOp(pending); + } + + /* + * Commit-time ops must run in registration order (oldest-first) so that + * nested directory removals proceed deepest-first. The pending list is + * LIFO, so the queue collected above is newest-first; reverse it for + * commit. Abort-time ops must keep LIFO order, which the collected queue + * already preserves, so they are left as-is. + */ + if (isCommit) + { + PendingFileOp *rev = NULL; + + while (exec_head != NULL) + { + next = exec_head->next; + exec_head->next = rev; + rev = exec_head; + exec_head = next; + } + exec_head = rev; + } + + /* Commit record written, nothing executed yet (for tests) */ + if (isCommit && exec_head != NULL) + INJECTION_POINT("fileops-commit-before-pending-ops", NULL); + + /* Pass 2: execute the queued ops in order, freeing each when done. */ + for (pending = exec_head; pending != NULL; pending = next) + { + next = pending->next; + + FileOpsExecOne(pending, isCommit, &max_lsn); + + FreePendingFileOp(pending); + } + + /* + * Each destructive commit op above XLogFlush()es its own record BEFORE + * performing the filesystem mutation (WAL-before-mutation), so no primary + * FS change can outrun its WAL record. This trailing flush is therefore + * redundant, kept only as a no-op safety backstop (max_lsn is already + * durable). Only the commit path ever sets max_lsn. + */ + if (!XLogRecPtrIsInvalid(max_lsn)) + XLogFlush(max_lsn); +} + +/* + * FileOpsExecOne - Execute one pending file operation + * + * Performs a single deferred op (commit- or abort-time) exactly as it would + * run in an ordinary transaction end. Shared by FileOpsDoPendingOps() and by + * the 2PC post-commit/post-abort callbacks, so an op prepared with PREPARE + * TRANSACTION runs identically to one that never left this backend. + * + * *max_lsn accumulates the highest WAL LSN written by a commit-time op so the + * caller can flush it. Each destructive commit op also flushes its own record + * before mutating the filesystem (WAL-before-mutation), so *max_lsn is only a + * backstop. + */ +static void +FileOpsExecOne(PendingFileOp *pending, bool isCommit, XLogRecPtr *max_lsn) +{ + { + switch (pending->type) + { + case PENDING_FILEOP_INVERSE: + break; + case PENDING_FILEOP_CREATE: + /* Creates are executed immediately, nothing to do */ + break; + + case PENDING_FILEOP_RENAME: + + /* + * Abort-time rename back of a FileOpsRename() at wal_level = + * minimal: path is the new name, newpath the old one. Only + * while the new name exists and the old one is free, as the + * UNDO does at higher wal_level. + */ + Assert(!pending->at_commit); + { + struct stat st; + + if (lstat(pending->path, &st) != 0 || + lstat(pending->newpath, &st) == 0) + break; + } + if (rename(pending->path, pending->newpath) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" back to \"%s\": %m", + pending->path, pending->newpath))); + else if (enableFsync) + { + fileops_fsync_parent(pending->newpath, WARNING); + fileops_fsync_parent(pending->path, WARNING); + } + break; + + case PENDING_FILEOP_DELETE: + + /* + * log-at-commit-execution, but ONLY for the commit path. A + * delete-on-abort registration (at_commit=false) executes + * here during abort; that must NOT be WAL-logged, because a + * standby must never apply an aborted transaction's + * filesystem effect. + */ + if (isCommit && XLogIsNeeded()) + { + xl_fileops_delete xlrec; + int pathlen = (int) strlen(pending->path) + 1; + + xlrec.at_commit = true; + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsDelete); + XLogRegisterData(pending->path, pathlen); + *max_lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_DELETE); + + /* + * WAL-before-mutation durability: flush this record + * BEFORE touching the filesystem. A crash after the + * unlink but before a batched flush would mutate the + * primary's FS without the record ever reaching the + * standby, diverging the two. + */ + XLogFlush(*max_lsn); + } + if (unlink(pending->path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\": %m", + pending->path))); + } + else + { + if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + } + break; + + case PENDING_FILEOP_RMDIR: + + /* + * log-at-commit-execution, commit path only (see the DELETE + * case above for the abort-path rationale). + */ + if (isCommit && XLogIsNeeded()) + { + xl_fileops_rmdir xlrec; + int pathlen = (int) strlen(pending->path) + 1; + + xlrec.at_commit = true; + XLogBeginInsert(); + XLogRegisterData(&xlrec, SizeOfFileOpsRmdir); + XLogRegisterData(pending->path, pathlen); + *max_lsn = XLogInsert(RM_FILEOPS_ID, XLOG_FILEOPS_RMDIR); + + /* WAL-before-mutation durability (see DELETE above). */ + XLogFlush(*max_lsn); + } + if (rmdir(pending->path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + pending->path))); + } + else + { + if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + } + break; + + case PENDING_FILEOP_RMTREE: + + /* + * Abort-time removal of a FileOpsMkdirTree() directory and + * whatever ended up in it. The UNDO record, applied earlier + * in the abort, or the caller's own error cleanup has usually + * removed it already. + */ + { + struct stat st; + + if (lstat(pending->path, &st) != 0 || !S_ISDIR(st.st_mode)) + break; + } + if (!rmtree(pending->path, true)) + ereport(WARNING, + (errmsg("could not remove directory tree \"%s\"", + pending->path))); + else if (enableFsync) + fileops_fsync_parent(pending->path, WARNING); + break; + + case PENDING_FILEOP_TRUNCATE: + { + /* + * Restore original file size on abort. The original + * length is stored in pending->length. + */ + int trunc_fd; + + trunc_fd = OpenTransientFile(pending->path, + O_RDWR | PG_BINARY); + if (trunc_fd >= 0) + { + if (ftruncate(trunc_fd, pending->length) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore file \"%s\" to original size %lld: %m", + pending->path, + (long long) pending->length))); + else if (enableFsync && pg_fsync(trunc_fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" after restoring size: %m", + pending->path))); + CloseTransientFile(trunc_fd); + } + else + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" to restore size: %m", + pending->path))); + } + } + break; + + case PENDING_FILEOP_CHMOD: + { + /* + * Restore original file mode on abort. The original mode + * is stored in pending->length (cast from mode_t). + */ + mode_t restore_mode = (mode_t) pending->length; + + if (chmod(pending->path, restore_mode) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore file \"%s\" to original mode 0%o: %m", + pending->path, + (unsigned int) restore_mode))); + } + } + break; + + case PENDING_FILEOP_SETXATTR: + { + /* + * Restore an xattr to its original value on abort. path = + * file, newpath = attr name, data = orig value. + */ + if (pending->data && pending->data_len > 0) + { + if (pg_setxattr(pending->path, pending->newpath, + pending->data, pending->data_len) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore extended attribute \"%s\" on \"%s\": %m", + pending->newpath, + pending->path))); + } + } + else + { + /* Zero-length original value */ + if (pg_setxattr(pending->path, pending->newpath, + "", 0) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore extended attribute \"%s\" on \"%s\": %m", + pending->newpath, + pending->path))); + } + } + } + break; + + case PENDING_FILEOP_REMOVEXATTR: + { + /* + * Remove an xattr on abort (attribute was newly created + * in the aborted transaction). + */ + if (pg_removexattr(pending->path, pending->newpath) < 0) + { + /* + * "Attribute already absent" is success for a + * remove-on-abort (idempotent, like our unlink/rmdir + * redo). Use PG_ENOATTR, the portable + * absent-attribute errno: on Linux that is ENODATA, + * on macOS/BSD ENOATTR -- the raw ENODATA/ENOENT + * check missed the macOS case and warned spuriously + * (sec3 portability bug). + */ + if (errno != ENOENT && errno != PG_ENOATTR) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\": %m", + pending->newpath, + pending->path))); + } + } + break; + + default: + break; + } + } +} + +/* + * AtSubCommit_FileOps - Handle subtransaction commit + * + * Reassign all pending ops from the current nesting level to the parent. + */ +void +AtSubCommit_FileOps(void) +{ + int nestLevel = GetCurrentTransactionNestLevel(); + PendingFileOp *pending; + + for (pending = pendingFileOps; pending != NULL; pending = pending->next) + { + if (pending->nestLevel >= nestLevel) + pending->nestLevel = nestLevel - 1; + } +} + +/* + * AtSubAbort_FileOps - Handle subtransaction abort + * + * Execute abort-time actions for the current nesting level immediately. + */ +void +AtSubAbort_FileOps(void) +{ + FileOpsDoPendingOps(false); +} + +/* + * PostPrepare_FileOps - Clean up after PREPARE TRANSACTION + * + * FileOpsAtPrepare() (called from EndPrepare, before the 2PC END sentinel) + * has already serialized every pending file operation into the two-phase + * state file, so this backend can now discard its in-memory list. The ops + * are performed later by the FILEOPS 2PC post-commit/post-abort callbacks + * (fileops_twophase_postcommit/postabort), possibly from a different backend + * after a crash, driven from the durable 2PC state. + */ +void +PostPrepare_FileOps(void) +{ + PendingFileOp *pending; + PendingFileOp *next; + + for (pending = pendingFileOps; pending != NULL; pending = next) + { + next = pending->next; + pendingFileOps = next; + FreePendingFileOp(pending); + } +} + +/* + * Serialized form of a PendingFileOp for the 2PC state file. + * + * A fixed header followed by two variable-length blobs (path, data), in that + * order. Self-describing so a crash-recovered COMMIT/ROLLBACK + * PREPARED -- run by any backend -- can rebuild an equivalent PendingFileOp + * and hand it to FileOpsExecOne(). Only the fields FileOpsExecOne() reads + * are carried; nestLevel is irrelevant once prepared. + */ +typedef struct FileOpsTwoPhaseRecord +{ + int type; /* PendingFileOpType */ + bool at_commit; /* run at COMMIT PREPARED vs ROLLBACK PREPARED */ + off_t length; /* truncate length / write offset / mode */ + uint32 path_len; /* bytes, including NUL, 0 if none */ + uint32 data_len; /* bytes, 0 if none */ + /* followed by path, data */ +} FileOpsTwoPhaseRecord; + +/* + * FileOpsAtPrepare - Serialize deferred commit-time file ops into 2PC state + * + * Called from EndPrepare() between StartPrepare() and the END sentinel. Each + * pending at_commit=true op (DELETE/RMDIR) is written as one + * TWOPHASE_RM_FILEOPS_ID record. This is what makes those deferred + * commit-time ops survive PREPARE for COMMIT PREPARED across a crash between + * PREPARE and the final decision. + * + * The immediate-operation inverse list is serialized separately. ROLLBACK + * PREPARED includes it in its abort record and applies it before releasing + * the prepared transaction's locks. + */ +void +FileOpsAtPrepare(void) +{ + PendingFileOp *pending; + char *data; + uint32 len; + + FileOpsGetAbortData(&data, &len); + if (len) + RegisterTwoPhaseRecord(TWOPHASE_RM_FILEOPS_ID, + FILEOPS_TWOPHASE_INVERSES, data, len); + if (data) + pfree(data); + + + for (pending = pendingFileOps; pending != NULL; pending = pending->next) + { + FileOpsTwoPhaseRecord rec; + char *buf; + uint32 path_len; + uint32 data_len; + uint32 total; + char *p; + + /* Immediate inverses were serialized as one blob above. */ + if (!pending->at_commit) + continue; + + path_len = pending->path ? (uint32) strlen(pending->path) + 1 : 0; + /* No commit-time op has a newpath. */ + Assert(pending->newpath == NULL); + data_len = (pending->data && pending->data_len > 0) ? + (uint32) pending->data_len : 0; + + rec.type = (int) pending->type; + rec.at_commit = pending->at_commit; + rec.length = pending->length; + rec.path_len = path_len; + rec.data_len = data_len; + + total = sizeof(FileOpsTwoPhaseRecord) + path_len + data_len; + buf = palloc(total); + memcpy(buf, &rec, sizeof(FileOpsTwoPhaseRecord)); + p = buf + sizeof(FileOpsTwoPhaseRecord); + if (path_len > 0) + { + memcpy(p, pending->path, path_len); + p += path_len; + } + if (data_len > 0) + memcpy(p, pending->data, data_len); + + RegisterTwoPhaseRecord(TWOPHASE_RM_FILEOPS_ID, 0, buf, total); + pfree(buf); + } +} + +/* + * fileops_twophase_do - Shared body for the FILEOPS 2PC callbacks + * + * Rebuild a PendingFileOp from the serialized 2PC record and, if it matches + * the transaction outcome, perform it exactly as FileOpsDoPendingOps() would. + * A commit-time op emits its own WAL (idempotent redo on a standby); the + * trailing flush makes any accumulated LSN durable. + */ +static void +fileops_twophase_do(bool isCommit, void *recdata, uint32 len) +{ + FileOpsTwoPhaseRecord rec; + PendingFileOp pending; + const char *p; + XLogRecPtr max_lsn = InvalidXLogRecPtr; + + if (len < sizeof(FileOpsTwoPhaseRecord)) + { + elog(WARNING, "FILEOPS 2PC record too short (%u bytes)", len); + return; + } + memcpy(&rec, recdata, sizeof(FileOpsTwoPhaseRecord)); + + /* Only ops matching this outcome run; the rest are simply discarded. */ + if (rec.at_commit != isCommit) + return; + + memset(&pending, 0, sizeof(pending)); + pending.type = (PendingFileOpType) rec.type; + pending.at_commit = rec.at_commit; + pending.length = rec.length; + + p = (const char *) recdata + sizeof(FileOpsTwoPhaseRecord); + if (rec.path_len > 0) + { + pending.path = (char *) p; + p += rec.path_len; + } + if (rec.data_len > 0) + { + pending.data = (void *) p; + pending.data_len = rec.data_len; + } + + FileOpsExecOne(&pending, isCommit, &max_lsn); + + if (!XLogRecPtrIsInvalid(max_lsn)) + XLogFlush(max_lsn); +} + +/* + * fileops_twophase_postcommit - COMMIT PREPARED callback (perform at_commit ops) + */ +void +fileops_twophase_postcommit(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len) +{ + if (info != FILEOPS_TWOPHASE_INVERSES) + fileops_twophase_do(true, recdata, len); +} + +/* + * fileops_twophase_postabort - ROLLBACK PREPARED callback + * + * Immediate inverses are applied with the abort decision, not here. Deferred + * commit-time operations are discarded. + */ +void +fileops_twophase_postabort(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len) +{ + /* Inverses were applied with the durable abort decision, before unlock. */ + if (info != FILEOPS_TWOPHASE_INVERSES) + fileops_twophase_do(false, recdata, len); +} + +/* + * fileops_redo - WAL redo function for FILEOPS records + * + * Replay file operations during crash recovery or standby apply. + * Each operation type has its own redo handler added in separate commits. + */ +void +fileops_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + char *data = XLogRecGetData(record); + + switch (info) + { + case XLOG_FILEOPS_CREATE: + { + xl_fileops_create *xlrec = (xl_fileops_create *) data; + const char *path = data + SizeOfFileOpsCreate; + int fd; + + fd = BasicOpenFilePerm(path, + (xlrec->flags & ~PG_O_DIRECT) | O_CREAT, + xlrec->mode); + if (fd < 0) + { + if (errno == ENOENT) + { + char parentpath[MAXPGPATH]; + char *sep; + + strlcpy(parentpath, path, MAXPGPATH); + sep = strrchr(parentpath, '/'); + if (sep != NULL) + { + *sep = '\0'; + if (MakePGDirectory(parentpath) < 0 && + errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\" during WAL replay: %m", + parentpath))); + } + + fd = BasicOpenFilePerm(path, + (xlrec->flags & ~PG_O_DIRECT) | O_CREAT, + xlrec->mode); + } + + if (fd < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create file \"%s\" during WAL replay: %m", + path))); + } + + if (fd >= 0) + { + if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + } + break; + + case XLOG_FILEOPS_WRITE: + { + xl_fileops_write *xlrec = (xl_fileops_write *) data; + const char *path = data + SizeOfFileOpsWrite; + const char *wdata = path + xlrec->path_len; + int fd; + + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for write during WAL replay: %m", + path))); + } + else + { + if (pg_pwrite(fd, wdata, xlrec->len, xlrec->offset) != + (ssize_t) xlrec->len) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\" during WAL replay: %m", + path))); + else if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + } + } + break; + + case XLOG_FILEOPS_RENAME: + + /* + * Redo of FileOpsRename(), logged before the rename was made. + * Converge on the state the primary reached, whatever part of it + * is on disk already: + * + * - oldpath absent: the rename was made already. + * + * - newpath absent: make it. + * + * - both present: the replay of earlier records (the CREATE, + * MKDIR or LINK of oldpath since the checkpoint) has made oldpath + * again, or a later operation has; newpath, which the primary + * never let the rename replace, is the later state. Remove + * oldpath, a directory only if empty, and keep newpath. Renaming + * over newpath would lose data: after a -> b, b -> c and a new b, + * with a older than the checkpoint, redo of b -> c finds the new + * b and would put it in place of c. A non-empty oldpath + * directory is left, with a WARNING. + * + * Redo replays onto the current, possibly later, filesystem state + * and cannot tell an object made again by redo from one made at + * the same path later (see README). + */ + { + xl_fileops_rename *xlrec = (xl_fileops_rename *) data; + const char *oldpath = data + SizeOfFileOpsRename; + const char *newpath = oldpath + xlrec->oldpath_len; + struct stat st; + struct stat nst; + + if (lstat(oldpath, &st) != 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\" during WAL replay: %m", + oldpath))); + } + else if (lstat(newpath, &nst) != 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\" during WAL replay: %m", + newpath))); + else if (rename(oldpath, newpath) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\" during WAL replay: %m", + oldpath, newpath))); + else if (enableFsync) + { + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + fileops_fsync_parent(oldpath, data_sync_elevel(ERROR)); + } + } + else if ((S_ISDIR(st.st_mode) ? rmdir(oldpath) : + unlink(oldpath)) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove \"%s\", renamed to \"%s\", during WAL replay: %m", + oldpath, newpath))); + else if (enableFsync) + fileops_fsync_parent(oldpath, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_DELETE: + + /* + * idempotent redo of a deferred delete executed at commit on the + * primary. An already-absent file (ENOENT) means the delete + * already took effect -- treat as success. Only commit-path + * deletes are ever logged (see FileOpsDoPendingOps), so redo of + * an aborted transaction's delete-on-abort can never occur here. + */ + { + xl_fileops_delete *xlrec pg_attribute_unused() = + (xl_fileops_delete *) data; + const char *path = data + SizeOfFileOpsDelete; + + if (unlink(path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\" during WAL replay: %m", + path))); + } + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_SYMLINK: + + /* + * sec2: intentional no-op during redo. + * + * FileOpsSymlink() has exactly one caller, + * create_tablespace_directories() in tablespace.c, and it is used + * ONLY for tablespace symlinks under $PGDATA/pg_tblspc. Standby + * (and primary crash-recovery) replay of those symlinks is OWNED + * by RM_TBLSPC: tblspc_redo() -> create_tablespace_directories() + * runs with InRecovery set, first removing any stale symlink + * (remove_tablespace_symlink) and then re-creating it with the + * recovery-time layout logic that honors the standby's own + * tablespace layout / allow_in_place_tablespaces / tablespace_map + * remap. + * + * Replaying the FILEOPS record here would be redundant (RM_TBLSPC + * already creates the link) and, worse, actively wrong on a + * remapped standby: it would unlink the correctly-remapped link + * and re-point it at the primary's LITERAL absolute target, which + * may not exist or may belong to a different location on the + * standby. So FILEOPS defers tablespace-symlink replay to + * RM_TBLSPC and does nothing here. + * + * If FileOpsSymlink ever gains a non-tablespace caller, this must + * be revisited to replay those (and only those) symlinks; that + * would require distinguishing the two cases in the record. + */ + break; + + case XLOG_FILEOPS_LINK: + { + xl_fileops_link *xlrec = (xl_fileops_link *) data; + const char *oldpath = data + SizeOfFileOpsLink; + const char *newpath = oldpath + xlrec->oldpath_len; + + /* Remove existing link first for idempotent redo */ + unlink(newpath); + if (link(oldpath, newpath) < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create hard link \"%s\" during WAL replay: %m", + newpath))); + else if (enableFsync) + fileops_fsync_parent(newpath, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_MKDIR: + { + xl_fileops_mkdir *xlrec pg_attribute_unused() = + (xl_fileops_mkdir *) data; + const char *path = data + SizeOfFileOpsMkdir; + + if (MakePGDirectory(path) < 0 && errno != EEXIST) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\" during WAL replay: %m", + path))); + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_RMDIR: + + /* + * idempotent redo of a deferred rmdir executed at commit on the + * primary. An already-absent directory (ENOENT) means the rmdir + * already took effect -- treat as success. + */ + { + xl_fileops_rmdir *xlrec pg_attribute_unused() = + (xl_fileops_rmdir *) data; + const char *path = data + SizeOfFileOpsRmdir; + + if (rmdir(path) < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\" during WAL replay: %m", + path))); + } + else if (enableFsync) + fileops_fsync_parent(path, data_sync_elevel(ERROR)); + } + break; + + case XLOG_FILEOPS_CHMOD: + { + xl_fileops_chmod *xlrec = (xl_fileops_chmod *) data; + const char *path = data + SizeOfFileOpsChmod; + + if (chmod(path, xlrec->mode) < 0 && errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not chmod file \"%s\" during WAL replay: %m", + path))); + } + break; + + case XLOG_FILEOPS_CHOWN: + { +#ifndef WIN32 + xl_fileops_chown *xlrec = (xl_fileops_chown *) data; + const char *path = data + SizeOfFileOpsChown; + + if (chown(path, xlrec->uid, xlrec->gid) < 0 && + errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not chown file \"%s\" during WAL replay: %m", + path))); +#endif + } + break; + + case XLOG_FILEOPS_TRUNCATE: + { + xl_fileops_truncate *xlrec = (xl_fileops_truncate *) data; + const char *path = data + SizeOfFileOpsTruncate; + int fd; + + XLogFlush(record->EndRecPtr); + + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for truncation during WAL replay: %m", + path))); + } + else + { + if (ftruncate(fd, xlrec->length) < 0) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not truncate file \"%s\" to %lld bytes during WAL replay: %m", + path, (long long) xlrec->length))); + else if (enableFsync && pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\" during WAL replay: %m", + path))); + close(fd); + } + } + break; + + case XLOG_FILEOPS_SETXATTR: + { + xl_fileops_setxattr *xlrec = (xl_fileops_setxattr *) data; + const char *path = data + SizeOfFileOpsSetxattr; + const char *name = path + xlrec->path_len; + const void *value = name + xlrec->name_len; + + if (pg_setxattr(path, name, value, xlrec->value_len) < 0 && + errno != ENOTSUP && errno != ENOENT) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not set extended attribute \"%s\" on \"%s\" during WAL replay: %m", + name, path))); + } + break; + + case XLOG_FILEOPS_REMOVEXATTR: + { + xl_fileops_removexattr *xlrec = + (xl_fileops_removexattr *) data; + const char *path = data + SizeOfFileOpsRemovexattr; + const char *name = path + xlrec->path_len; + + if (pg_removexattr(path, name) < 0 && + errno != ENOTSUP && errno != ENOENT && + errno != PG_ENOATTR) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove extended attribute \"%s\" from \"%s\" during WAL replay: %m", + name, path))); + } + break; + + default: + elog(PANIC, "fileops_redo: unknown op code %u", info); + break; + } +} diff --git a/src/backend/storage/file/fileops_undo.c b/src/backend/storage/file/fileops_undo.c new file mode 100644 index 0000000000000..018a6ff52b6d6 --- /dev/null +++ b/src/backend/storage/file/fileops_undo.c @@ -0,0 +1,1065 @@ +/*------------------------------------------------------------------------- + * + * fileops_undo.c + * FILEOPS UNDO resource manager + * + * This module implements the UNDO apply callbacks for transactional file + * operations. The inverse payloads cover structural changes and bounded + * before-images for overwritten or truncated file contents. + * + * On transaction abort, each FILEOPS UNDO record reverses the structural + * change made by the corresponding FileOps* function, if the object still + * shows that change (see fileops_undo_apply): + * - CREATE: unlink the created file + * - WRITE: restore overwritten bytes and original length + * - TRUNCATE: restore removed bytes and original length + * - CHMOD: chmod to original mode + * - CHOWN: chown to original uid/gid + * - MKDIR: rmdir the created directory, or remove its whole tree + * - SYMLINK: unlink the symlink + * - LINK: unlink the hard link + * - RENAME: rename the new name back to the old one + * - SETXATTR: restore/remove the original xattr + * - REMOVEXATTR: restore the original xattr value + * + * DELETE and RMDIR are handled by PendingFileOps (deferred execution) and do + * not need UNDO records since they execute only at commit time. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/file/fileops_undo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include + +#include "access/undo_xlog.h" +#include "access/undormgr.h" +#include "access/xact.h" +#include "access/transam.h" +#include "access/xlog.h" +#include "access/xlogutils.h" +#include "port/pg_xattr.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "storage/md.h" +#include "utils/hsearch.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" + +/* + * UNDO subtype constants and payload structures are defined in fileops.h, + * shared between this file (apply side) and fileops.c (insert side). + */ + +static bool applyingAbort = false; +bool FileOpsSkipUndo = false; + +/* + * Recovery: redo, and the undo phase that follows it. An object an UNDO + * record describes may have been created again by redo, with another inode. + */ +#define FILEOPS_RECOVERING() (InRecovery || UndoRecoveryRollingBack) + +/* Forward declarations */ +static void fileops_note_inverse(uint16 info, const char *data, uint32 len, + int elevel); +static UndoApplyResult fileops_undo_apply(uint8 rmid, uint16 info, + TransactionId xid, Oid reloid, + const char *payload, Size payload_len, + UndoRecPtr urec_ptr); +static void fileops_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len); + +/* + * Report an xattr restore that failed, and return UNDO_APPLY_ERROR so the + * caller counts it as a record that could not be applied. Like the other + * subtypes, a file that no longer exists is not a failure. + */ +static UndoApplyResult +fileops_undo_xattr_failed(const char *op, const char *path, const char *name) +{ + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO %s: could not restore attribute \"%s\" of \"%s\": %m", + op, name, path))); + return UNDO_APPLY_ERROR; +} + +/* The FILEOPS UNDO RM registration entry */ +static const UndoRmgrData fileops_undo_rmgr = { + .rm_name = "fileops", + .rm_undo = fileops_undo_apply, + .rm_desc = fileops_undo_desc, +}; + +/* + * FileopsUndoRmgrInit - Register the FILEOPS UNDO resource manager + */ +void +FileopsUndoRmgrInit(void) +{ + RegisterUndoRmgr(UNDO_RMID_FILEOPS, &fileops_undo_rmgr); + + /* + * Wire FILEOPS's pending-structural-file-operation callbacks into the + * AM-neutral hooks in xact.c (see access/xact.h for why xact.c cannot + * call these by name directly). + */ + PendingPhysOpsDo_hook = FileOpsDoPendingOps; + PendingPhysOpsPostPrepare_hook = PostPrepare_FileOps; + PendingPhysOpsAtSubCommit_hook = AtSubCommit_FileOps; + PendingPhysOpsAtSubAbort_hook = AtSubAbort_FileOps; +} + +/* + * fileops_undo_lstat - lstat() for an UNDO action + * + * Returns 1 if path exists, 0 if it does not (nothing left to undo), and -1 + * after a WARNING for any other failure. + */ +static int +fileops_undo_lstat(const char *op, const char *path, struct stat *st) +{ + if (lstat(path, st) == 0) + return 1; + if (errno == ENOENT) + return 0; + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO %s: could not stat \"%s\": %m", op, path))); + return -1; +} + +/* + * fileops_undo_superseded - Report an object that no longer shows the effect + * being undone: it was already reverted by an earlier walk of this chain, or + * changed since. Either way it is not ours to touch. + */ +static UndoApplyResult +fileops_undo_superseded(const char *op, const char *path) +{ + ereport(DEBUG2, + (errmsg("FILEOPS UNDO %s: \"%s\" was changed since, leaving it", + op, path))); + return UNDO_APPLY_SUCCESS; +} + +/* + * fileops_xattr_is - Does attribute name of path hold exactly value? + * + * Returns 1 if so, 0 if not (or the attribute is absent), and -1 with errno + * set if it cannot be read. + */ +static int +fileops_xattr_is(const char *path, const char *name, const char *value, + uint32 len) +{ + ssize_t cur_len; + char *cur; + int result; + + cur_len = pg_getxattr(path, name, NULL, 0); + if (cur_len < 0) + return (errno == PG_ENOATTR) ? 0 : -1; + if ((size_t) cur_len != len) + return 0; + if (len == 0) + return 1; + + cur = palloc(len); + cur_len = pg_getxattr(path, name, cur, len); + if (cur_len < 0) + result = (errno == PG_ENOATTR || errno == ERANGE) ? 0 : -1; + else + result = ((size_t) cur_len == len && memcmp(cur, value, len) == 0); + pfree(cur); + return result; +} + +/* + * fileops_undo_apply - Apply a single FILEOPS UNDO record + * + * Reverses the structural filesystem operation, but only if the object still + * shows its effect, as recorded in the payload. The same chain can be walked + * again later (see storage/fileops.h), when the object may already have been + * reverted, or changed by someone else; then the record is a no-op. A CREATE + * or MKDIR cannot tell its own file or directory from one created again at + * the same path, since the inode is not known when the UNDO is written, and + * a REMOVEXATTR cannot tell an attribute it removed from one removed again; + * nor can a RENAME tell the object it renamed from one put at the new name + * since. + */ +static UndoApplyResult +fileops_undo_apply(uint8 rmid, uint16 info, TransactionId xid, Oid reloid, + const char *payload, Size payload_len, UndoRecPtr urec_ptr) +{ + UndoApplyResult result = UNDO_APPLY_SUCCESS; + struct stat st; + + Assert(rmid == UNDO_RMID_FILEOPS); + + /* Abort WAL owns completed aborts; generic UNDO is for crash losers. */ + if (!applyingAbort && FileOpsSkipUndo) + return UNDO_APPLY_SUCCESS; + + INJECTION_POINT("fileops-undo-apply-begin", NULL); + + switch (info) + { + case FILEOPS_UNDO_CREATE: + { + FileopsUndoCreate hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoCreate)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoCreate)); + path = payload + sizeof(FileopsUndoCreate); + + switch (fileops_undo_lstat("CREATE", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!S_ISREG(st.st_mode)) + result = fileops_undo_superseded("CREATE", path); + else if (unlink(path) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CREATE: could not unlink \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_WRITE: + { + FileopsUndoWrite hdr; + const char *path; + const char *before; + const char *after; + char *current; + int fd; + int rc; + + if (payload_len < sizeof(hdr)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(hdr)); + if (payload_len != sizeof(hdr) + hdr.path_len + + hdr.before_len + hdr.after_len) + return UNDO_APPLY_ERROR; + path = payload + sizeof(hdr); + before = path + hdr.path_len; + after = before + hdr.before_len; + + /* + * 0 (gone) is nothing to undo, but -1 (a stat error) must + * fail like every other subtype, so a crash loser's + * overwritten bytes are reported as not restored rather than + * silently dropped. + */ + rc = fileops_undo_lstat("WRITE", path, &st); +#ifdef USE_INJECTION_POINTS + /* Lets a test reach the stat-error (-1) arm. */ + if (rc == 1 && + IS_INJECTION_POINT_ATTACHED("fileops-undo-write-lstat-error")) + rc = -1; +#endif + if (rc == -1) + result = UNDO_APPLY_ERROR; + if (rc != 1) + break; + if (!S_ISREG(st.st_mode) || st.st_size != hdr.new_length) + break; + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO WRITE: could not open \"%s\": %m", path))); + return UNDO_APPLY_ERROR; + } + current = palloc(hdr.after_len ? hdr.after_len : 1); + if (pg_pread(fd, current, hdr.after_len, hdr.offset) == hdr.after_len && + memcmp(current, after, hdr.after_len) == 0) + { + if ((hdr.before_len && + pg_pwrite(fd, before, hdr.before_len, hdr.offset) != hdr.before_len) || + ftruncate(fd, hdr.orig_length) != 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO WRITE: could not restore \"%s\": %m", path))); + result = UNDO_APPLY_ERROR; + } + } + pfree(current); + close(fd); + } + break; + + case FILEOPS_UNDO_TRUNCATE: + { + FileopsUndoTruncate hdr; + const char *path; + int fd; + + if (payload_len < sizeof(FileopsUndoTruncate)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoTruncate)); + path = payload + sizeof(FileopsUndoTruncate); + + switch (fileops_undo_lstat("TRUNCATE", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!S_ISREG(st.st_mode) || st.st_size != hdr.new_length) + { + result = fileops_undo_superseded("TRUNCATE", path); + break; + } + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO TRUNCATE: could not open \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + } + else + { + if (hdr.before_len && + pg_pwrite(fd, path + hdr.path_len, hdr.before_len, + hdr.new_length) != hdr.before_len) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not restore bytes of file \"%s\": %m", path))); + result = UNDO_APPLY_ERROR; + } + if (ftruncate(fd, hdr.orig_length) < 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO TRUNCATE: could not restore length of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + close(fd); + } + break; + } + } + break; + + case FILEOPS_UNDO_CHMOD: + { + FileopsUndoChmod hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoChmod)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoChmod)); + path = payload + sizeof(FileopsUndoChmod); + + switch (fileops_undo_lstat("CHMOD", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if ((st.st_mode & 07777) != hdr.new_mode) + result = fileops_undo_superseded("CHMOD", path); + else if (chmod(path, hdr.orig_mode) < 0 && + errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CHMOD: could not restore mode of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_CHOWN: + { +#ifndef WIN32 + FileopsUndoChown hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoChown)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoChown)); + path = payload + sizeof(FileopsUndoChown); + + switch (fileops_undo_lstat("CHOWN", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (st.st_uid != hdr.new_uid || st.st_gid != hdr.new_gid) + result = fileops_undo_superseded("CHOWN", path); + else if (chown(path, hdr.orig_uid, hdr.orig_gid) < 0 && + errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO CHOWN: could not restore ownership of \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } +#endif + } + break; + + case FILEOPS_UNDO_MKDIR: + { + FileopsUndoMkdir hdr; + const char *path; + + if (payload_len < sizeof(FileopsUndoMkdir)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoMkdir)); + path = payload + sizeof(FileopsUndoMkdir); + + switch (fileops_undo_lstat("MKDIR", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!S_ISDIR(st.st_mode)) + result = fileops_undo_superseded("MKDIR", path); + else if ((hdr.flags & FILEOPS_MKDIR_TREE) && + !FILEOPS_RECOVERING() && + (st.st_dev != hdr.dev || st.st_ino != hdr.ino)) + result = fileops_undo_superseded("MKDIR", path); + else if (hdr.flags & FILEOPS_MKDIR_TREE) + { + if (OidIsValid(hdr.dboid)) + { + DropDatabaseBuffers(hdr.dboid); + ForgetDatabaseSyncRequests(hdr.dboid); + } + if (!rmtree(path, true)) + { + ereport(WARNING, + (errmsg("FILEOPS UNDO MKDIR: could not remove directory tree \"%s\"", + path))); + result = UNDO_APPLY_ERROR; + } + } + else if (rmdir(path) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO MKDIR: could not rmdir \"%s\": %m", + path))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_SYMLINK: + { + FileopsUndoSymlink hdr; + const char *linkpath; + const char *target; + char cur[MAXPGPATH]; + ssize_t cur_len; + + if (payload_len < sizeof(FileopsUndoSymlink)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoSymlink)); + if (payload_len < sizeof(FileopsUndoSymlink) + + hdr.linkpath_len + hdr.target_len) + return UNDO_APPLY_ERROR; + linkpath = payload + sizeof(FileopsUndoSymlink); + target = linkpath + hdr.linkpath_len; + + cur_len = readlink(linkpath, cur, sizeof(cur)); + if (cur_len < 0) + { + if (errno == ENOENT) + break; + if (errno == EINVAL) /* not a symlink any more */ + { + result = fileops_undo_superseded("SYMLINK", linkpath); + break; + } + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO SYMLINK: could not read symbolic link \"%s\": %m", + linkpath))); + result = UNDO_APPLY_ERROR; + } + else if ((size_t) cur_len + 1 != hdr.target_len || + memcmp(cur, target, cur_len) != 0) + result = fileops_undo_superseded("SYMLINK", linkpath); + else if (unlink(linkpath) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO SYMLINK: could not unlink \"%s\": %m", + linkpath))); + result = UNDO_APPLY_ERROR; + } + } + break; + + case FILEOPS_UNDO_LINK: + { + FileopsUndoLink hdr; + const char *newpath; + + if (payload_len < sizeof(FileopsUndoLink)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoLink)); + newpath = payload + sizeof(FileopsUndoLink); + + switch (fileops_undo_lstat("LINK", newpath, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + if (!FILEOPS_RECOVERING() && + (st.st_dev != hdr.dev || st.st_ino != hdr.ino)) + result = fileops_undo_superseded("LINK", newpath); + else if (unlink(newpath) < 0 && errno != ENOENT) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO LINK: could not unlink \"%s\": %m", + newpath))); + result = UNDO_APPLY_ERROR; + } + break; + } + } + break; + + case FILEOPS_UNDO_RENAME: + { + FileopsUndoRename hdr; + const char *oldpath; + const char *newpath; + + if (payload_len < sizeof(FileopsUndoRename)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoRename)); + if (payload_len < sizeof(FileopsUndoRename) + + hdr.oldpath_len + hdr.newpath_len) + return UNDO_APPLY_ERROR; + oldpath = payload + sizeof(FileopsUndoRename); + newpath = oldpath + hdr.oldpath_len; + + /* + * Rename back while newpath exists and oldpath is free (see + * FileopsUndoRename). An old name reused since is not ours + * to replace. + */ + switch (fileops_undo_lstat("RENAME", newpath, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + switch (fileops_undo_lstat("RENAME", oldpath, &st)) + { + case 0: + if (rename(newpath, oldpath) < 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("FILEOPS UNDO RENAME: could not rename \"%s\" back to \"%s\": %m", + newpath, oldpath))); + result = UNDO_APPLY_ERROR; + } + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + result = fileops_undo_superseded("RENAME", + oldpath); + break; + } + break; + } + } + break; + + case FILEOPS_UNDO_SETXATTR: + { + FileopsUndoSetxattr hdr; + const char *path; + const char *name; + const char *orig_value; + const char *new_value; + + if (payload_len < sizeof(FileopsUndoSetxattr)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoSetxattr)); + if (payload_len < sizeof(FileopsUndoSetxattr) + hdr.path_len + + hdr.name_len + hdr.orig_value_len + hdr.new_value_len) + return UNDO_APPLY_ERROR; + path = payload + sizeof(FileopsUndoSetxattr); + name = path + hdr.path_len; + orig_value = name + hdr.name_len; + new_value = orig_value + hdr.orig_value_len; + + switch (fileops_undo_lstat("SETXATTR", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + switch (fileops_xattr_is(path, name, new_value, + hdr.new_value_len)) + { + case 0: + result = fileops_undo_superseded("SETXATTR", + path); + break; + case -1: + result = fileops_undo_xattr_failed("SETXATTR", + path, name); + break; + default: + if (hdr.had_value) + { + if (pg_setxattr(path, name, orig_value, + hdr.orig_value_len) < 0 && + errno != ENOENT) + result = fileops_undo_xattr_failed("SETXATTR", + path, name); + } + else if (pg_removexattr(path, name) < 0 && + errno != ENOENT && + errno != PG_ENOATTR) + result = fileops_undo_xattr_failed("SETXATTR", + path, name); + break; + } + break; + } + } + break; + + case FILEOPS_UNDO_REMOVEXATTR: + { + FileopsUndoRemovexattr hdr; + const char *path; + const char *name; + const void *value; + + if (payload_len < sizeof(FileopsUndoRemovexattr)) + return UNDO_APPLY_ERROR; + memcpy(&hdr, payload, sizeof(FileopsUndoRemovexattr)); + path = payload + sizeof(FileopsUndoRemovexattr); + name = path + hdr.path_len; + value = name + hdr.name_len; + + switch (fileops_undo_lstat("REMOVEXATTR", path, &st)) + { + case 0: + break; + case -1: + result = UNDO_APPLY_ERROR; + break; + default: + /* Restore the removed xattr, unless it is back */ + if (pg_getxattr(path, name, NULL, 0) >= 0) + result = fileops_undo_superseded("REMOVEXATTR", + path); + else if (errno != PG_ENOATTR) + result = fileops_undo_xattr_failed("REMOVEXATTR", + path, name); + else if (pg_setxattr(path, name, value, + hdr.value_len) < 0 && + errno != ENOENT) + result = fileops_undo_xattr_failed("REMOVEXATTR", + path, name); + break; + } + } + break; + + default: + ereport(WARNING, + (errmsg("FILEOPS UNDO: unknown subtype %u", info))); + return UNDO_APPLY_ERROR; + } + + INJECTION_POINT("fileops-undo-apply-end", NULL); + + /* + * Recovery logs a crash loser's rollback once it has applied the records + * (XLOG_UNDO_ROLLBACK), so make the inverse durable before returning. + */ + if (!applyingAbort && result == UNDO_APPLY_SUCCESS) + { + fileops_note_inverse(info, payload, payload_len, + data_sync_elevel(ERROR)); + if (FILEOPS_RECOVERING()) + FileOpsSyncInverses(data_sync_elevel(ERROR)); + } + + return result; +} + +/* + * Apply an abort-WAL inverse with the same identity tests as loser recovery. + * Ordinary syscall failures remain warnings. A failed inverse can therefore + * leave an online effect until recovery replays the abort record; callers must + * not infer successful filesystem rollback merely from transaction abort. + * + * The caller makes the inverses durable with FileOpsSyncInverses(PANIC) once + * it has applied them all. Failing to is a PANIC regardless of + * data_sync_retry. Online, we run after the abort record is flushed and the + * transaction is marked aborted in clog, so an ERROR would re-enter + * AbortTransaction and log a second abort record. The caller holds off + * checkpoints until the inverses are durable, so crash recovery replays the + * abort record and applies the inverses again. + */ +void +FileOpsApplyInverse(uint16 info, const char *data, uint32 len) +{ + applyingAbort = true; + PG_TRY(); + { + (void) fileops_undo_apply(UNDO_RMID_FILEOPS, info, InvalidTransactionId, + InvalidOid, data, len, InvalidUndoRecPtr); + fileops_note_inverse(info, data, len, PANIC); + } + PG_FINALLY(); + { + applyingAbort = false; + } + PG_END_TRY(); +} + +/* + * Paths that applied inverses changed and that are not durable yet, each with + * its parent directory. N inverses on one file need one fsync of the file + * and one of its directory, not N of each: every caller makes a whole + * rollback durable before it records the rollback as complete, by calling + * FileOpsSyncInverses() once. + * + * Online, the inverses are applied inside AbortTransaction, where an ERROR + * would re-enter it. So the table is created when the transaction writes its + * first FILEOPS UNDO (FileOpsReserveInverseSyncs), and an entry that cannot be + * allocated there is not an error: that path is synced at once instead. + */ +typedef struct FileOpsSyncEntry +{ + char path[MAXPGPATH]; /* hash key */ + bool target; /* an inverse's own path, not just a parent */ +} FileOpsSyncEntry; + +static HTAB *fileopsPendingSyncs = NULL; + +/* Create the table now, where an out-of-memory ERROR is still harmless. */ +void +FileOpsReserveInverseSyncs(void) +{ + HASHCTL ctl; + + if (fileopsPendingSyncs != NULL) + return; + ctl.keysize = MAXPGPATH; + ctl.entrysize = sizeof(FileOpsSyncEntry); + ctl.hcxt = TopMemoryContext; + fileopsPendingSyncs = hash_create("FILEOPS inverses to sync", 16, &ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); +} + +/* fsync one path an inverse changed, as it is now; see FileOpsSyncInverses. */ +static void +fileops_sync_path(const char *path, bool target, int elevel) +{ + struct stat st; + +#ifdef USE_INJECTION_POINTS + if (target && IS_INJECTION_POINT_ATTACHED("fileops-inverse-fsync-fail")) + { + errno = EIO; + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", path))); + } +#endif + if (lstat(path, &st) == 0 && !S_ISLNK(st.st_mode)) + { + INJECTION_POINT("fileops-inverse-fsync", unconstify(char *, path)); + fsync_fname_ext(path, S_ISDIR(st.st_mode), true, elevel); + } +} + +static void +fileops_note_sync(const char *path, bool target, int elevel) +{ + FileOpsSyncEntry *entry = NULL; + bool found; + + if (!enableFsync) + return; + + /* + * Inside AbortTransaction, an out-of-memory ERROR would re-enter it, so + * the table was created with the transaction's first FILEOPS UNDO. + * Elsewhere (abort redo, the startup undo phase) the ERROR is harmless. + */ + if (fileopsPendingSyncs == NULL && !(applyingAbort && !InRecovery)) + FileOpsReserveInverseSyncs(); + if (fileopsPendingSyncs != NULL) + entry = hash_search(fileopsPendingSyncs, path, HASH_ENTER_NULL, + &found); + if (entry == NULL) + { + /* No memory to defer it: make this path durable now. */ + fileops_sync_path(path, target, elevel); + return; + } + if (!found) + entry->target = false; + entry->target |= target; +} + +/* Remember the directory of a path an inverse changed, for syncing. */ +static void +fileops_note_parent(const char *path, int elevel) +{ + char parent[MAXPGPATH]; + char *sep; + + strlcpy(parent, path, sizeof(parent)); + sep = strrchr(parent, '/'); + if (sep == NULL) + strlcpy(parent, ".", sizeof(parent)); + else if (sep == parent) + parent[1] = '\0'; + else + *sep = '\0'; + + fileops_note_sync(parent, false, elevel); +} + +/* Remember the paths an inverse changed, and their directories. */ +static void +fileops_note_inverse(uint16 info, const char *data, uint32 len, int elevel) +{ + Size hdrlen = 0; + + switch (info) + { + case FILEOPS_UNDO_CREATE: + hdrlen = sizeof(FileopsUndoCreate); + break; + case FILEOPS_UNDO_WRITE: + hdrlen = sizeof(FileopsUndoWrite); + break; + case FILEOPS_UNDO_TRUNCATE: + hdrlen = sizeof(FileopsUndoTruncate); + break; + case FILEOPS_UNDO_CHMOD: + hdrlen = sizeof(FileopsUndoChmod); + break; + case FILEOPS_UNDO_CHOWN: + hdrlen = sizeof(FileopsUndoChown); + break; + case FILEOPS_UNDO_MKDIR: + hdrlen = sizeof(FileopsUndoMkdir); + break; + case FILEOPS_UNDO_SYMLINK: + hdrlen = sizeof(FileopsUndoSymlink); + break; + case FILEOPS_UNDO_LINK: + hdrlen = sizeof(FileopsUndoLink); + break; + case FILEOPS_UNDO_RENAME: + { + FileopsUndoRename hdr; + + /* + * The old name is back, in its directory, and the new name is + * gone from its own. + */ + if (len < sizeof(hdr)) + break; + memcpy(&hdr, data, sizeof(hdr)); + if (hdr.oldpath_len == 0 || hdr.newpath_len == 0 || + len < sizeof(hdr) + hdr.oldpath_len + hdr.newpath_len) + break; + fileops_note_sync(data + sizeof(hdr), true, elevel); + fileops_note_parent(data + sizeof(hdr), elevel); + fileops_note_parent(data + sizeof(hdr) + hdr.oldpath_len, + elevel); + return; + } + case FILEOPS_UNDO_SETXATTR: + hdrlen = sizeof(FileopsUndoSetxattr); + break; + case FILEOPS_UNDO_REMOVEXATTR: + hdrlen = sizeof(FileopsUndoRemovexattr); + break; + } + if (hdrlen == 0 || len <= hdrlen) + elog(PANIC, "invalid FILEOPS inverse payload"); + fileops_note_sync(data + hdrlen, true, elevel); + fileops_note_parent(data + hdrlen, elevel); +} + +/* + * FileOpsSyncInverses - Make the inverses applied so far durable + * + * fsyncs each path noted since the last call once, as it is now: a file or + * directory that still exists (a symlink is covered by its directory), and + * the directory of each. A path a later inverse removed is skipped; its + * removal is in its directory, which an inverse that removed the directory + * itself has noted in turn. Failures are reported at elevel. After an + * ERROR the paths not yet synced stay noted, and the next call syncs them. + */ +void +FileOpsSyncInverses(int elevel) +{ + HASH_SEQ_STATUS seq; + FileOpsSyncEntry *entry; + FileOpsSyncEntry **entries; + long n; + long i = 0; + + if (fileopsPendingSyncs == NULL || + hash_get_num_entries(fileopsPendingSyncs) == 0) + return; + + /* + * Sync from a copy of the entries: an ERROR must not leave a hash scan + * open, and every entry must stay noted until it is synced. Without + * memory for the copy (we may be inside AbortTransaction), take the + * entries one at a time instead, each with a scan of its own. + */ + n = hash_get_num_entries(fileopsPendingSyncs); + entries = palloc_extended(n * sizeof(FileOpsSyncEntry *), + MCXT_ALLOC_NO_OOM); + if (entries != NULL) + { + hash_seq_init(&seq, fileopsPendingSyncs); + while ((entry = hash_seq_search(&seq)) != NULL) + entries[i++] = entry; + for (i = 0; i < n; i++) + { + fileops_sync_path(entries[i]->path, entries[i]->target, elevel); + (void) hash_search(fileopsPendingSyncs, entries[i]->path, + HASH_REMOVE, NULL); + } + pfree(entries); + } + else + { + for (;;) + { + hash_seq_init(&seq, fileopsPendingSyncs); + entry = hash_seq_search(&seq); + if (entry == NULL) + break; + hash_seq_term(&seq); + fileops_sync_path(entry->path, entry->target, elevel); + (void) hash_search(fileopsPendingSyncs, entry->path, HASH_REMOVE, + NULL); + } + } +} + +/* + * fileops_undo_desc - Describe a FILEOPS UNDO record for debugging + */ +static void +fileops_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len) +{ + const char *opname; + + switch (info) + { + case FILEOPS_UNDO_CREATE: + opname = "CREATE"; + break; + case FILEOPS_UNDO_TRUNCATE: + opname = "TRUNCATE"; + break; + case FILEOPS_UNDO_CHMOD: + opname = "CHMOD"; + break; + case FILEOPS_UNDO_CHOWN: + opname = "CHOWN"; + break; + case FILEOPS_UNDO_MKDIR: + opname = "MKDIR"; + break; + case FILEOPS_UNDO_SYMLINK: + opname = "SYMLINK"; + break; + case FILEOPS_UNDO_LINK: + opname = "LINK"; + break; + case FILEOPS_UNDO_RENAME: + opname = "RENAME"; + break; + case FILEOPS_UNDO_SETXATTR: + opname = "SETXATTR"; + break; + case FILEOPS_UNDO_REMOVEXATTR: + opname = "REMOVEXATTR"; + break; + default: + opname = "UNKNOWN"; + break; + } + + appendStringInfo(buf, "fileops %s", opname); +} diff --git a/src/backend/storage/file/meson.build b/src/backend/storage/file/meson.build index 795402589b0b9..7e487324c5f3e 100644 --- a/src/backend/storage/file/meson.build +++ b/src/backend/storage/file/meson.build @@ -4,6 +4,8 @@ backend_sources += files( 'buffile.c', 'copydir.c', 'fd.c', + 'fileops.c', + 'fileops_undo.c', 'fileset.c', 'reinit.c', 'sharedfileset.c', diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c index 8570f17916fc3..b16c93f51a31b 100644 --- a/src/bin/pg_waldump/rmgrdesc.c +++ b/src/bin/pg_waldump/rmgrdesc.c @@ -11,6 +11,7 @@ #include "access/brin_xlog.h" #include "access/clog.h" #include "access/commit_ts.h" +#include "access/fileops_xlog.h" #include "access/generic_xlog.h" #include "access/ginxlog.h" #include "access/gistxlog.h" diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index fbeb19ee4a4e8..ded5e37feda65 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -97,7 +97,8 @@ Generic LogicalMessage XLOG2 -Undo$/, +Undo +FileOps$/, 'rmgr list'); diff --git a/src/include/access/fileops_xlog.h b/src/include/access/fileops_xlog.h new file mode 100644 index 0000000000000..b3ece9a268691 --- /dev/null +++ b/src/include/access/fileops_xlog.h @@ -0,0 +1,40 @@ +/*------------------------------------------------------------------------- + * + * fileops_xlog.h + * Transactional file operations XLOG resource manager definitions + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/include/access/fileops_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILEOPS_XLOG_H +#define FILEOPS_XLOG_H + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* XLOG stuff - all record types defined upfront for WAL compatibility */ +#define XLOG_FILEOPS_CREATE 0x00 +#define XLOG_FILEOPS_DELETE 0x10 +#define XLOG_FILEOPS_RENAME 0x20 +#define XLOG_FILEOPS_WRITE 0x30 +#define XLOG_FILEOPS_TRUNCATE 0x40 +#define XLOG_FILEOPS_CHMOD 0x50 +#define XLOG_FILEOPS_CHOWN 0x60 +#define XLOG_FILEOPS_MKDIR 0x70 +#define XLOG_FILEOPS_RMDIR 0x80 +#define XLOG_FILEOPS_SYMLINK 0x90 +#define XLOG_FILEOPS_LINK 0xA0 +#define XLOG_FILEOPS_SETXATTR 0xB0 +#define XLOG_FILEOPS_REMOVEXATTR 0xC0 + +/* Resource manager functions */ +extern void fileops_redo(XLogReaderState *record); +extern void fileops_desc(StringInfo buf, XLogReaderState *record); +extern const char *fileops_identify(uint8 info); + +#endif /* FILEOPS_XLOG_H */ diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index 046ba45cb2532..6d8f61f724f9d 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -49,3 +49,4 @@ PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL, logicalmsg_decode) PG_RMGR(RM_XLOG2_ID, "XLOG2", xlog2_redo, xlog2_desc, xlog2_identify, NULL, NULL, NULL, xlog2_decode) PG_RMGR(RM_UNDO_ID, "Undo", undo_redo, undo_desc, undo_identify, NULL, NULL, NULL, NULL) +PG_RMGR(RM_FILEOPS_ID, "FileOps", fileops_redo, fileops_desc, fileops_identify, NULL, NULL, NULL, NULL) diff --git a/src/include/access/twophase_rmgr.h b/src/include/access/twophase_rmgr.h index 8927f369c39b5..37be3c7b2900c 100644 --- a/src/include/access/twophase_rmgr.h +++ b/src/include/access/twophase_rmgr.h @@ -28,7 +28,8 @@ typedef uint8 TwoPhaseRmgrId; #define TWOPHASE_RM_PGSTAT_ID 2 #define TWOPHASE_RM_MULTIXACT_ID 3 #define TWOPHASE_RM_PREDICATELOCK_ID 4 -#define TWOPHASE_RM_MAX_ID TWOPHASE_RM_PREDICATELOCK_ID +#define TWOPHASE_RM_FILEOPS_ID 5 +#define TWOPHASE_RM_MAX_ID TWOPHASE_RM_FILEOPS_ID extern PGDLLIMPORT const TwoPhaseCallback twophase_recover_callbacks[]; extern PGDLLIMPORT const TwoPhaseCallback twophase_postcommit_callbacks[]; @@ -39,4 +40,10 @@ extern PGDLLIMPORT const TwoPhaseCallback twophase_standby_recover_callbacks[]; extern void RegisterTwoPhaseRecord(TwoPhaseRmgrId rmid, uint16 info, const void *data, uint32 len); +/* FILEOPS two-phase commit callbacks (defined in storage/file/fileops.c) */ +extern void fileops_twophase_postcommit(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); +extern void fileops_twophase_postabort(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); + #endif /* TWOPHASE_RMGR_H */ diff --git a/src/include/access/undormgrlist.h b/src/include/access/undormgrlist.h index 4ab60e2141881..a9f8ec5afd730 100644 --- a/src/include/access/undormgrlist.h +++ b/src/include/access/undormgrlist.h @@ -31,3 +31,6 @@ */ /* there is deliberately not an #ifndef UNDORMGRLIST_H here */ + +/* FILEOPS: transactional filesystem operations UNDO rmgr (UNDO_RMID_FILEOPS=2) */ +UNDO_RMGR_INIT(FileopsUndoRmgrInit) diff --git a/src/include/access/xact.h b/src/include/access/xact.h index 3abc534fa20a1..799ddcd6896f7 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -213,6 +213,7 @@ typedef struct SavedTransactionCharacteristics #define XACT_XINFO_HAS_AE_LOCKS (1U << 6) #define XACT_XINFO_HAS_GID (1U << 7) #define XACT_XINFO_HAS_DROPPED_STATS (1U << 8) +#define XACT_XINFO_HAS_FILEOPS (1U << 9) /* * Also stored in xinfo, these indicating a variety of additional actions that @@ -464,6 +465,10 @@ typedef struct xl_xact_parsed_abort XLogRecPtr origin_lsn; TimestampTz origin_timestamp; + + /* FILEOPS inverses to apply on redo (XACT_XINFO_HAS_FILEOPS) */ + uint32 fileops_len; + const char *fileops; } xl_xact_parsed_abort; @@ -555,7 +560,8 @@ extern XLogRecPtr XactLogAbortRecord(TimestampTz abort_time, int ndroppedstats, xl_xact_stats_item *droppedstats, int xactflags, TransactionId twophase_xid, - const char *twophase_gid); + const char *twophase_gid, + const char *fileops, uint32 fileops_len); extern void xact_redo(XLogReaderState *record); /* xactdesc.c */ diff --git a/src/include/port/pg_xattr.h b/src/include/port/pg_xattr.h new file mode 100644 index 0000000000000..7159fc27a26a1 --- /dev/null +++ b/src/include/port/pg_xattr.h @@ -0,0 +1,40 @@ +/*------------------------------------------------------------------------- + * + * pg_xattr.h + * Cross-platform extended attribute abstraction + * + * Provides pg_setxattr() and pg_removexattr() that work across: + * - Linux: setxattr/removexattr + * - macOS: setxattr/removexattr (extra options param) + * - FreeBSD: extattr_set_file/extattr_delete_file + * - Windows: NTFS Alternate Data Streams + * - Fallback: returns ENOTSUP with WARNING + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * src/include/port/pg_xattr.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_XATTR_H +#define PG_XATTR_H + +/* + * ENODATA is Linux-specific. FreeBSD/macOS use ENOATTR for "attribute not + * found". Provide a portable PG_ENOATTR so callers don't need #ifdefs. + */ +#if defined(ENOATTR) +#define PG_ENOATTR ENOATTR +#elif defined(ENODATA) +#define PG_ENOATTR ENODATA +#else +#define PG_ENOATTR ENOENT /* last-resort fallback */ +#endif + +extern int pg_setxattr(const char *path, const char *name, + const void *value, size_t size); +extern ssize_t pg_getxattr(const char *path, const char *name, + void *value, size_t size); +extern int pg_removexattr(const char *path, const char *name); + +#endif /* PG_XATTR_H */ diff --git a/src/include/storage/fileops.h b/src/include/storage/fileops.h new file mode 100644 index 0000000000000..523f1240f38c6 --- /dev/null +++ b/src/include/storage/fileops.h @@ -0,0 +1,519 @@ +/*------------------------------------------------------------------------- + * + * fileops.h + * Transactional file operations API + * + * This module provides transactional filesystem operations that are + * WAL-logged and integrated with PostgreSQL's transaction management. + * Constructive operations and renames take effect at once and are reversed + * if the transaction aborts; deletions are deferred to transaction end (see + * src/backend/storage/file/README). + * + * The RM_FILEOPS_ID resource manager handles WAL replay for these + * operations, ensuring correct behavior during crash recovery and + * standby replay. + * + * The operation set follows the Berkeley DB fileops.src model: each + * filesystem operation is a composable unit with its own WAL record + * type, redo handler, and descriptor. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/fileops.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILEOPS_H +#define FILEOPS_H + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* + * WAL record types for FILEOPS operations. + * + * The high 4 bits of the info byte are used for record type, + * leaving the low bits for flags (following PostgreSQL convention). + * + * Following the Berkeley DB fileops.src model, each filesystem + * operation has its own WAL record type for independent redo. + */ +#define XLOG_FILEOPS_CREATE 0x00 +#define XLOG_FILEOPS_DELETE 0x10 +#define XLOG_FILEOPS_RENAME 0x20 +#define XLOG_FILEOPS_WRITE 0x30 +#define XLOG_FILEOPS_TRUNCATE 0x40 +#define XLOG_FILEOPS_CHMOD 0x50 +#define XLOG_FILEOPS_CHOWN 0x60 +#define XLOG_FILEOPS_MKDIR 0x70 +#define XLOG_FILEOPS_RMDIR 0x80 +#define XLOG_FILEOPS_SYMLINK 0x90 +#define XLOG_FILEOPS_LINK 0xA0 +#define XLOG_FILEOPS_SETXATTR 0xB0 +#define XLOG_FILEOPS_REMOVEXATTR 0xC0 + +/* + * PendingFileOp - Deferred file operation entry + * + * File operations are collected in a linked list during a transaction + * and executed at commit or abort time. This follows the same pattern + * used by PendingRelDelete in catalog/storage.c. + */ +typedef enum PendingFileOpType +{ + PENDING_FILEOP_CREATE, + PENDING_FILEOP_DELETE, + PENDING_FILEOP_RENAME, + PENDING_FILEOP_WRITE, + PENDING_FILEOP_TRUNCATE, + PENDING_FILEOP_CHMOD, + PENDING_FILEOP_CHOWN, + PENDING_FILEOP_MKDIR, + PENDING_FILEOP_RMDIR, + PENDING_FILEOP_RMTREE, /* abort: remove a FileOpsMkdirTree() tree */ + PENDING_FILEOP_SYMLINK, + PENDING_FILEOP_LINK, + PENDING_FILEOP_SETXATTR, + PENDING_FILEOP_REMOVEXATTR, + PENDING_FILEOP_INVERSE +} PendingFileOpType; + +typedef struct PendingFileOp +{ + PendingFileOpType type; /* operation type */ + char *path; /* primary file path */ + char *newpath; /* old path to restore (RENAME), or xattr name */ + off_t length; /* truncation length, write offset, or mode */ + void *data; /* generic data (e.g., original xattr value) */ + size_t data_len; /* length of data */ + bool at_commit; /* execute at commit (true) or abort (false) */ + int nestLevel; /* transaction nesting level */ + struct PendingFileOp *next; /* linked list link */ +} PendingFileOp; + +/* + * Public API for transactional file operations + * + * These functions handle platform-specific differences automatically: + * - O_DIRECT: PG_O_DIRECT (Linux/FreeBSD native, macOS F_NOCACHE, + * Windows FILE_FLAG_NO_BUFFERING) + * - fsync: pg_fsync() (Linux fdatasync, macOS F_FULLFSYNC, + * BSD fsync, Windows FlushFileBuffers) + * - Directory sync: fsync_parent_path() (Unix only, no-op on Windows) + * + * Operation-specific API functions are declared below their WAL + * record structures in subsequent sections. + */ + +/* Abort WAL carries a length-prefixed sequence of inverse payloads. */ +#define FILEOPS_MAX_ABORT_BYTES (16 * 1024 * 1024) +#define FILEOPS_MAX_DATA_BYTES (1024 * 1024) +#define FILEOPS_TWOPHASE_INVERSES 1 +extern void FileOpsGetAbortData(char **data, uint32 *len); +extern void FileOpsApplyAbort(const char *data, uint32 len); + +/* Scoped to a generic walk whose FILEOPS inverses belong to abort WAL. */ +extern PGDLLIMPORT bool FileOpsSkipUndo; +extern void FileOpsApplyInverse(uint16 info, const char *data, uint32 len); +extern void FileOpsReserveInverseSyncs(void); +extern void FileOpsSyncInverses(int elevel); +extern void FileOpsBeginAbort(void); +extern void FileOpsEndAbort(void); + +/* Transaction lifecycle hooks */ +extern void FileOpsDoPendingOps(bool isCommit); +extern void AtSubCommit_FileOps(void); +extern void AtSubAbort_FileOps(void); +extern void PostPrepare_FileOps(void); +extern void FileOpsAtPrepare(void); + +/* + * xl_fileops_create - WAL record for file creation + * + * Records that a file was created within a transaction. If the transaction + * aborts, the file will be deleted. The path is stored as variable-length + * data following the fixed header. + */ +typedef struct xl_fileops_create +{ + int flags; /* open flags used for creation */ + mode_t mode; /* file permission mode */ + bool register_delete; /* register for delete-on-abort */ + /* variable-length path follows */ +} xl_fileops_create; + +#define SizeOfFileOpsCreate (offsetof(xl_fileops_create, register_delete) + sizeof(bool)) + +/* File creation API */ +extern int FileOpsCreate(const char *path, int flags, mode_t mode, + bool register_delete); + +/* + * xl_fileops_delete - WAL record for file deletion + * + * Records that a file deletion was requested. The at_commit flag indicates + * whether the deletion should happen at commit (true) or was registered + * as a delete-on-abort from a prior create (false). + */ +typedef struct xl_fileops_delete +{ + bool at_commit; /* true = delete at commit, false = at abort */ + /* variable-length path follows */ +} xl_fileops_delete; + +#define SizeOfFileOpsDelete (offsetof(xl_fileops_delete, at_commit) + sizeof(bool)) + +/* File deletion API */ +extern void FileOpsDelete(const char *path, bool at_commit); + +/* + * xl_fileops_rename - WAL record for file rename + * + * Records that a file or directory is being renamed; it is written before + * the rename is made. Both old and new paths are stored as variable-length + * data: oldpath_len bytes of old path, then the new path follows. + */ +typedef struct xl_fileops_rename +{ + uint16 oldpath_len; /* length of old path (including NUL) */ + /* variable-length old path follows, then new path */ +} xl_fileops_rename; + +#define SizeOfFileOpsRename (offsetof(xl_fileops_rename, oldpath_len) + sizeof(uint16)) + +/* + * File rename API: immediate, refuses an existing newpath, renamed back if + * the transaction aborts. + */ +extern int FileOpsRename(const char *oldpath, const char *newpath); + +/* + * xl_fileops_write - WAL record for file write at offset + * + * Records that data was written to a file at a specific offset. + * The path and data are stored as variable-length data following + * the fixed header. + */ +typedef struct xl_fileops_write +{ + off_t offset; /* write offset in file */ + uint32 len; /* data length */ + uint16 path_len; /* length of path (including NUL) */ + /* variable-length path follows, then data */ +} xl_fileops_write; + +#define SizeOfFileOpsWrite (offsetof(xl_fileops_write, path_len) + sizeof(uint16)) + +/* File write API */ +extern int FileOpsWrite(const char *path, off_t offset, + const void *data, uint32 len); + +/* + * xl_fileops_truncate - WAL record for file truncation + */ +typedef struct xl_fileops_truncate +{ + off_t length; /* new file length */ + /* variable-length path follows */ +} xl_fileops_truncate; + +#define SizeOfFileOpsTruncate (offsetof(xl_fileops_truncate, length) + sizeof(off_t)) + +/* File truncation API */ +extern void FileOpsTruncate(const char *path, off_t length); + +/* + * xl_fileops_chmod - WAL record for file permission change + */ +typedef struct xl_fileops_chmod +{ + mode_t mode; /* new permission mode */ + mode_t orig_mode; /* pre-chmod mode (crash-revert before-state) */ + /* variable-length path follows */ +} xl_fileops_chmod; + +#define SizeOfFileOpsChmod (offsetof(xl_fileops_chmod, orig_mode) + sizeof(mode_t)) + +/* + * xl_fileops_chown - WAL record for file ownership change + */ +typedef struct xl_fileops_chown +{ + uid_t uid; /* new owner user id */ + gid_t gid; /* new owner group id */ + uid_t orig_uid; /* pre-chown uid (crash-revert before-state) */ + gid_t orig_gid; /* pre-chown gid (crash-revert before-state) */ + /* variable-length path follows */ +} xl_fileops_chown; + +#define SizeOfFileOpsChown (offsetof(xl_fileops_chown, orig_gid) + sizeof(gid_t)) + +/* File metadata API */ +extern int FileOpsChmod(const char *path, mode_t mode); +extern int FileOpsChown(const char *path, uid_t uid, gid_t gid); + +/* + * xl_fileops_mkdir - WAL record for directory creation + */ +typedef struct xl_fileops_mkdir +{ + mode_t mode; /* directory permission mode */ + uint16 flags; /* FILEOPS_MKDIR_* */ + /* variable-length path follows */ +} xl_fileops_mkdir; + +#define SizeOfFileOpsMkdir (offsetof(xl_fileops_mkdir, flags) + sizeof(uint16)) + +/* + * The transaction fills the directory itself (with files FILEOPS does not + * track, e.g. through smgr), so undoing the mkdir removes the whole tree. + */ +#define FILEOPS_MKDIR_TREE 0x0001 + +/* + * xl_fileops_rmdir - WAL record for directory removal + */ +typedef struct xl_fileops_rmdir +{ + bool at_commit; /* true = rmdir at commit, false = at abort */ + /* variable-length path follows */ +} xl_fileops_rmdir; + +#define SizeOfFileOpsRmdir (offsetof(xl_fileops_rmdir, at_commit) + sizeof(bool)) + +/* Directory lifecycle API */ +extern int FileOpsMkdir(const char *path, mode_t mode); +extern int FileOpsMkdirTree(const char *path, mode_t mode, Oid dboid); +extern void FileOpsRmdir(const char *path, bool at_commit); + +/* + * xl_fileops_symlink - WAL record for symbolic link creation + */ +typedef struct xl_fileops_symlink +{ + uint16 target_len; /* length of target (including NUL) */ + /* variable-length target follows, then linkpath */ +} xl_fileops_symlink; + +#define SizeOfFileOpsSymlink (offsetof(xl_fileops_symlink, target_len) + sizeof(uint16)) + +/* + * xl_fileops_link - WAL record for hard link creation + */ +typedef struct xl_fileops_link +{ + uint16 oldpath_len; /* length of old path (including NUL) */ + /* variable-length old path follows, then new path */ +} xl_fileops_link; + +#define SizeOfFileOpsLink (offsetof(xl_fileops_link, oldpath_len) + sizeof(uint16)) + +/* Link operations API */ +extern int FileOpsSymlink(const char *target, const char *linkpath); +extern int FileOpsLink(const char *oldpath, const char *newpath); + +/* + * xl_fileops_setxattr - WAL record for setting an extended attribute + */ +typedef struct xl_fileops_setxattr +{ + uint16 name_len; /* attribute name length (including NUL) */ + uint32 value_len; /* attribute value length */ + uint16 path_len; /* file path length (including NUL) */ + uint32 orig_value_len; /* pre-set value length (crash-revert) */ + bool had_value; /* true if xattr existed before this set */ + /* variable-length: path, name, value, orig_value (if had_value) */ +} xl_fileops_setxattr; + +#define SizeOfFileOpsSetxattr (offsetof(xl_fileops_setxattr, had_value) + sizeof(bool)) + +/* + * xl_fileops_removexattr - WAL record for removing an extended attribute + */ +typedef struct xl_fileops_removexattr +{ + uint16 name_len; /* attribute name length (including NUL) */ + uint16 path_len; /* file path length (including NUL) */ + uint32 orig_value_len; /* removed value length (crash-revert); 0 = + * none */ + bool had_value; /* true if xattr existed before this remove */ + /* variable-length: path, name, orig_value (if had_value) */ +} xl_fileops_removexattr; + +#define SizeOfFileOpsRemovexattr (offsetof(xl_fileops_removexattr, had_value) + sizeof(bool)) + +/* Extended attribute API */ +extern int FileOpsSetXattr(const char *path, const char *name, + const void *value, size_t len); +extern int FileOpsRemoveXattr(const char *path, const char *name); + +/* WAL redo and descriptor functions */ +extern void fileops_redo(XLogReaderState *record); +extern void fileops_desc(StringInfo buf, XLogReaderState *record); +extern const char *fileops_identify(uint8 info); + +/* FILEOPS UNDO RM (fileops_undo.c) */ +extern void FileopsUndoRmgrInit(void); + +/* + * UNDO resource-manager ID for FILEOPS (stored in urec_rmid). Defined here, + * with FILEOPS, rather than in the generic access/undormgr.h, so the UNDO + * core names no specific consumer. See access/undormgr.h for the shared, + * WAL-durable ID number space. + */ +#define UNDO_RMID_FILEOPS 2 + +/* FILEOPS UNDO subtypes (stored in urec_info) */ +#define FILEOPS_UNDO_WRITE 0x0002 +#define FILEOPS_UNDO_CREATE 0x0001 +#define FILEOPS_UNDO_TRUNCATE 0x0003 +#define FILEOPS_UNDO_CHMOD 0x0004 +#define FILEOPS_UNDO_CHOWN 0x0005 +#define FILEOPS_UNDO_MKDIR 0x0006 +#define FILEOPS_UNDO_SYMLINK 0x0007 +#define FILEOPS_UNDO_LINK 0x0008 +#define FILEOPS_UNDO_SETXATTR 0x0009 +#define FILEOPS_UNDO_REMOVEXATTR 0x000A +#define FILEOPS_UNDO_RENAME 0x000B + +/* + * FILEOPS UNDO payload structures + * + * These structs define the on-disk layout of UNDO record payloads for each + * FILEOPS operation. Variable-length paths and data follow the fixed header. + * + * An inverse can be applied more than once (online at ROLLBACK, and again by + * redo of the abort record after a crash), and the object may have changed in + * between. + * So each payload also records what the operation left behind (its post-state), + * and fileops_undo_apply() reverts only an object still in that state. + */ + +/* CREATE undo payload: the path to unlink, if it is still a regular file */ +typedef struct FileopsUndoCreate +{ + uint16 path_len; /* including NUL */ + /* followed by path */ +} FileopsUndoCreate; + +/* WRITE undo: path, before-image, then after-image. */ +typedef struct FileopsUndoWrite +{ + off_t orig_length; + off_t new_length; + off_t offset; + uint32 before_len; + uint32 after_len; + uint16 path_len; +} FileopsUndoWrite; + +/* TRUNCATE undo payload: path, followed by the bytes removed by a shrink. */ +typedef struct FileopsUndoTruncate +{ + off_t orig_length; + off_t new_length; + uint32 before_len; + uint16 path_len; + /* followed by path */ +} FileopsUndoTruncate; + +/* CHMOD undo payload: path + original and new mode */ +typedef struct FileopsUndoChmod +{ + mode_t orig_mode; + mode_t new_mode; + uint16 path_len; + /* followed by path */ +} FileopsUndoChmod; + +/* CHOWN undo payload: path + original and new uid/gid */ +typedef struct FileopsUndoChown +{ + uid_t orig_uid; + gid_t orig_gid; + uid_t new_uid; + gid_t new_gid; + uint16 path_len; + /* followed by path */ +} FileopsUndoChown; + +/* + * MKDIR undo payload: the path to rmdir, if it is still a directory. + * + * With FILEOPS_MKDIR_TREE the whole tree is removed instead, after dropping + * the shared buffers and pending fsync requests of database dboid (if valid), + * as a database directory's files are written through shared buffers. Since + * that destroys whatever is in the tree, outside recovery it is done only + * while the path is still the directory created (dev, ino): a later walk of + * the chain must not remove a directory created again at the same path, e.g. + * by a retried ALTER DATABASE SET TABLESPACE. Crash recovery cannot rely on + * the inode (redo on a standby or a restored backup creates the directory + * anew), and needs not: until the transaction's locks are gone no other + * transaction can have made that path. Even so, the inode is evidence that + * the directory is ours, not proof. + */ +typedef struct FileopsUndoMkdir +{ + dev_t dev; /* the directory created */ + ino_t ino; + Oid dboid; /* database filling the tree, or InvalidOid */ + uint16 flags; /* FILEOPS_MKDIR_* */ + uint16 path_len; + /* followed by path */ +} FileopsUndoMkdir; + +/* SYMLINK undo payload: the linkpath to unlink, if it still points to target */ +typedef struct FileopsUndoSymlink +{ + uint16 linkpath_len; + uint16 target_len; + /* followed by linkpath, then target */ +} FileopsUndoSymlink; + +/* LINK undo payload: the newpath to unlink, if it is still the linked inode */ +typedef struct FileopsUndoLink +{ + dev_t dev; /* device and inode of the link source */ + ino_t ino; + uint16 newpath_len; + /* followed by newpath */ +} FileopsUndoLink; + +/* + * RENAME undo payload: rename newpath back to oldpath, while newpath exists + * and oldpath does not. Like CREATE and MKDIR, the test is the state of the + * two paths alone, not the device and inode of the object renamed: those are + * not stable across a restore, a promoted standby or redo that makes the + * object again, so ROLLBACK PREPARED on a promoted standby, for one, would + * take its own copy of the object for a different one and leave it renamed. + * The paths are evidence that the object is the one renamed, not proof. + */ +typedef struct FileopsUndoRename +{ + uint16 oldpath_len; /* including NUL */ + uint16 newpath_len; /* including NUL */ + /* followed by oldpath, then newpath */ +} FileopsUndoRename; + +/* SETXATTR undo payload: path + name + original value (if any) + new value */ +typedef struct FileopsUndoSetxattr +{ + uint16 path_len; + uint16 name_len; + uint32 orig_value_len; /* 0 if xattr didn't exist before */ + uint32 new_value_len; + bool had_value; /* true if xattr existed before setxattr */ + /* followed by path, name, original value (if had_value), new value */ +} FileopsUndoSetxattr; + +/* REMOVEXATTR undo payload: path + name + removed value */ +typedef struct FileopsUndoRemovexattr +{ + uint16 path_len; + uint16 name_len; + uint32 value_len; + /* followed by path, name, value */ +} FileopsUndoRemovexattr; + +#endif /* FILEOPS_H */ diff --git a/src/port/Makefile b/src/port/Makefile index 7e9b58776529a..39e1388227adc 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -51,6 +51,7 @@ OBJS = \ pg_popcount_aarch64.o \ pg_popcount_x86.o \ pg_strong_random.o \ + pg_xattr.o \ pgcheckdir.o \ pgmkdirp.o \ pgsleep.o \ diff --git a/src/port/meson.build b/src/port/meson.build index 922b3f646768d..20e1ca0043110 100644 --- a/src/port/meson.build +++ b/src/port/meson.build @@ -14,6 +14,7 @@ pgport_sources = [ 'pg_popcount_aarch64.c', 'pg_popcount_x86.c', 'pg_strong_random.c', + 'pg_xattr.c', 'pgcheckdir.c', 'pgmkdirp.c', 'pgsleep.c', diff --git a/src/port/pg_xattr.c b/src/port/pg_xattr.c new file mode 100644 index 0000000000000..58a3d422817f7 --- /dev/null +++ b/src/port/pg_xattr.c @@ -0,0 +1,217 @@ +/*------------------------------------------------------------------------- + * + * pg_xattr.c + * Cross-platform extended attribute abstraction + * + * Platform detection uses compiler-defined macros rather than + * configure-time checks, avoiding meson.build/configure.ac changes. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/port/pg_xattr.c + * + *------------------------------------------------------------------------- + */ +#include "c.h" + +#include +#include "port/pg_xattr.h" + +/* + * Platform detection via compiler macros. + * Linux and macOS both provide but with different APIs. + * FreeBSD uses . Windows uses NTFS Alternate Data Streams. + */ +#if defined(__linux__) || defined(__APPLE__) +#include +#define PG_HAVE_XATTR 1 +#elif defined(__FreeBSD__) +#include +#include +#define PG_HAVE_EXTATTR 1 +#elif defined(WIN32) +#define PG_HAVE_ADS 1 +#endif + +#if defined(PG_HAVE_ADS) +/* + * Set errno for a failed operation on the stream "path:name". A missing + * stream fails with ERROR_FILE_NOT_FOUND, like a missing file; report it as + * PG_ENOATTR when the file itself exists, as the other platforms do. + */ +static void +pg_ads_set_errno(const char *path, DWORD err) +{ + _dosmaperr(err); + if (errno == ENOENT && GetFileAttributesA(path) != INVALID_FILE_ATTRIBUTES) + errno = PG_ENOATTR; +} +#endif + +/* + * pg_setxattr - Set an extended attribute on a file + * + * Returns 0 on success, -1 on failure (errno set). + */ +int +pg_setxattr(const char *path, const char *name, + const void *value, size_t size) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return setxattr(path, name, value, size, 0, 0); +#else + return setxattr(path, name, value, size, 0); +#endif + +#elif defined(PG_HAVE_EXTATTR) + ssize_t ret; + + ret = extattr_set_file(path, EXTATTR_NAMESPACE_USER, + name, value, size); + return (ret >= 0) ? 0 : -1; + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + HANDLE hFile; + DWORD written; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + + hFile = CreateFileA(ads_path, GENERIC_WRITE, 0, NULL, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) + { + _dosmaperr(GetLastError()); + return -1; + } + + if (!WriteFile(hFile, value, (DWORD) size, &written, NULL) || + written != (DWORD) size) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + CloseHandle(hFile); + return 0; + +#else + /* Unsupported platform: succeed in WAL but no-op locally */ + (void) path; + (void) name; + (void) value; + (void) size; + errno = ENOTSUP; + return -1; +#endif +} + +/* + * pg_getxattr - Get an extended attribute value from a file + * + * Returns the number of bytes placed in value on success, + * or -1 on failure (errno set). If value is NULL or size is 0, + * returns the size of the attribute value without reading it. + */ +ssize_t +pg_getxattr(const char *path, const char *name, + void *value, size_t size) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return getxattr(path, name, value, size, 0, 0); +#else + return getxattr(path, name, value, size); +#endif + +#elif defined(PG_HAVE_EXTATTR) + return extattr_get_file(path, EXTATTR_NAMESPACE_USER, + name, value, size); + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + HANDLE hFile; + DWORD bytesRead; + LARGE_INTEGER fileSize; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + + hFile = CreateFileA(ads_path, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) + { + pg_ads_set_errno(path, GetLastError()); + return -1; + } + + if (!GetFileSizeEx(hFile, &fileSize)) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + if (value == NULL || size == 0) + { + CloseHandle(hFile); + return (ssize_t) fileSize.QuadPart; + } + + if (!ReadFile(hFile, value, (DWORD) size, &bytesRead, NULL)) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + CloseHandle(hFile); + return (ssize_t) bytesRead; + +#else + (void) path; + (void) name; + (void) value; + (void) size; + errno = ENOTSUP; + return -1; +#endif +} + +/* + * pg_removexattr - Remove an extended attribute from a file + * + * Returns 0 on success, -1 on failure (errno set). + */ +int +pg_removexattr(const char *path, const char *name) +{ +#if defined(PG_HAVE_XATTR) +#if defined(__APPLE__) + return removexattr(path, name, 0); +#else + return removexattr(path, name); +#endif + +#elif defined(PG_HAVE_EXTATTR) + return extattr_delete_file(path, EXTATTR_NAMESPACE_USER, name); + +#elif defined(PG_HAVE_ADS) + char ads_path[MAXPGPATH]; + + snprintf(ads_path, sizeof(ads_path), "%s:%s", path, name); + if (DeleteFileA(ads_path)) + return 0; + + pg_ads_set_errno(path, GetLastError()); + return -1; + +#else + (void) path; + (void) name; + errno = ENOTSUP; + return -1; +#endif +} diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 6c1504ac9323a..f16d803fa2c85 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -32,6 +32,7 @@ SUBDIRS = \ test_escape \ test_extensible \ test_extensions \ + test_fileops \ test_ginpostinglist \ test_int128 \ test_integerset \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index ef3cdb8faf9c9..49cc09d041b6f 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -34,6 +34,7 @@ subdir('test_dsm_registry') subdir('test_escape') subdir('test_extensible') subdir('test_extensions') +subdir('test_fileops') subdir('test_ginpostinglist') subdir('test_int128') subdir('test_integerset') diff --git a/src/test/modules/test_fileops/Makefile b/src/test/modules/test_fileops/Makefile new file mode 100644 index 0000000000000..69100c7527288 --- /dev/null +++ b/src/test/modules/test_fileops/Makefile @@ -0,0 +1,19 @@ +MODULE_big = test_fileops +OBJS = test_fileops.o +EXTENSION = test_fileops +DATA = test_fileops--1.0.sql +PGFILEDESC = "test_fileops - test transactional file operations" +REGRESS = test_fileops +TAP_TESTS = 1 +EXTRA_INSTALL = src/test/modules/injection_points +export enable_injection_points +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_fileops +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_fileops/expected/test_fileops.out b/src/test/modules/test_fileops/expected/test_fileops.out new file mode 100644 index 0000000000000..b529cfe3bb29f --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops.out @@ -0,0 +1,413 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif +CREATE EXTENSION test_fileops; +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; + initial_size +-------------- + 1024 +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + initial_mode +-------------- + 384 +(1 row) + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; + size_during_txn +----------------- + 256 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + size_after_rollback +--------------------- + 1024 +(1 row) + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); + test_fileops_truncate +----------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + size_after_commit +------------------- + 512 +(1 row) + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + mode_baseline +--------------- + 420 +(1 row) + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; + mode_during_txn +----------------- + 384 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + mode_after_rollback +--------------------- + 420 +(1 row) + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal + test_fileops_chmod +-------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; + link_during_txn +----------------- + t +(1 row) + +ROLLBACK; +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + link_after_rollback +--------------------- + f +(1 row) + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + link_after_commit +------------------- + t +(1 row) + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + size_before_test7 +------------------- + 1024 +(1 row) + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + test_fileops_truncate +----------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; + size_in_savepoint +------------------- + 100 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; + size_after_rollback_to +------------------------ + 800 +(1 row) + +COMMIT; +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + size_after_commit +------------------- + 800 +(1 row) + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + test_fileops_chmod +-------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; + mode_in_savepoint +------------------- + 256 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; + mode_after_rollback_to +------------------------ + 448 +(1 row) + +COMMIT; +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; + link2_in_savepoint +-------------------- + t +(1 row) + +ROLLBACK TO sp1; +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; + link2_after_rollback_to +------------------------- + f +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + link2_after_commit +-------------------- + f +(1 row) + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset +\if :xattr_supported +-- Test 10a: SetXattr UNDO - rollback removes newly-set xattr +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.rollback_test', 'will_vanish'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_during_txn; + xattr_during_txn +------------------ + will_vanish +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_after_rollback; + xattr_after_rollback +---------------------- + +(1 row) + +-- Test 10b: SetXattr overwrite UNDO - rollback restores old value +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'overwritten'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_overwritten; + xattr_overwritten +------------------- + overwritten +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_restored; + xattr_restored +---------------- + initial_value +(1 row) + +-- Test 10c: RemoveXattr UNDO - rollback restores removed xattr +BEGIN; +SELECT test_fileops_removexattr(:'filepath_a', 'user.test_key'); + test_fileops_removexattr +-------------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove; + xattr_after_remove +-------------------- + +(1 row) + +ROLLBACK; +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove_rollback; + xattr_after_remove_rollback +----------------------------- + initial_value +(1 row) + +-- Test 10d: Xattr in subtransaction +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_setxattr(:'filepath_a', 'user.sub_key', 'sub_value'); + test_fileops_setxattr +----------------------- + t +(1 row) + +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_in_savepoint; + xattr_in_savepoint +-------------------- + sub_value +(1 row) + +ROLLBACK TO sp1; +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_after_sp_rollback; + xattr_after_sp_rollback +------------------------- + +(1 row) + +COMMIT; +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; +\endif +-- ================================================================ +-- Cleanup +-- ================================================================ +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/expected/test_fileops_1.out b/src/test/modules/test_fileops/expected/test_fileops_1.out new file mode 100644 index 0000000000000..41193304cd627 --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops_1.out @@ -0,0 +1,332 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif +CREATE EXTENSION test_fileops; +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; + initial_size +-------------- + 1024 +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + initial_mode +-------------- + 384 +(1 row) + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; + size_during_txn +----------------- + 256 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + size_after_rollback +--------------------- + 1024 +(1 row) + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); + test_fileops_truncate +----------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + size_after_commit +------------------- + 512 +(1 row) + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + mode_baseline +--------------- + 420 +(1 row) + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; + mode_during_txn +----------------- + 384 +(1 row) + +ROLLBACK; +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + mode_after_rollback +--------------------- + 420 +(1 row) + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal + test_fileops_chmod +-------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; + link_during_txn +----------------- + t +(1 row) + +ROLLBACK; +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + link_after_rollback +--------------------- + f +(1 row) + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); + test_fileops_link +------------------- + +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + link_after_commit +------------------- + t +(1 row) + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + size_before_test7 +------------------- + 1024 +(1 row) + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + test_fileops_truncate +----------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); + test_fileops_truncate +----------------------- + +(1 row) + +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; + size_in_savepoint +------------------- + 100 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; + size_after_rollback_to +------------------------ + 800 +(1 row) + +COMMIT; +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + size_after_commit +------------------- + 800 +(1 row) + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + test_fileops_chmod +-------------------- + +(1 row) + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + test_fileops_chmod +-------------------- + +(1 row) + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 + test_fileops_chmod +-------------------- + +(1 row) + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; + mode_in_savepoint +------------------- + 256 +(1 row) + +ROLLBACK TO sp1; +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; + mode_after_rollback_to +------------------------ + 448 +(1 row) + +COMMIT; +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + mode_after_commit +------------------- + 448 +(1 row) + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); + test_fileops_link +------------------- + +(1 row) + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; + link2_in_savepoint +-------------------- + t +(1 row) + +ROLLBACK TO sp1; +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; + link2_after_rollback_to +------------------------- + f +(1 row) + +COMMIT; +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + link2_after_commit +-------------------- + f +(1 row) + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset +\if :xattr_supported +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; + notice +----------------------------------------------------------------- + xattr tests skipped (not supported on this platform/filesystem) +(1 row) + +\endif +-- ================================================================ +-- Cleanup +-- ================================================================ +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/expected/test_fileops_2.out b/src/test/modules/test_fileops/expected/test_fileops_2.out new file mode 100644 index 0000000000000..d25aa81d54bab --- /dev/null +++ b/src/test/modules/test_fileops/expected/test_fileops_2.out @@ -0,0 +1,18 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit diff --git a/src/test/modules/test_fileops/meson.build b/src/test/modules/test_fileops/meson.build new file mode 100644 index 0000000000000..3f0087fe6e429 --- /dev/null +++ b/src/test/modules/test_fileops/meson.build @@ -0,0 +1,54 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +test_fileops_sources = files( + 'test_fileops.c', +) + +if host_system == 'windows' + test_fileops_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_fileops', + '--FILEDESC', 'test_fileops - regression testing of FILEOPS subsystem',]) +endif + +test_fileops = shared_module('test_fileops', + test_fileops_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_fileops + +test_install_data += files( + 'test_fileops.control', + 'test_fileops--1.0.sql', +) + +tests += { + 'name': 'test_fileops', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_fileops', + ], + }, + 'tap': { + 'env': { + 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', + }, + 'tests': [ + 't/001_fileops_apply_failure.pl', + 't/004_fileops_fs_errors.pl', + 't/005_fileops_checkpoint_loser.pl', + 't/007_fileops_commit_crash.pl', + 't/008_fileops_standby_abort.pl', + 't/018_fileops_rename.pl', + 't/023_fileops_inline_abort_ctx.pl', + 't/024_fileops_recovery_apply_failure.pl', + 't/025_fileops_create_existing.pl', + 't/026_fileops_abort_checkpoint_order.pl', + 't/027_fileops_crash_loser_standby.pl', + 't/028_fileops_insert_holdoff.pl', + 't/029_fileops_crash_loser_recycled_wal.pl', + 't/030_fileops_rollback_record.pl', + ], + }, +} diff --git a/src/test/modules/test_fileops/sql/test_fileops.sql b/src/test/modules/test_fileops/sql/test_fileops.sql new file mode 100644 index 0000000000000..ef1a10523d10f --- /dev/null +++ b/src/test/modules/test_fileops/sql/test_fileops.sql @@ -0,0 +1,229 @@ +-- +-- Tests for FILEOPS UNDO rollback of direct file operations. +-- +-- This test module provides SQL-callable wrappers around FileOps C functions +-- that have no DDL-level callers, enabling direct testing of UNDO rollback +-- for: FileOpsTruncate, FileOpsChmod, FileOpsLink, FileOpsSetXattr, +-- FileOpsRemoveXattr. +-- +-- REQUIRES: UNDO subsystem (always active) +-- +-- These are POSIX file semantics: modes, hard links that do not share the +-- target's attributes, extended attributes. Windows has none of them (chmod +-- only toggles the read-only attribute, which a hard link shares), so the test +-- does not run there; the TAP tests cover the operations FILEOPS supports on +-- Windows. +SELECT version() ~ '(Visual C\+\+|mingw32|msvc|windows)' AS skip_test \gset +\if :skip_test +\quit +\endif + +CREATE EXTENSION test_fileops; + +-- ================================================================ +-- Setup: create test files in the data directory +-- ================================================================ + +SELECT test_fileops_create_tempfile('test_fileops_a.dat') AS filepath_a \gset +SELECT test_fileops_create_tempfile('test_fileops_b.dat') AS filepath_b \gset + +-- Record initial state (umask may affect mode) +SELECT test_fileops_file_size(:'filepath_a') AS initial_size; +SELECT test_fileops_get_mode(:'filepath_a') AS initial_mode; + +-- ================================================================ +-- Test 1: FileOpsTruncate UNDO - rollback restores original size +-- ================================================================ + +BEGIN; +SELECT test_fileops_truncate(:'filepath_a', 256); +SELECT test_fileops_file_size(:'filepath_a') AS size_during_txn; +ROLLBACK; + +-- After rollback, UNDO should restore original size (1024) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback; + +-- ================================================================ +-- Test 2: FileOpsTruncate commit - change persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_truncate(:'filepath_b', 512); +COMMIT; + +SELECT test_fileops_file_size(:'filepath_b') AS size_after_commit; + +-- ================================================================ +-- Test 3: FileOpsChmod UNDO - rollback restores original permissions +-- ================================================================ + +-- Change to a known starting mode first (committed) +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 = 420 decimal +SELECT test_fileops_get_mode(:'filepath_a') AS mode_baseline; + +BEGIN; +-- Change to 0600 = 384 decimal +SELECT test_fileops_chmod(:'filepath_a', 384); +SELECT test_fileops_get_mode(:'filepath_a') AS mode_during_txn; +ROLLBACK; + +-- After rollback, UNDO should restore to 0644 = 420 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback; + +-- ================================================================ +-- Test 4: FileOpsChmod commit - change persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 = 448 decimal +COMMIT; + +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + +-- Restore for subsequent tests +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + +-- ================================================================ +-- Test 5: FileOpsLink UNDO - rollback removes the hard link +-- ================================================================ + +SELECT test_fileops_data_dir() || '/test_fileops_link.dat' AS linkpath \gset + +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); +SELECT test_fileops_file_exists(:'linkpath') AS link_during_txn; +ROLLBACK; + +-- After rollback, UNDO should remove the hard link +SELECT test_fileops_file_exists(:'linkpath') AS link_after_rollback; + +-- ================================================================ +-- Test 6: FileOpsLink commit - link persists +-- ================================================================ + +BEGIN; +SELECT test_fileops_link(:'filepath_a', :'linkpath'); +COMMIT; + +SELECT test_fileops_file_exists(:'linkpath') AS link_after_commit; + +-- ================================================================ +-- Test 7: FileOpsTruncate in subtransaction - ROLLBACK TO undoes it +-- ================================================================ + +-- Ensure file is 1024 bytes +SELECT test_fileops_file_size(:'filepath_a') AS size_before_test7; + +BEGIN; +-- Outer truncate (should persist after commit) +SELECT test_fileops_truncate(:'filepath_a', 800); + +SAVEPOINT sp1; +-- Inner truncate (should be rolled back) +SELECT test_fileops_truncate(:'filepath_a', 100); +SELECT test_fileops_file_size(:'filepath_a') AS size_in_savepoint; +ROLLBACK TO sp1; + +-- After ROLLBACK TO, should be back to 800 (the outer truncate value) +SELECT test_fileops_file_size(:'filepath_a') AS size_after_rollback_to; +COMMIT; + +-- After commit, the outer truncate to 800 should persist +SELECT test_fileops_file_size(:'filepath_a') AS size_after_commit; + +-- ================================================================ +-- Test 8: FileOpsChmod in subtransaction - ROLLBACK TO restores mode +-- ================================================================ + +-- Set a known mode baseline +SELECT test_fileops_chmod(:'filepath_a', 420); -- 0644 + +BEGIN; +-- Outer chmod (persists) +SELECT test_fileops_chmod(:'filepath_a', 448); -- 0700 + +SAVEPOINT sp1; +-- Inner chmod (rolled back) +SELECT test_fileops_chmod(:'filepath_a', 256); -- 0400 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_in_savepoint; +ROLLBACK TO sp1; + +-- After ROLLBACK TO, should be back to 0700 = 448 +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_rollback_to; +COMMIT; + +-- After commit, outer chmod (0700 = 448) persists +SELECT test_fileops_get_mode(:'filepath_a') AS mode_after_commit; + +-- ================================================================ +-- Test 9: FileOpsLink in subtransaction - ROLLBACK TO removes link +-- ================================================================ + +SELECT test_fileops_data_dir() || '/test_fileops_link2.dat' AS linkpath2 \gset + +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_link(:'filepath_a', :'linkpath2'); +SELECT test_fileops_file_exists(:'linkpath2') AS link2_in_savepoint; +ROLLBACK TO sp1; + +-- Link should be removed by UNDO +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_rollback_to; +COMMIT; + +SELECT test_fileops_file_exists(:'linkpath2') AS link2_after_commit; + +-- ================================================================ +-- Test 10: FileOpsSetXattr / FileOpsRemoveXattr UNDO +-- (skipped on platforms without xattr support) +-- ================================================================ + +-- Try to set an xattr; returns false if unsupported (ENOTSUP/EPERM/EACCES) +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'initial_value') + AS xattr_supported \gset + +\if :xattr_supported + +-- Test 10a: SetXattr UNDO - rollback removes newly-set xattr +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.rollback_test', 'will_vanish'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_during_txn; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.rollback_test') AS xattr_after_rollback; + +-- Test 10b: SetXattr overwrite UNDO - rollback restores old value +BEGIN; +SELECT test_fileops_setxattr(:'filepath_a', 'user.test_key', 'overwritten'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_overwritten; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_restored; + +-- Test 10c: RemoveXattr UNDO - rollback restores removed xattr +BEGIN; +SELECT test_fileops_removexattr(:'filepath_a', 'user.test_key'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove; +ROLLBACK; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.test_key') AS xattr_after_remove_rollback; + +-- Test 10d: Xattr in subtransaction +BEGIN; +SAVEPOINT sp1; +SELECT test_fileops_setxattr(:'filepath_a', 'user.sub_key', 'sub_value'); +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_in_savepoint; +ROLLBACK TO sp1; + +SELECT test_fileops_getxattr(:'filepath_a', 'user.sub_key') AS xattr_after_sp_rollback; +COMMIT; + +\else +SELECT 'xattr tests skipped (not supported on this platform/filesystem)' AS notice; +\endif + +-- ================================================================ +-- Cleanup +-- ================================================================ + +DROP EXTENSION test_fileops; diff --git a/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl b/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl new file mode 100644 index 0000000000000..77a131292640a --- /dev/null +++ b/src/test/modules/test_fileops/t/001_fileops_apply_failure.pl @@ -0,0 +1,38 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +# A FILEOPS inverse that cannot be applied at ROLLBACK is reported as a +# failure. The inverse rides in the abort record, so crash replay retries it. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_apply_failure'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf( + 'postgresql.conf', q{ +wal_level = replica +}); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $path = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('fileops_fail.dat')"); + +my $bg = $node->background_psql('postgres'); +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_truncate('$path', 10)"); +chmod(0444, $path) or die "chmod $path: $!"; +$bg->query('ROLLBACK'); +$bg->quit; + +like( + slurp_file($node->logfile), + qr/FILEOPS UNDO TRUNCATE: could not open/, + 'the failure was reported'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/004_fileops_fs_errors.pl b/src/test/modules/test_fileops/t/004_fileops_fs_errors.pl new file mode 100644 index 0000000000000..7d947c62b9fb6 --- /dev/null +++ b/src/test/modules/test_fileops/t/004_fileops_fs_errors.pl @@ -0,0 +1,107 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +# An ordinary filesystem error in an immediate FileOps operation (permission +# denied, no space, ...) must be a plain ERROR, exactly as without FILEOPS: +# the server must not PANIC, and the failed operation must leave nothing in +# the WAL for a standby to replay. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($windows_os) +{ + plan skip_all => 'POSIX permission semantics required'; +} + +my $node = PostgreSQL::Test::Cluster->new('fileops_fs_errors'); +$node->init; +$node->append_conf('postgresql.conf', "wal_level = replica\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; +my $ro = "$datadir/ro_dir"; +mkdir($ro) or die "mkdir $ro: $!"; +my $file = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('fserr.dat')"); +my $rofile = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('fserr_ro.dat')"); +my $has_xattr = $node->safe_psql('postgres', + "SELECT test_fileops_setxattr('$rofile', 'user.pgtest', 'v')"); +chmod(0500, $ro) or die "chmod $ro: $!"; +chmod(0444, $rofile) or die "chmod $rofile: $!"; + +my $start_time = + $node->safe_psql('postgres', 'SELECT pg_postmaster_start_time()'); +my $start_lsn = + $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); + +my %cases = ( + create => "SELECT test_fileops_create('$ro/new.dat', 384)", + write => "SELECT test_fileops_write('/dev/full', 0, 'x')", + truncate => "SELECT test_fileops_truncate('$rofile', 10)", + chmod => "SELECT test_fileops_chmod('/', 448)", + chown => "SELECT test_fileops_chown('$file', 0, 0)", + mkdir => "SELECT test_fileops_mkdir('$ro/sub', 448)", + symlink => "SELECT test_fileops_symlink('$file', '$ro/lnk')", + link => "SELECT test_fileops_link('$file', '$ro/hard')",); + +if ($has_xattr eq 't') +{ + $cases{setxattr} = + "SELECT test_fileops_setxattr('$rofile', 'user.pgtest2', 'v')"; + $cases{removexattr} = + "SELECT test_fileops_removexattr('$rofile', 'user.pgtest')"; +} + +foreach my $op (sort keys %cases) +{ + my ($ret, $stdout, $stderr) = $node->psql('postgres', $cases{$op}); + + # The xattr wrappers report a refused operation as false. + ok( ($stderr =~ /ERROR:/ && $stderr !~ /PANIC|FATAL/) + || ($stdout eq 'f' && $stderr eq ''), + "$op: a filesystem error is an ERROR, not a PANIC" + ) or diag("stdout: $stdout\nstderr: $stderr"); + is($node->safe_psql('postgres', 'SELECT pg_postmaster_start_time()'), + $start_time, "$op: the server did not restart"); +} + +unlike( + slurp_file($node->logfile), + qr/PANIC|terminated by signal/, + 'no PANIC or backend crash in the server log'); + +# CREATE TABLESPACE on a directory we do not own fails as it does without +# FILEOPS. "/" is owned by root, and the server never runs as root. +my ($ret, $stdout, $stderr) = + $node->psql('postgres', "CREATE TABLESPACE t LOCATION '/'"); +like( + $stderr, + qr/ERROR: could not set permissions on directory "\/": (?:Operation not permitted|Read-only file system)/, + 'CREATE TABLESPACE on a root-owned directory gives the usual error'); +is($node->safe_psql('postgres', 'SELECT pg_postmaster_start_time()'), + $start_time, 'the server survived CREATE TABLESPACE'); + +# Nothing the failed operations attempted reached the WAL. +my $end_lsn = + $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +my ($walout, $walerr); +IPC::Run::run [ + 'pg_waldump', + '--path' => "$datadir/pg_wal", + '--start' => $start_lsn, + '--end' => $end_lsn, + '--rmgr' => 'FileOps' + ], + '>' => \$walout, + '2>' => \$walerr; +is($walout, '', 'no FILEOPS WAL record was written for a failed operation'); + +chmod(0700, $ro); +chmod(0644, $rofile); +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/005_fileops_checkpoint_loser.pl b/src/test/modules/test_fileops/t/005_fileops_checkpoint_loser.pl new file mode 100644 index 0000000000000..034ac1ccc2a9f --- /dev/null +++ b/src/test/modules/test_fileops/t/005_fileops_checkpoint_loser.pl @@ -0,0 +1,38 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A transaction that the crash leaves in progress must be rolled back even +# when all of its UNDO precedes the last checkpoint's redo pointer. Redo +# starts there, so it sees the chain only through the checkpoint's +# XLOG_UNDO_INFLIGHT record. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('checkpoint_loser'); +$node->init; +$node->append_conf('postgresql.conf', + "wal_level = replica\nautovacuum = off\ncheckpoint_timeout = '1h'\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $pd = $node->data_dir; +foreach my $ckpt (0, 1) +{ + my $f = "loser_$ckpt.dat"; + my $bg = $node->background_psql('postgres'); + $bg->query_until(qr/ready/, + "BEGIN;\nSELECT test_fileops_create('$f', 384);\n\\echo ready\n"); + $node->safe_psql('postgres', 'CHECKPOINT') if $ckpt; + + # Crash before closing the client, so the backend cannot roll back first. + $node->stop('immediate'); + $bg->{run}->kill_kill; + $node->start; + ok(!-e "$pd/$f", + "checkpoint=$ckpt: the loser's file is removed after a crash"); +} + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl new file mode 100644 index 0000000000000..0ff3e80890f5a --- /dev/null +++ b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl @@ -0,0 +1,77 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +# A deferred commit-time file operation must survive a crash right after +# its transaction's commit record: the commit is durable, so the deletion, +# rename or rmdir it implies must happen too, on restart if not before. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('fileops_commit_crash'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +shared_preload_libraries = 'injection_points' +wal_level = replica +}); +$node->start; +if (!$node->check_extension('injection_points')) +{ + plan skip_all => 'Extension injection_points not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $datadir = $node->data_dir; + +# Commit $sql, crash right after the commit record, restart. +sub commit_and_crash +{ + my ($sql) = @_; + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-commit-before-pending-ops', 'wait')" + ); + my $bg = $node->background_psql('postgres', on_error_stop => 0); + $bg->query_until(qr/start/, "\\echo start\n$sql\n"); + $node->wait_for_event('client backend', + 'fileops-commit-before-pending-ops'); + $node->stop('immediate'); + $bg->{run}->finish; + $node->start; +} + +$node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('del.dat')"); +$node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('ren.dat')"); +$node->safe_psql('postgres', "SELECT test_fileops_mkdir('rmd', 448)"); +commit_and_crash( + q{BEGIN; +SELECT test_fileops_delete('del.dat'); +SELECT test_fileops_rename('ren.dat', 'ren2.dat'); +SELECT test_fileops_rmdir('rmd'); +COMMIT;}); +TODO: +{ + # The deferred ops are WAL-logged only when executed, after the commit + # record, so this crash loses them. Their intent must be durable before + # the commit record and executed by commit redo. + local $TODO = + 'commit-time FILEOPS ops are logged after the commit record'; + ok(!-e "$datadir/del.dat", + 'committed FileOpsDelete happened despite the crash'); + ok( !-e "$datadir/ren.dat" && -e "$datadir/ren2.dat", + 'committed FileOpsRename happened despite the crash'); + ok(!-e "$datadir/rmd", + 'committed FileOpsRmdir happened despite the crash'); +} + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/008_fileops_standby_abort.pl b/src/test/modules/test_fileops/t/008_fileops_standby_abort.pl new file mode 100644 index 0000000000000..aee36a32fc919 --- /dev/null +++ b/src/test/modules/test_fileops/t/008_fileops_standby_abort.pl @@ -0,0 +1,89 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', "max_prepared_transactions = 5\n"); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->safe_psql('postgres', 'CREATE EXTENSION injection_points') + if $ENV{enable_injection_points} eq 'yes'; +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; +$primary->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_create('base.dat', 384); +SELECT test_fileops_write('base.dat', 0, 'committed-base'); +COMMIT; +BEGIN; +SELECT test_fileops_create('abort.dat', 384); +SELECT test_fileops_write('abort.dat', 0, 'aborted'); +ROLLBACK; +BEGIN; SELECT test_fileops_mkdir('abortdir', 448); ROLLBACK; +BEGIN; SELECT test_fileops_chmod('base.dat', 448); ROLLBACK; +BEGIN; SELECT test_fileops_write('base.dat', 0, 'OVERWRITTEN!!!'); ROLLBACK; +BEGIN; SELECT test_fileops_truncate('base.dat', 3); ROLLBACK; +}); +$primary->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_write('base.dat', 0, 'prepared'); +SELECT test_fileops_truncate('base.dat', 2); +PREPARE TRANSACTION 'p'; +ROLLBACK PREPARED 'p'; +BEGIN; +SAVEPOINT a; +SELECT test_fileops_chmod('base.dat', 448); +ROLLBACK TO a; +COMMIT; +}); +$primary->wait_for_catchup($standby); +if ($ENV{enable_injection_points} eq 'yes') +{ + $standby->safe_psql('postgres', + "SELECT injection_points_attach('fileops-abort-after-flush', 'wait')" + ); + $primary->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_create('standby-window.dat', 384); +ROLLBACK; +}); + $standby->wait_for_event('startup', 'fileops-abort-after-flush'); + ok(-e $standby->data_dir . '/standby-window.dat', + 'standby paused before inverse'); + $standby->stop('immediate'); + $standby->start; + $primary->wait_for_catchup($standby); + ok(!-e $standby->data_dir . '/standby-window.dat', + 'standby crash replays inverse'); +} +for my $stage ('streaming', 'promoted') +{ + $standby->promote if $stage eq 'promoted'; + for my $node ($primary, $standby) + { + my $dir = $node->data_dir; + my $name = $node->name; + ok(!-e "$dir/abort.dat", "$stage $name: aborted create absent"); + ok(!-e "$dir/abortdir", "$stage $name: aborted mkdir absent"); + SKIP: + { + skip 'POSIX permission semantics required', 1 if $windows_os; + is((stat("$dir/base.dat"))[2] & 07777, + 0600, "$stage $name: original mode"); + } + is(slurp_file("$dir/base.dat"), + 'committed-base', "$stage $name: original bytes"); + } +} +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/018_fileops_rename.pl b/src/test/modules/test_fileops/t/018_fileops_rename.pl new file mode 100644 index 0000000000000..1967f84427b25 --- /dev/null +++ b/src/test/modules/test_fileops/t/018_fileops_rename.pl @@ -0,0 +1,347 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# FileOpsRename() renames at once, so the new name is usable inside the +# transaction, and the rename is reversed like every other immediate FILEOPS +# operation: on ROLLBACK, ROLLBACK TO SAVEPOINT and ROLLBACK PREPARED, by the +# startup undo phase after a crash, and on a standby that replays the abort. +# At wal_level = minimal an abort-time action renames it back. A rename onto +# an existing name, or of a missing file, is refused and changes nothing. +# Crash redo of committed renames converges on the state before the crash. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $injection = $ENV{enable_injection_points} eq 'yes'; + +my $node = PostgreSQL::Test::Cluster->new('fileops_rename'); +$node->init(allows_streaming => 1); +$node->append_conf( + 'postgresql.conf', q{ +autovacuum = off +max_prepared_transactions = 5 +}); +$node->append_conf('postgresql.conf', + "shared_preload_libraries = 'injection_points'\n") + if $injection; +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $datadir = $node->data_dir; + +# Commit file $name holding $content on $n. FILEOPS paths are relative to +# the data directory, and these are WAL-logged, so a standby gets them. +sub mkfile +{ + my ($n, $name, $content) = @_; + $n->safe_psql( + 'postgres', qq{ +BEGIN; +SELECT test_fileops_create('$name', 384); +SELECT test_fileops_write('$name', 0, '$content'); +COMMIT; +}); +} + +# What file $name in $dir holds, or ''. +sub content +{ + my ($dir, $name) = @_; + return -f "$dir/$name" ? slurp_file("$dir/$name") : ''; +} + +# Rename $from to $to in $bg's open transaction; it must show at once. +sub rename_in +{ + my ($bg, $dir, $from, $to, $tag) = @_; + $bg->query_safe("SELECT test_fileops_rename('$from', '$to')"); + ok( !-e "$dir/$from" && -e "$dir/$to", + "$tag: renamed inside the transaction"); +} + +my $bg = $node->background_psql('postgres', on_error_stop => 0); + +# a. COMMIT keeps the rename +mkfile($node, 'a.dat', 'content-a'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'a.dat', 'a2.dat', 'a'); +is(content($datadir, 'a2.dat'), + 'content-a', 'a: the new name has the content inside the transaction'); +$bg->query_safe('COMMIT'); +is(content($datadir, 'a2.dat'), 'content-a', 'a: COMMIT keeps the new name'); +ok(!-e "$datadir/a.dat", 'a: COMMIT keeps the old name gone'); + +# b. ROLLBACK restores the old name +mkfile($node, 'b.dat', 'content-b'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'b.dat', 'b2.dat', 'b'); +$bg->query_safe('ROLLBACK'); +is(content($datadir, 'b.dat'), + 'content-b', 'b: ROLLBACK restores the old name with its content'); +ok(!-e "$datadir/b2.dat", 'b: ROLLBACK removes the new name'); + +# c. ROLLBACK TO SAVEPOINT restores it, and COMMIT does not redo it +mkfile($node, 'c.dat', 'content-c'); +$bg->query_safe('BEGIN'); +$bg->query_safe('SAVEPOINT s'); +rename_in($bg, $datadir, 'c.dat', 'c2.dat', 'c'); +$bg->query_safe('ROLLBACK TO SAVEPOINT s'); +is(content($datadir, 'c.dat'), + 'content-c', 'c: ROLLBACK TO SAVEPOINT restores the old name'); +ok(!-e "$datadir/c2.dat", 'c: ROLLBACK TO SAVEPOINT removes the new name'); +$bg->query_safe('COMMIT'); +is(content($datadir, 'c.dat'), 'content-c', 'c: still restored after COMMIT'); +ok(!-e "$datadir/c2.dat", 'c: new name still absent after COMMIT'); + +# d. a later operation on the new name is undone first (LIFO) +mkfile($node, 'd.dat', 'content-d'); +my ($ret, $stdout, $stderr) = $node->psql( + 'postgres', q{BEGIN; +SELECT test_fileops_rename('d.dat', 'd2.dat'); +SELECT test_fileops_write('d2.dat', 0, 'CHANGED-AND-LONGER'); +SELECT pg_read_file('d2.dat'); +ROLLBACK;}); +like($stdout, qr/^CHANGED-AND-LONGER$/m, 'd: the write went to the new name'); +is(content($datadir, 'd.dat'), + 'content-d', 'd: ROLLBACK undid the write, then the rename'); +ok(!-e "$datadir/d2.dat", 'd: new name absent after ROLLBACK'); + +# e. refused: an existing destination, a missing source +mkfile($node, 'e.dat', 'content-e'); +mkfile($node, 'e2.dat', 'content-e2'); +($ret, $stdout, $stderr) = + $node->psql('postgres', "SELECT test_fileops_rename('e.dat', 'e2.dat')"); +isnt($ret, 0, 'e: renaming onto an existing name fails'); +like($stderr, qr/already exists/, 'e: ... saying the name exists'); +is(content($datadir, 'e.dat'), 'content-e', 'e: source unchanged'); +is(content($datadir, 'e2.dat'), 'content-e2', 'e: destination unchanged'); +($ret, $stdout, $stderr) = $node->psql('postgres', + "SELECT test_fileops_rename('e-missing.dat', 'e3.dat')"); +isnt($ret, 0, 'e: renaming a missing file fails'); +like($stderr, qr/could not stat file/, 'e: ... saying it is missing'); +ok(!-e "$datadir/e3.dat", 'e: ... and creates nothing'); + +# f. a directory, with a file in it +$node->safe_psql('postgres', "SELECT test_fileops_mkdir('fdir', 448)"); +mkfile($node, 'fdir/inner.dat', 'content-f'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'fdir', 'fdir2', 'f'); +is(content($datadir, 'fdir2/inner.dat'), + 'content-f', 'f: the directory contents moved with it'); +$bg->query_safe('ROLLBACK'); +is(content($datadir, 'fdir/inner.dat'), + 'content-f', 'f: ROLLBACK restores the directory and its contents'); +ok(!-e "$datadir/fdir2", 'f: new directory name absent after ROLLBACK'); + +# l. the rollback leaves an old name that was reused in the meantime alone +mkfile($node, 'l.dat', 'content-l'); +$bg->query_safe('BEGIN'); +rename_in($bg, $datadir, 'l.dat', 'l2.dat', 'l: old name reused'); +append_to_file("$datadir/l.dat", 'reused'); +$bg->query_safe('ROLLBACK'); +is(content($datadir, 'l.dat'), 'reused', 'l: the reused old name is kept'); +is(content($datadir, 'l2.dat'), + 'content-l', 'l: the renamed file stays under the new name'); +$bg->quit; + +# g. crash recovery reverses the rename of an in-flight transaction +mkfile($node, 'g.dat', 'content-g'); +my $bgc = $node->background_psql('postgres', on_error_stop => 0); +$bgc->query_safe('BEGIN'); +rename_in($bgc, $datadir, 'g.dat', 'g2.dat', 'g'); +$node->stop('immediate'); +$bgc->{run}->finish; +$node->start; +is(content($datadir, 'g.dat'), + 'content-g', 'g: crash recovery restores the old name'); +ok(!-e "$datadir/g2.dat", 'g: crash recovery removes the new name'); + +# g. a committed rename of a file created since the last checkpoint: redo +# re-creates the old name from the earlier records, and the rename's redo must +# not leave it behind +mkfile($node, 'g3.dat', 'content-g3'); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('g3.dat', 'g4.dat')"); +$node->stop('immediate'); +$node->start; +is(content($datadir, 'g4.dat'), + 'content-g3', 'g: a committed rename survives a crash'); +ok(!-e "$datadir/g3.dat", 'g: ... and redo leaves no old name behind'); + +# g. committed history a -> b, b -> c, then b made again, where a predates the +# checkpoint. Redo of b -> c finds the later b and c both present; renaming +# over c would replace the renamed file with the later b. +mkfile($node, 'ga.dat', 'content-ga'); +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('ga.dat', 'gb.dat')"); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('gb.dat', 'gc.dat')"); +mkfile($node, 'gb.dat', 'new-gb'); +$node->stop('immediate'); +$node->start; +ok(!-e "$datadir/ga.dat", 'g: a -> b -> c, b again: no a after the crash'); +is(content($datadir, 'gb.dat'), 'new-gb', 'g: ... b is the later file'); +is(content($datadir, 'gc.dat'), + 'content-ga', 'g: ... c is still the renamed file'); + +# g. a renamed directory that redo re-creates and fills under its old name is +# not removed: both names are left, with a WARNING, and nothing is lost +$node->safe_psql('postgres', "SELECT test_fileops_mkdir('gdir', 448)"); +mkfile($node, 'gdir/x.dat', 'content-gx'); +$node->safe_psql('postgres', "SELECT test_fileops_rename('gdir', 'gdir2')"); +my $offset = -s $node->logfile; +$node->stop('immediate'); +$node->start; +is(content($datadir, 'gdir2/x.dat'), + 'content-gx', 'g: a renamed directory keeps its contents after a crash'); +like( + slurp_file($node->logfile, $offset), + qr/WARNING: .*"gdir"/, + 'g: redo warns about the re-created old directory it leaves'); + +# h. two-phase commit +mkfile($node, 'h1.dat', 'content-h1'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('h1.dat', 'h1b.dat'); PREPARE TRANSACTION 'h1';" +); +ok(!-e "$datadir/h1.dat" && -e "$datadir/h1b.dat", + 'h: renamed while prepared'); +$node->safe_psql('postgres', "ROLLBACK PREPARED 'h1'"); +is(content($datadir, 'h1.dat'), + 'content-h1', 'h: ROLLBACK PREPARED restores the old name'); +ok(!-e "$datadir/h1b.dat", 'h: ROLLBACK PREPARED removes the new name'); + +mkfile($node, 'h2.dat', 'content-h2'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('h2.dat', 'h2b.dat'); PREPARE TRANSACTION 'h2';" +); +$node->stop('immediate'); +$node->start; +ok( !-e "$datadir/h2.dat" && -e "$datadir/h2b.dat", + 'h: still renamed while prepared after a crash'); +$node->safe_psql('postgres', "ROLLBACK PREPARED 'h2'"); +is(content($datadir, 'h2.dat'), + 'content-h2', 'h: ROLLBACK PREPARED after a crash restores the old name'); +ok(!-e "$datadir/h2b.dat", + 'h: ROLLBACK PREPARED after a crash removes the new name'); + +mkfile($node, 'h3.dat', 'content-h3'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('h3.dat', 'h3b.dat'); PREPARE TRANSACTION 'h3';" +); +$node->safe_psql('postgres', "COMMIT PREPARED 'h3'"); +is(content($datadir, 'h3b.dat'), + 'content-h3', 'h: COMMIT PREPARED keeps the new name'); +ok(!-e "$datadir/h3.dat", 'h: COMMIT PREPARED keeps the old name gone'); + +# i. a streaming standby converges, for COMMIT and for ROLLBACK +$node->backup('bk'); +my $standby = PostgreSQL::Test::Cluster->new('fileops_rename_standby'); +$standby->init_from_backup($node, 'bk', has_streaming => 1); +$standby->start; +my $sdir = $standby->data_dir; +mkfile($node, 'i1.dat', 'content-i1'); +mkfile($node, 'i2.dat', 'content-i2'); +$node->safe_psql('postgres', + "SELECT test_fileops_rename('i1.dat', 'i1b.dat')"); +my $bgi = $node->background_psql('postgres', on_error_stop => 0); +$bgi->query_safe('BEGIN'); +$bgi->query_safe("SELECT test_fileops_rename('i2.dat', 'i2b.dat')"); +$node->wait_for_catchup($standby); +is(content($sdir, 'i1b.dat'), + 'content-i1', 'i: standby has the committed new name'); +ok(!-e "$sdir/i1.dat", 'i: standby lost the committed old name'); +ok( !-e "$sdir/i2.dat" && -e "$sdir/i2b.dat", + 'i: standby shows a rename in flight'); +$bgi->query_safe('ROLLBACK'); +$bgi->quit; +$node->wait_for_catchup($standby); +is(content($sdir, 'i2.dat'), + 'content-i2', 'i: standby restores the old name of a rolled-back rename'); +ok(!-e "$sdir/i2b.dat", 'i: standby has no new name of a rolled-back rename'); + +# i. ROLLBACK PREPARED on a promoted standby renames back, though the +# standby's file is not the primary's inode +mkfile($node, 'i3.dat', 'content-i3'); +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_rename('i3.dat', 'i3b.dat'); PREPARE TRANSACTION 'i3';" +); +$node->wait_for_catchup($standby); +ok( !-e "$sdir/i3.dat" && -e "$sdir/i3b.dat", + 'i: standby shows the prepared rename'); +$standby->promote; +$standby->safe_psql('postgres', "ROLLBACK PREPARED 'i3'"); +is(content($sdir, 'i3.dat'), + 'content-i3', + 'i: ROLLBACK PREPARED on the promoted standby restores the old name'); +ok(!-e "$sdir/i3b.dat", + 'i: ROLLBACK PREPARED on the promoted standby removes the new name'); +$node->safe_psql('postgres', "ROLLBACK PREPARED 'i3'"); +$standby->stop; + +# j. wal_level = minimal: no WAL, an abort-time action renames back +my $min = PostgreSQL::Test::Cluster->new('fileops_rename_minimal'); +$min->init; +$min->append_conf('postgresql.conf', + "wal_level = minimal\nmax_wal_senders = 0\n"); +$min->start; +$min->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +is($min->safe_psql('postgres', 'SHOW wal_level'), + 'minimal', 'j: wal_level is minimal'); +my $mdir = $min->data_dir; +mkfile($min, 'j.dat', 'content-j'); +my $bgm = $min->background_psql('postgres', on_error_stop => 0); +$bgm->query_safe('BEGIN'); +rename_in($bgm, $mdir, 'j.dat', 'j2.dat', 'j'); +$bgm->query_safe('ROLLBACK'); +is(content($mdir, 'j.dat'), 'content-j', 'j: ROLLBACK restores the old name'); +ok(!-e "$mdir/j2.dat", 'j: ROLLBACK removes the new name'); +$bgm->query_safe('BEGIN'); +$bgm->query_safe('SAVEPOINT s'); +rename_in($bgm, $mdir, 'j.dat', 'j2.dat', 'j: subtransaction'); +$bgm->query_safe('ROLLBACK TO SAVEPOINT s'); +is(content($mdir, 'j.dat'), + 'content-j', 'j: ROLLBACK TO SAVEPOINT restores the old name'); +rename_in($bgm, $mdir, 'j.dat', 'j3.dat', 'j: then'); +$bgm->query_safe('COMMIT'); +is(content($mdir, 'j3.dat'), 'content-j', 'j: COMMIT keeps the new name'); +ok(!-e "$mdir/j.dat" && !-e "$mdir/j2.dat", 'j: and no other name'); +$bgm->quit; +$min->stop; + +# k. a crash after the WAL is flushed but before rename(): redo performs the +# rename, and the loser's UNDO reverses it +SKIP: +{ + skip 'Injection points not supported by this build', 4 unless $injection; + $node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + mkfile($node, 'k.dat', 'content-k'); + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-rename-after-wal-flush', 'wait')" + ); + my $bgk = $node->background_psql('postgres', on_error_stop => 0); + $bgk->query_until( + qr/start/, q(\echo start +BEGIN; +SELECT test_fileops_rename('k.dat', 'k2.dat'); +)); + $node->wait_for_event('client backend', 'fileops-rename-after-wal-flush'); + ok(-e "$datadir/k.dat" && !-e "$datadir/k2.dat", + 'k: logged, not yet renamed'); + $offset = -s $node->logfile; + $node->stop('immediate'); + $bgk->{run}->finish; + $node->start; + like( + slurp_file($node->logfile, $offset), + qr/UNDO recovery: rolling back transaction/, + 'k: the transaction was rolled back as a loser'); + is(content($datadir, 'k.dat'), 'content-k', 'k: the old name is kept'); + ok(!-e "$datadir/k2.dat", 'k: the new name is absent'); +} + +$node->stop; + +done_testing(); diff --git a/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl b/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl new file mode 100644 index 0000000000000..edce86837732d --- /dev/null +++ b/src/test/modules/test_fileops/t/023_fileops_inline_abort_ctx.pl @@ -0,0 +1,60 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# AtAbort_XactUndo()'s inline-abort path must restore the memory context and +# free its working context on every exit, including the early exit taken when +# the batch LSN does not point at a valid UNDO batch. +# +# That exit jumped over the MemoryContextSwitchTo(old_ctx) / +# MemoryContextDelete(undo_ctx) pair, so the "Inline UNDO Apply" context (a +# child of TopMemoryContext) leaked on every such abort, and the rest of the +# abort ran in, and returned into, that context. One leaked context is +# enough to show it: it survives into the next statement of the same +# backend, where pg_backend_memory_contexts lists it. +# +# The invalid-batch exit is forced with the "inline-undo-invalid-batch" +# injection point. A FILEOPS chmod puts one UNDO record in the +# transaction's chain so the inline path runs at all. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('fileops_inline_abort_ctx'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf('postgresql.conf', 'wal_level = replica'); +$node->start; +$node->safe_psql('postgres', + 'CREATE EXTENSION test_fileops; CREATE EXTENSION injection_points;'); + +my $path = $node->safe_psql('postgres', + "SELECT test_fileops_create_tempfile('inline_abort_ctx.dat')"); + +# One backend: the leaked context has to be looked for in the backend that +# aborted, after its transaction has ended. +my $bg = $node->background_psql('postgres'); +$bg->query_safe('SELECT injection_points_set_local()'); +$bg->query_safe( + "SELECT injection_points_attach('inline-undo-invalid-batch', 'error')"); + +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_chmod('$path', 384)"); +$bg->query('ROLLBACK'); + +# The ROLLBACK warns "inline UNDO failed"; query() leaves that on stderr. +my $leaked = $bg->query( + "SELECT count(*) FROM pg_backend_memory_contexts " + . "WHERE name = 'Inline UNDO Apply'"); +chomp $leaked; +is($leaked, '0', + 'the inline-abort working context is freed on the invalid-batch exit'); + +$bg->quit; +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl b/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl new file mode 100644 index 0000000000000..40eb16e6026ff --- /dev/null +++ b/src/test/modules/test_fileops/t/024_fileops_recovery_apply_failure.pl @@ -0,0 +1,64 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# The startup undo phase must not report a crash loser as rolled back when one +# of its UNDO records could not be applied. +# +# A crash loser changed two files' modes through FILEOPS. Before the restart +# the files' directory loses its search permission, so both inverse chmod()s +# fail with a real EACCES. Nothing retries them, so the modes stay changed; +# the summary at the end of the undo phase has to say so, and count both. +# +# Must not run as root (root ignores the directory permission). +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +plan skip_all => 'running as root: EACCES cannot be produced' if $> == 0; +plan skip_all => 'directory search permission is POSIX-only' + if $PostgreSQL::Test::Utils::windows_os; + +my $node = PostgreSQL::Test::Cluster->new('fileops_recovery_apply_failure'); +$node->init; +# FILEOPS writes UNDO only when WAL is needed. +$node->append_conf('postgresql.conf', "wal_level = replica\nautovacuum = off\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $dir = PostgreSQL::Test::Utils::tempdir('eacces'); +my @paths = ("$dir/m.dat", "$dir/n.dat"); +foreach my $path (@paths) +{ + open(my $fh, '>', $path) or die "open $path: $!"; + close($fh); + chmod(0644, $path) or die "chmod $path: $!"; +} + +# Two chmods in flight when the server crashes: a crash loser. +my $bg = $node->background_psql('postgres', on_error_stop => 0); +$bg->query_safe('BEGIN'); +$bg->query_safe("SELECT test_fileops_chmod('$_', 384)") # 0600 + foreach @paths; +$node->safe_psql('postgres', 'SELECT pg_switch_wal()'); +my $offset = -s $node->logfile; +$node->stop('immediate'); +$bg->{run}->finish; + +chmod(0600, $dir) or die "chmod $dir: $!"; +$node->start; +chmod(0700, $dir) or die "chmod $dir: $!"; + +my $log = slurp_file($node->logfile, $offset); +like($log, qr/FILEOPS UNDO CHMOD: could not (?:stat|restore mode of) .*: Permission denied/, + 'the inverse failed with a real EACCES'); +like($log, qr/UNDO recovery complete: 1 transactions processed, 0 records applied, 2 records could not be applied/, + 'the undo phase reports the records it could not apply'); +unlike($log, qr/UNDO recovery complete: \d+ transactions rolled back/, + 'the loser is not reported as rolled back'); +is(sprintf('%04o', (stat($_))[2] & 07777), '0600', + "the mode change of $_ was not rolled back") + foreach @paths; + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/025_fileops_create_existing.pl b/src/test/modules/test_fileops/t/025_fileops_create_existing.pl new file mode 100644 index 0000000000000..a0db2dc29f0e6 --- /dev/null +++ b/src/test/modules/test_fileops/t/025_fileops_create_existing.pl @@ -0,0 +1,56 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# FileOpsCreate() on a path that already exists, then ROLLBACK. The UNDO +# record of a create unlinks the path, so if the call opened an existing file +# a rollback would remove a committed one. The committed file must survive +# on the primary, on a standby replaying the abort record, and on the primary +# after crash recovery replays it. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; + +$primary->safe_psql('postgres', q{ +BEGIN; +SELECT test_fileops_create('existing.dat', 384); +SELECT test_fileops_write('existing.dat', 0, 'committed'); +COMMIT; +}); + +my ($ret, $out, $err) = $primary->psql('postgres', q{ +BEGIN; +SELECT test_fileops_create('existing.dat', 384); +ROLLBACK; +}); +like($err, qr/could not create file "existing.dat": File exists/, + 'creating an existing path is an ERROR'); + +$primary->wait_for_catchup($standby); +for my $stage ('online', 'after crash') +{ + if ($stage eq 'after crash') + { + $primary->stop('immediate'); + $primary->start; + } + my $f = $primary->data_dir . '/existing.dat'; + is(-e $f ? slurp_file($f) : '(missing)', + 'committed', "$stage: primary keeps the committed file"); +} +my $sf = $standby->data_dir . '/existing.dat'; +is(-e $sf ? slurp_file($sf) : '(missing)', + 'committed', 'standby keeps the committed file'); + +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/026_fileops_abort_checkpoint_order.pl b/src/test/modules/test_fileops/t/026_fileops_abort_checkpoint_order.pl new file mode 100644 index 0000000000000..3bb9dfe6eda3d --- /dev/null +++ b/src/test/modules/test_fileops/t/026_fileops_abort_checkpoint_order.pl @@ -0,0 +1,77 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A checkpoint must not move the redo pointer past an abort record whose +# FILEOPS inverses are not yet durable. Pause the aborting backend after the +# abort record is flushed and before it applies the inverses, request a +# CHECKPOINT, then crash. Recovery must replay the abort record and remove +# the file, for ROLLBACK and for ROLLBACK PREPARED. Without the hold-off the +# checkpoint completes during the pause, redo starts after the abort record, +# and the file survives. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +plan skip_all => 'injection points not supported by this build' + unless $ENV{enable_injection_points} eq 'yes'; + +my $node = PostgreSQL::Test::Cluster->new('n'); +$node->init(allows_streaming => 1); +$node->append_conf('postgresql.conf', + "max_prepared_transactions = 5\ncheckpoint_timeout = 1h\n"); +$node->start; +$node->safe_psql('postgres', + 'CREATE EXTENSION test_fileops; CREATE EXTENSION injection_points'); + +for my $case ('rollback', 'rollback prepared') +{ + (my $f = "ck-$case.dat") =~ s/ /-/g; + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-abort-after-flush', 'wait')"); + my $before = $node->safe_psql('postgres', + 'SELECT checkpoint_lsn FROM pg_control_checkpoint()'); + + my $a = $node->background_psql('postgres'); + if ($case eq 'rollback') + { + $a->query_until(qr/go/, qq{\\echo go +BEGIN; SELECT test_fileops_create('$f', 384); ROLLBACK; +}); + } + else + { + $node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_create('$f', 384); PREPARE TRANSACTION 'p';"); + $a->query_until(qr/go/, qq{\\echo go +ROLLBACK PREPARED 'p'; +}); + } + $node->wait_for_event('client backend', 'fileops-abort-after-flush'); + ok(-e $node->data_dir . "/$f", "$case: file still there at the pause"); + + my $c = $node->background_psql('postgres'); + $c->query_until(qr/go/, "\\echo go\nCHECKPOINT;\n"); + + # Give the checkpoint time to finish if nothing holds it back. + my $after = $before; + for (1 .. 50) + { + $after = $node->safe_psql('postgres', + 'SELECT checkpoint_lsn FROM pg_control_checkpoint()'); + last if $after ne $before; + usleep(100_000); + } + note "$case: checkpoint completed during the pause: " + . ($after ne $before ? 'yes' : 'no'); + + $node->stop('immediate'); + eval { $a->quit; }; + eval { $c->quit; }; + $node->start; + ok(!-e $node->data_dir . "/$f", "$case: crash recovery removed the file"); +} + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/027_fileops_crash_loser_standby.pl b/src/test/modules/test_fileops/t/027_fileops_crash_loser_standby.pl new file mode 100644 index 0000000000000..9e36c7b7ee14a --- /dev/null +++ b/src/test/modules/test_fileops/t/027_fileops_crash_loser_standby.pl @@ -0,0 +1,64 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A transaction is in flight with a FILEOPS create and mkdir when the primary +# crashes, so it writes no abort record. The primary's crash recovery rolls +# it back and logs the rollback (XLOG_UNDO_ROLLBACK), and a streaming standby +# must end with the primary's files when it replays that record. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', "restart_after_crash = on\n"); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->backup('b'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'b', has_streaming => 1); +$standby->start; + +for my $how ('immediate stop', 'backend killed') +{ + (my $f = "loser-$how.dat") =~ s/ /-/g; + my $a = $primary->background_psql('postgres'); + $a->query_safe("BEGIN; SELECT test_fileops_create('$f', 384);" + . " SELECT test_fileops_mkdir('$f.d', 448);"); + # Make it durable and replicated. + $primary->safe_psql('postgres', + 'CREATE TABLE IF NOT EXISTS flush_me (a int)'); + $primary->wait_for_catchup($standby); + ok(-e $standby->data_dir . "/$f", "$how: standby has the in-flight file"); + + if ($how eq 'immediate stop') + { + $primary->stop('immediate'); + eval { $a->quit; }; + $primary->start; + } + else + { + my $pid = $a->query_safe('SELECT pg_backend_pid()'); + chomp $pid; + PostgreSQL::Test::Utils::system_or_bail('pg_ctl', 'kill', 'KILL', $pid); + eval { $a->quit; }; + $primary->poll_query_until('postgres', 'SELECT 1', '1'); + } + ok(!-e $primary->data_dir . "/$f", "$how: primary reversed the loser"); + $primary->safe_psql('postgres', 'CHECKPOINT'); + $primary->wait_for_catchup($standby); + ok(!-e $standby->data_dir . "/$f", "$how: standby reversed the loser file"); + ok(!-e $standby->data_dir . "/$f.d", "$how: standby reversed the loser dir"); +} + +$standby->promote; +for my $how ('immediate-stop', 'backend-killed') +{ + ok(!-e $standby->data_dir . "/loser-$how.dat", + "promoted standby: loser-$how.dat absent"); +} +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/028_fileops_insert_holdoff.pl b/src/test/modules/test_fileops/t/028_fileops_insert_holdoff.pl new file mode 100644 index 0000000000000..bfa48073a345d --- /dev/null +++ b/src/test/modules/test_fileops/t/028_fileops_insert_holdoff.pl @@ -0,0 +1,67 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# InsertXactUndoData() holds off a checkpoint's redo-pointer phase between +# inserting an UNDO batch and publishing the chain head that the checkpoint's +# XLOG_UNDO_INFLIGHT record lists. Stop a backend in that interval, request +# a CHECKPOINT, and crash. Without the hold-off the checkpoint completes, its +# redo pointer follows the batch, and its record omits the chain, so nothing +# reverses the create. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +plan skip_all => 'injection points not supported by this build' + unless $ENV{enable_injection_points} eq 'yes'; + +my $node = PostgreSQL::Test::Cluster->new('n'); +$node->init(allows_streaming => 1); +$node->append_conf('postgresql.conf', "checkpoint_timeout = 1h\n"); +$node->start; +$node->safe_psql('postgres', + 'CREATE EXTENSION test_fileops; CREATE EXTENSION injection_points'); + +# The point is in a critical section, so the waiting backend must attach to +# the injection points' shared state beforehand. +my $a = $node->background_psql('postgres'); +$a->query_safe('SELECT injection_points_set_local()'); +$a->query_safe( + "SELECT injection_points_attach('undo-batch-after-wal-insert', 'wait')"); +$a->query_until(qr/go/, + "\\echo go\nBEGIN; SELECT test_fileops_create('holdoff.dat', 384);\n"); +$node->wait_for_event('client backend', 'undo-batch-after-wal-insert'); + +# Make the create and its UNDO batch durable, so that the crash below falls +# outside the documented window between a system call and its WAL record: +# another transaction's commit flushes everything inserted before it. +my $ins = $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +$node->safe_psql('postgres', 'CREATE TABLE flush_me (a int)'); +is( $node->safe_psql( + 'postgres', "SELECT pg_current_wal_flush_lsn() >= '$ins'::pg_lsn"), + 't', + 'the create is durable before the checkpoint'); + +# The checkpoint either waits for the backend or, without the hold-off, +# completes. +my $before = $node->safe_psql('postgres', + 'SELECT checkpoint_lsn FROM pg_control_checkpoint()'); +my $c = $node->background_psql('postgres'); +$c->query_until(qr/go/, "\\echo go\nCHECKPOINT;\n"); +$node->poll_query_until( + 'postgres', qq{ +SELECT EXISTS (SELECT 1 FROM pg_stat_activity + WHERE backend_type = 'checkpointer' + AND wait_event = 'CheckpointDelayStart') + OR (SELECT checkpoint_lsn <> '$before' FROM pg_control_checkpoint())}) + or die 'checkpoint neither waited nor completed'; + +$node->stop('immediate'); +eval { $a->quit; }; +eval { $c->quit; }; +$node->start; +ok(!-e $node->data_dir . '/holdoff.dat', + 'crash recovery removed the loser file'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/029_fileops_crash_loser_recycled_wal.pl b/src/test/modules/test_fileops/t/029_fileops_crash_loser_recycled_wal.pl new file mode 100644 index 0000000000000..54837e160da19 --- /dev/null +++ b/src/test/modules/test_fileops/t/029_fileops_crash_loser_recycled_wal.pl @@ -0,0 +1,53 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A crash loser's UNDO batches can lie in WAL that a standby has already +# removed at a restartpoint by the time it replays the end of the primary's +# crash recovery. The primary logs the rollback with the UNDO records it +# applied (XLOG_UNDO_ROLLBACK), so the standby does not need the batches. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', + "min_wal_size = 32MB\nmax_wal_size = 32MB\ncheckpoint_timeout = 1h\n"); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->backup('b'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'b', has_streaming => 1); +$standby->start; + +my $bg = $primary->background_psql('postgres'); +$bg->query_safe("BEGIN; SELECT test_fileops_create('old-loser.dat', 384);"); +my $seg = $primary->safe_psql('postgres', + 'SELECT pg_walfile_name(pg_current_wal_insert_lsn())'); + +# Move the standby's restartpoints past the loser's batch, so that it can +# remove the segment. The primary keeps it for the open transaction. +$primary->safe_psql('postgres', 'CREATE TABLE t (a int)'); +for (1 .. 6) +{ + $primary->safe_psql('postgres', + 'INSERT INTO t SELECT generate_series(1, 1000);' + . ' SELECT pg_switch_wal(); CHECKPOINT;'); + $primary->wait_for_catchup($standby); + $standby->safe_psql('postgres', 'CHECKPOINT'); +} +my $sdir = $standby->data_dir; +ok(!-e "$sdir/pg_wal/$seg", "standby removed $seg"); +ok(-e $standby->data_dir . '/old-loser.dat', 'standby has the loser file'); + +$primary->stop('immediate'); +eval { $bg->quit; }; +$primary->start; +$primary->wait_for_catchup($standby); +ok(!-e $primary->data_dir . '/old-loser.dat', 'primary rolled back the loser'); +ok(!-e "$sdir/old-loser.dat", 'standby rolled back the loser'); + +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/030_fileops_rollback_record.pl b/src/test/modules/test_fileops/t/030_fileops_rollback_record.pl new file mode 100644 index 0000000000000..c41661fc99ab6 --- /dev/null +++ b/src/test/modules/test_fileops/t/030_fileops_rollback_record.pl @@ -0,0 +1,74 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# The undo phase at the end of crash recovery logs each loser's rollback in +# an XLOG_UNDO_ROLLBACK record, carrying the records that applied. A standby +# applies them when it replays that record. So the standby must keep a +# change the primary could not roll back, and a second crash of the primary +# must not apply the loser's records again over a committed reuse of its +# name. +# +# Must not run as root (root ignores directory permissions). +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +plan skip_all => 'running as root: EACCES cannot be produced' if $> == 0; +plan skip_all => 'directory search permission is POSIX-only' + if $PostgreSQL::Test::Utils::windows_os; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->backup('b'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'b', has_streaming => 1); +$standby->start; +my $pd = $primary->data_dir; +my $sd = $standby->data_dir; + +# A loser creates two files, one inside a directory that loses its search +# permission before the restart, so the primary cannot remove that one. +mkdir("$pd/locked") or die "mkdir: $!"; +mkdir("$sd/locked") or die "mkdir: $!"; +my $a = $primary->background_psql('postgres'); +$a->query_safe("BEGIN; SELECT test_fileops_create('loser.dat', 384);" + . " SELECT test_fileops_create('locked/stuck.dat', 384);"); +$primary->safe_psql('postgres', 'CREATE TABLE flush_me (a int)'); +$primary->wait_for_catchup($standby); +ok(-e "$sd/loser.dat", 'standby has the loser file'); + +$primary->stop('immediate'); +eval { $a->quit; }; +chmod(0600, "$pd/locked") or die "chmod: $!"; +$primary->start; +chmod(0700, "$pd/locked") or die "chmod: $!"; +ok(!-e "$pd/loser.dat", 'primary removed the loser file'); +ok(-e "$pd/locked/stuck.dat", 'primary could not remove the locked file'); + +# The standby replays the rollback record and does what the primary did. +$primary->wait_for_catchup($standby); +ok(!-e "$sd/loser.dat", 'standby removed the loser file'); +ok(-e "$sd/locked/stuck.dat", + 'standby keeps the file the primary could not remove'); + +# The name is reused and committed. A second crash replays the rollback +# record, which must leave the committed file alone. +$primary->safe_psql( + 'postgres', q{ +BEGIN; SELECT test_fileops_create('loser.dat', 384); +SELECT test_fileops_write('loser.dat', 0, 'committed'); COMMIT; +}); +$primary->stop('immediate'); +$primary->start; +is(slurp_file("$pd/loser.dat"), 'committed', + 'a second crash leaves the committed reuse'); +$primary->wait_for_catchup($standby); +is(slurp_file("$sd/loser.dat"), 'committed', + 'the standby has the committed reuse'); + +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/test_fileops--1.0.sql b/src/test/modules/test_fileops/test_fileops--1.0.sql new file mode 100644 index 0000000000000..80545574727a7 --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops--1.0.sql @@ -0,0 +1,99 @@ +/* src/test/modules/test_fileops/test_fileops--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_fileops" to load this file. \quit + +-- Create a temporary file and return its path +CREATE FUNCTION test_fileops_create_tempfile(filename text) + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a file via FileOpsCreate (exercises the WAL-before-syscall path) +CREATE FUNCTION test_fileops_create(filepath text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Truncate a file to the given length (uses FileOpsTruncate) +CREATE FUNCTION test_fileops_truncate(filepath text, length bigint) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get file size +CREATE FUNCTION test_fileops_file_size(filepath text) + RETURNS bigint + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Chmod a file (uses FileOpsChmod) +CREATE FUNCTION test_fileops_chmod(filepath text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get file permissions (mode) +CREATE FUNCTION test_fileops_get_mode(filepath text) + RETURNS int + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a hard link (uses FileOpsLink) +CREATE FUNCTION test_fileops_link(oldpath text, newpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Check if a file exists +CREATE FUNCTION test_fileops_file_exists(filepath text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Set extended attribute (uses FileOpsSetXattr) +CREATE FUNCTION test_fileops_setxattr(filepath text, attrname text, attrvalue text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get extended attribute value +CREATE FUNCTION test_fileops_getxattr(filepath text, attrname text) + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Remove extended attribute (uses FileOpsRemoveXattr) +CREATE FUNCTION test_fileops_removexattr(filepath text, attrname text) + RETURNS boolean + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Get the data directory path (for constructing absolute paths) +CREATE FUNCTION test_fileops_data_dir() + RETURNS text + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred delete at commit (uses FileOpsDelete, at_commit=true) +CREATE FUNCTION test_fileops_delete(filepath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred rename at commit (uses FileOpsRename) +CREATE FUNCTION test_fileops_rename(oldpath text, newpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a directory (uses FileOpsMkdir) +CREATE FUNCTION test_fileops_mkdir(path text, mode int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Deferred rmdir at commit (uses FileOpsRmdir, at_commit=true) +CREATE FUNCTION test_fileops_rmdir(path text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Write a string at an offset (uses FileOpsWrite) +CREATE FUNCTION test_fileops_write(filepath text, "offset" bigint, data text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Change ownership (uses FileOpsChown) +CREATE FUNCTION test_fileops_chown(filepath text, uid int, gid int) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +-- Create a symbolic link (uses FileOpsSymlink) +CREATE FUNCTION test_fileops_symlink(target text, linkpath text) + RETURNS void + AS 'MODULE_PATHNAME' LANGUAGE C STRICT; diff --git a/src/test/modules/test_fileops/test_fileops.c b/src/test/modules/test_fileops/test_fileops.c new file mode 100644 index 0000000000000..c871d974427e9 --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops.c @@ -0,0 +1,471 @@ +/*------------------------------------------------------------------------- + * + * test_fileops.c + * Test module exposing FileOps C API to SQL for regression testing. + * + * This module provides SQL-callable wrappers around the FileOps functions + * that have no DDL-level callers, enabling direct testing of: + * - FileOpsTruncate (with UNDO rollback of file size) + * - FileOpsChmod (with UNDO rollback of permissions) + * - FileOpsLink (with UNDO rollback of hard links) + * - FileOpsSetXattr / FileOpsRemoveXattr (with UNDO rollback) + * + * Copyright (c) 2024-2026, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "port/pg_xattr.h" +#include "access/xlogdefs.h" +#include "storage/fd.h" +#include "storage/fileops.h" +#include "utils/builtins.h" +#include "utils/pg_lsn.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(test_fileops_create_tempfile); +PG_FUNCTION_INFO_V1(test_fileops_create); +PG_FUNCTION_INFO_V1(test_fileops_truncate); +PG_FUNCTION_INFO_V1(test_fileops_file_size); +PG_FUNCTION_INFO_V1(test_fileops_chmod); +PG_FUNCTION_INFO_V1(test_fileops_get_mode); +PG_FUNCTION_INFO_V1(test_fileops_link); +PG_FUNCTION_INFO_V1(test_fileops_file_exists); +PG_FUNCTION_INFO_V1(test_fileops_setxattr); +PG_FUNCTION_INFO_V1(test_fileops_getxattr); +PG_FUNCTION_INFO_V1(test_fileops_removexattr); +PG_FUNCTION_INFO_V1(test_fileops_data_dir); +PG_FUNCTION_INFO_V1(test_fileops_write); +PG_FUNCTION_INFO_V1(test_fileops_chown); +PG_FUNCTION_INFO_V1(test_fileops_symlink); + +PG_FUNCTION_INFO_V1(test_fileops_delete); +PG_FUNCTION_INFO_V1(test_fileops_rename); +PG_FUNCTION_INFO_V1(test_fileops_mkdir); +PG_FUNCTION_INFO_V1(test_fileops_rmdir); + +/* + * test_fileops_create_tempfile - Create a test file in data directory. + * + * Creates a file with some content using FileOpsCreate + write, returning + * the absolute path. The file is created with 1024 bytes of content. + */ +Datum +test_fileops_create_tempfile(PG_FUNCTION_ARGS) +{ + text *filename = PG_GETARG_TEXT_PP(0); + char *fname = text_to_cstring(filename); + char filepath[MAXPGPATH]; + int fd; + char buf[1024]; + + snprintf(filepath, MAXPGPATH, "%s/%s", DataDir, fname); + + /* Use raw create + write so we have a file to manipulate */ + fd = open(filepath, O_CREAT | O_WRONLY | O_TRUNC | PG_BINARY, 0644); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", filepath))); + + /* Fill with 1024 bytes */ + memset(buf, 'X', sizeof(buf)); + if (write(fd, buf, sizeof(buf)) != sizeof(buf)) + { + int save_errno = errno; + + close(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", filepath))); + } + close(fd); + + PG_RETURN_TEXT_P(cstring_to_text(filepath)); +} + +/* + * test_fileops_create - Create a file using FileOpsCreate. + * + * Unlike test_fileops_create_tempfile (which uses raw open()/write() to set + * up fixture files outside any transactional path), this calls FileOpsCreate + * directly so tests can exercise its WAL-before-syscall crash-recovery + * behavior. + */ +Datum +test_fileops_create(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int mode = PG_GETARG_INT32(1); + char *filepath = text_to_cstring(filepath_text); + int fd; + + fd = FileOpsCreate(filepath, O_WRONLY | PG_BINARY, (mode_t) mode, true); + if (fd >= 0) + CloseTransientFile(fd); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_truncate - Truncate a file using FileOpsTruncate. + * + * This exercises the UNDO path: on rollback, the original file size + * should be restored. + */ +Datum +test_fileops_truncate(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int64 length = PG_GETARG_INT64(1); + char *filepath = text_to_cstring(filepath_text); + + FileOpsTruncate(filepath, (off_t) length); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_file_size - Get the current size of a file. + */ +Datum +test_fileops_file_size(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + + if (stat(filepath, &st) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_INT64((int64) st.st_size); +} + +/* + * test_fileops_chmod - Change file permissions using FileOpsChmod. + * + * This exercises the UNDO path: on rollback, the original permissions + * should be restored. + */ +Datum +test_fileops_chmod(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + int mode = PG_GETARG_INT32(1); + char *filepath = text_to_cstring(filepath_text); + + if (FileOpsChmod(filepath, (mode_t) mode) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not chmod file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_get_mode - Get file permission bits. + */ +Datum +test_fileops_get_mode(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + + if (stat(filepath, &st) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", filepath))); + + pfree(filepath); + PG_RETURN_INT32((int32) (st.st_mode & 0777)); +} + +/* + * test_fileops_link - Create a hard link using FileOpsLink. + * + * This exercises the UNDO path: on rollback, the link should be removed. + */ +Datum +test_fileops_link(PG_FUNCTION_ARGS) +{ + text *oldpath_text = PG_GETARG_TEXT_PP(0); + text *newpath_text = PG_GETARG_TEXT_PP(1); + char *oldpath = text_to_cstring(oldpath_text); + char *newpath = text_to_cstring(newpath_text); + + if (FileOpsLink(oldpath, newpath) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not link \"%s\" to \"%s\": %m", + oldpath, newpath))); + + pfree(oldpath); + pfree(newpath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_file_exists - Check whether a file exists. + */ +Datum +test_fileops_file_exists(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + char *filepath = text_to_cstring(filepath_text); + struct stat st; + bool exists; + + exists = (stat(filepath, &st) == 0); + + pfree(filepath); + PG_RETURN_BOOL(exists); +} + +/* + * test_fileops_setxattr - Set an extended attribute using FileOpsSetXattr. + * + * Returns true if the platform supports xattrs, false if ENOTSUP. + * This exercises the UNDO path: on rollback, the original xattr value + * (or absence) should be restored. + */ +Datum +test_fileops_setxattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + text *value_text = PG_GETARG_TEXT_PP(2); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + char *value = text_to_cstring(value_text); + int ret; + + ret = FileOpsSetXattr(filepath, name, value, strlen(value)); + if (ret != 0) + { + if (errno == ENOTSUP || errno == EOPNOTSUPP || + errno == EPERM || errno == EACCES) + { + pfree(filepath); + pfree(name); + pfree(value); + PG_RETURN_BOOL(false); + } + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set xattr \"%s\" on \"%s\": %m", + name, filepath))); + } + + pfree(filepath); + pfree(name); + pfree(value); + PG_RETURN_BOOL(true); +} + +/* + * test_fileops_getxattr - Get an extended attribute value. + * + * Returns NULL if the attribute doesn't exist or xattrs are unsupported. + */ +Datum +test_fileops_getxattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + char buf[1024]; + ssize_t len; + + len = pg_getxattr(filepath, name, buf, sizeof(buf) - 1); + if (len < 0) + { + pfree(filepath); + pfree(name); + PG_RETURN_NULL(); + } + + buf[len] = '\0'; + + pfree(filepath); + pfree(name); + PG_RETURN_TEXT_P(cstring_to_text(buf)); +} + +/* + * test_fileops_removexattr - Remove an extended attribute using + * FileOpsRemoveXattr. + * + * Returns true if successful, false if xattrs are unsupported. + * This exercises the UNDO path: on rollback, the removed xattr should + * be restored. + */ +Datum +test_fileops_removexattr(PG_FUNCTION_ARGS) +{ + text *filepath_text = PG_GETARG_TEXT_PP(0); + text *name_text = PG_GETARG_TEXT_PP(1); + char *filepath = text_to_cstring(filepath_text); + char *name = text_to_cstring(name_text); + int ret; + + ret = FileOpsRemoveXattr(filepath, name); + if (ret != 0) + { + if (errno == ENOTSUP || errno == EOPNOTSUPP) + { + pfree(filepath); + pfree(name); + PG_RETURN_BOOL(false); + } + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not remove xattr \"%s\" from \"%s\": %m", + name, filepath))); + } + + pfree(filepath); + pfree(name); + PG_RETURN_BOOL(true); +} + +/* + * test_fileops_data_dir - Return the data directory path. + */ +Datum +test_fileops_data_dir(PG_FUNCTION_ARGS) +{ + PG_RETURN_TEXT_P(cstring_to_text(DataDir)); +} + +/* + * test_fileops_delete - Deferred delete via FileOpsDelete(at_commit=true). + * + * Exercises the log-at-commit-execution path: the delete happens (and is + * WAL-logged) in FileOpsDoPendingOps() at commit, so a physical standby + * reproduces it by replay. + */ +Datum +test_fileops_delete(PG_FUNCTION_ARGS) +{ + char *filepath = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + FileOpsDelete(filepath, true); + pfree(filepath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_rename - Deferred rename via FileOpsRename (always at_commit). + */ +Datum +test_fileops_rename(PG_FUNCTION_ARGS) +{ + char *oldpath = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *newpath = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + if (FileOpsRename(oldpath, newpath) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not rename \"%s\" to \"%s\": %m", + oldpath, newpath))); + pfree(oldpath); + pfree(newpath); + PG_RETURN_VOID(); +} + +/* + * test_fileops_mkdir - Create a directory via FileOpsMkdir. + * + * FileOpsMkdir creates the directory immediately and registers an + * rmdir-on-abort; on commit the directory persists. Used to set up a + * directory the test can then remove with test_fileops_rmdir. + */ +Datum +test_fileops_mkdir(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + int mode = PG_GETARG_INT32(1); + + if (FileOpsMkdir(path, (mode_t) mode) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", path))); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_rmdir - Deferred rmdir via FileOpsRmdir(at_commit=true). + */ +Datum +test_fileops_rmdir(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + FileOpsRmdir(path, true); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_write - Write a string at an offset via FileOpsWrite. + */ +Datum +test_fileops_write(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + int64 offset = PG_GETARG_INT64(1); + char *data = text_to_cstring(PG_GETARG_TEXT_PP(2)); + + (void) FileOpsWrite(path, (off_t) offset, data, (uint32) strlen(data)); + pfree(path); + pfree(data); + PG_RETURN_VOID(); +} + +/* + * test_fileops_chown - Change ownership via FileOpsChown. + */ +Datum +test_fileops_chown(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + (void) FileOpsChown(path, (uid_t) PG_GETARG_INT32(1), + (gid_t) PG_GETARG_INT32(2)); + pfree(path); + PG_RETURN_VOID(); +} + +/* + * test_fileops_symlink - Create a symbolic link via FileOpsSymlink. + */ +Datum +test_fileops_symlink(PG_FUNCTION_ARGS) +{ + char *target = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *linkpath = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + (void) FileOpsSymlink(target, linkpath); + pfree(target); + pfree(linkpath); + PG_RETURN_VOID(); +} diff --git a/src/test/modules/test_fileops/test_fileops.control b/src/test/modules/test_fileops/test_fileops.control new file mode 100644 index 0000000000000..1d39ab2a9166b --- /dev/null +++ b/src/test/modules/test_fileops/test_fileops.control @@ -0,0 +1,4 @@ +comment = 'Test module for transactional file operations (FILEOPS)' +default_version = '1.0' +module_pathname = '$libdir/test_fileops' +relocatable = true diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile index e90e0b1ad4c61..30181a5fd9a48 100644 --- a/src/test/recovery/Makefile +++ b/src/test/recovery/Makefile @@ -14,6 +14,7 @@ EXTRA_INSTALL=contrib/pg_prewarm \ contrib/pg_visibility \ contrib/test_decoding \ src/test/modules/injection_points \ + src/test/modules/test_fileops \ src/test/modules/test_wait_lsn subdir = src/test/recovery diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index cb3df6ed89502..c8aef86b59682 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -70,6 +70,11 @@ tests += { 't/059_remote_apply_status_interval.pl', 't/060_multixact_crashed_updater.pl', 't/061_recovery_target_prefetch.pl', + 't/062_fileops_2pc_deferred.pl', + 't/063_fileops_recovery.pl', + 't/064_fileops_2pc.pl', + 't/065_fileops_standby_divergence.pl', + 't/066_fileops_crash_undo.pl', ], }, } diff --git a/src/test/recovery/t/062_fileops_2pc_deferred.pl b/src/test/recovery/t/062_fileops_2pc_deferred.pl new file mode 100644 index 0000000000000..fb6728cab79f7 --- /dev/null +++ b/src/test/recovery/t/062_fileops_2pc_deferred.pl @@ -0,0 +1,172 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test that DEFERRED commit-time FILEOPS (delete) and abort-time filesystem +# undo (of a create and a rename) survive PREPARE TRANSACTION across a crash. +# +# The bug: PostPrepare_FileOps() used to free the pending-op list without +# recording it anywhere, so a transaction PREPARE'd with a deferred delete +# (at_commit=true) lost that op entirely -- COMMIT PREPARED performed +# nothing -- and an immediate create's abort-time undo was lost for ROLLBACK +# PREPARED. The fix serializes the pending ops into the 2PC state file at +# PREPARE (TWOPHASE_RM_FILEOPS_ID records) and performs them in the FILEOPS +# post-commit / post-abort callbacks, so they survive a crash between PREPARE +# and the final decision, driven from the durable 2PC state. +# +# Unlike 064_fileops_2pc.pl (which only exercises chmod, a constructive op +# reversed via the cluster-wide UNDO chain), this test drives the deferred +# commit-time delete path -- the one that was silently dropped -- next to an +# immediate rename, which ROLLBACK PREPARED must rename back. +# +# NB: FILEOPS writes WAL/UNDO only when XLogIsNeeded() (wal_level >= replica), +# so 2PC durability requires wal_level = replica; this test runs there. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_2pc_deferred'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +max_prepared_transactions = 10 +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; + +sub file_exists +{ + my ($path) = @_; + return $node->safe_psql('postgres', + qq{SELECT test_fileops_file_exists('$path')}); +} + +# Poll until the file's existence matches $want ('t'/'f'). +sub wait_for_exists +{ + my ($path, $want, $desc) = @_; + my $ok = $node->poll_query_until('postgres', + qq{SELECT test_fileops_file_exists('$path') = '$want'}); + is($ok, '1', $desc); +} + +# ================================================================ +# Scenario A: COMMIT PREPARED performs the deferred commit-time ops +# - deferred delete of an existing file +# - immediate rename of an existing file +# - immediate create of a new file (register_delete) +# PREPARE, crash, restart, then COMMIT PREPARED. All three must take effect. +# ================================================================ +my $del_a = "$datadir/fo2pc_del_a.dat"; +my $ren_a = "$datadir/fo2pc_ren_a.dat"; +my $ren_a2 = "$datadir/fo2pc_ren_a.renamed"; +my $new_a = "$datadir/fo2pc_new_a.dat"; + +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_del_a.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_ren_a.dat')}); + +is(file_exists($del_a), 't', 'A: delete target exists before txn'); +is(file_exists($ren_a), 't', 'A: rename source exists before txn'); + +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('$del_a'); +SELECT test_fileops_rename('$ren_a', '$ren_a2'); +SELECT test_fileops_create('$new_a', 384); +PREPARE TRANSACTION 'fo2pc_commit'; +)); + +# While prepared, the deferred delete has NOT run yet; the immediate rename +# and create have. +is(file_exists($del_a), 't', + 'A: delete deferred -- file still present while prepared'); +is(file_exists($ren_a), 'f', + 'A: rename immediate -- source gone while prepared'); +is(file_exists($ren_a2), 't', + 'A: rename immediate -- dest present while prepared'); +is(file_exists($new_a), 't', 'A: immediate create present while prepared'); + +# Crash between PREPARE and COMMIT PREPARED. +$node->stop('immediate'); +$node->start; + +is($node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}), + 'fo2pc_commit', 'A: prepared xact survives crash'); + +$node->safe_psql('postgres', q{COMMIT PREPARED 'fo2pc_commit'}); + +is(file_exists($del_a), 'f', + 'A: COMMIT PREPARED performed the deferred delete'); +is(file_exists($ren_a), 'f', + 'A: COMMIT PREPARED kept the rename (source gone)'); +is(file_exists($ren_a2), 't', + 'A: COMMIT PREPARED kept the rename (dest present)'); +is(file_exists($new_a), 't', + 'A: created file persists after COMMIT PREPARED'); + +# ================================================================ +# Scenario B: ROLLBACK PREPARED does NOT perform the deferred commit-time +# delete, and DOES undo the immediate rename and create. +# PREPARE, crash, restart, then ROLLBACK PREPARED. +# ================================================================ +my $del_b = "$datadir/fo2pc_del_b.dat"; +my $ren_b = "$datadir/fo2pc_ren_b.dat"; +my $ren_b2 = "$datadir/fo2pc_ren_b.renamed"; +my $new_b = "$datadir/fo2pc_new_b.dat"; + +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_del_b.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('fo2pc_ren_b.dat')}); + +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('$del_b'); +SELECT test_fileops_rename('$ren_b', '$ren_b2'); +SELECT test_fileops_create('$new_b', 384); +PREPARE TRANSACTION 'fo2pc_rollback'; +)); + +is(file_exists($ren_b2), 't', 'B: immediate rename made while prepared'); +is(file_exists($new_b), 't', 'B: immediate create present while prepared'); + +# Crash between PREPARE and ROLLBACK PREPARED. +$node->stop('immediate'); +$node->start; + +is($node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}), + 'fo2pc_rollback', 'B: prepared xact survives crash'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo2pc_rollback'}); + +is(file_exists($del_b), 't', + 'B: ROLLBACK PREPARED did NOT delete (file still present)'); +is(file_exists($ren_b), 't', + 'B: ROLLBACK PREPARED renamed back (source present)'); +is(file_exists($ren_b2), 'f', + 'B: ROLLBACK PREPARED renamed back (dest absent)'); +# The immediate create is undone by the constructive op's cluster-wide UNDO +# record on ROLLBACK PREPARED. +wait_for_exists($new_b, 'f', + 'B: ROLLBACK PREPARED undid the immediate create'); + +# Server healthy at the end. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after all deferred-2PC scenarios'); + +$node->stop; +done_testing(); diff --git a/src/test/recovery/t/063_fileops_recovery.pl b/src/test/recovery/t/063_fileops_recovery.pl new file mode 100644 index 0000000000000..fb8f23b63ae72 --- /dev/null +++ b/src/test/recovery/t/063_fileops_recovery.pl @@ -0,0 +1,423 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test crash recovery for transactional file operations (FILEOPS). +# +# These tests verify that FILEOPS WAL replay correctly handles: +# - Crash during file creation (with delete-on-abort) +# - Crash during deferred file deletion +# - Crash during file operations on standby +# - Multiple sequential crashes + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_recovery'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +log_min_messages = debug2 +)); +$node->start; + +# ================================================================ +# Test 1: CREATE TABLE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE fileops_test (id int, data text); +INSERT INTO fileops_test VALUES (1, 'created_table'); +)); + +$node->stop('immediate'); +$node->start; + +my $result = + $node->safe_psql("postgres", "SELECT data FROM fileops_test WHERE id = 1"); +is($result, 'created_table', 'CREATE TABLE survives crash'); + +# ================================================================ +# Test 2: DROP TABLE is properly handled after crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE drop_me (id int); +INSERT INTO drop_me VALUES (1); +)); + +# Get the relfilenode before dropping +my $relpath = + $node->safe_psql("postgres", "SELECT pg_relation_filepath('drop_me')"); + +$node->safe_psql("postgres", "DROP TABLE drop_me"); + +$node->stop('immediate'); +$node->start; + +# Table should be gone +my ($ret, $stdout, $stderr) = + $node->psql("postgres", "SELECT * FROM drop_me"); +isnt($ret, 0, 'dropped table is gone after crash recovery'); + +# ================================================================ +# Test 3: Crash during transaction with CREATE TABLE (uncommitted) +# ================================================================ + +# This table is committed +$node->safe_psql( + "postgres", qq( +CREATE TABLE committed_table (id int); +INSERT INTO committed_table VALUES (42); +)); + +# Crash the server +$node->stop('immediate'); +$node->start; + +# Committed table should exist +$result = $node->safe_psql("postgres", "SELECT id FROM committed_table"); +is($result, '42', 'committed CREATE TABLE survives crash'); + +# ================================================================ +# Test 4: Multiple CREATE and DROP operations then crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE TABLE t1 (id int); +CREATE TABLE t2 (id int); +CREATE TABLE t3 (id int); +INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (2); +INSERT INTO t3 VALUES (3); +DROP TABLE t2; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT id FROM t1"); +is($result, '1', 't1 survives crash'); + +($ret, $stdout, $stderr) = $node->psql("postgres", "SELECT * FROM t2"); +isnt($ret, 0, 't2 (dropped) is gone after crash'); + +$result = $node->safe_psql("postgres", "SELECT id FROM t3"); +is($result, '3', 't3 survives crash'); + +# ================================================================ +# Test 5: Crash after checkpoint with file operations +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t3; +CREATE TABLE checkpoint_test (id int); +INSERT INTO checkpoint_test VALUES (1); +CHECKPOINT; +INSERT INTO checkpoint_test VALUES (2); +)); + +$node->stop('immediate'); +$node->start; + +$result = + $node->safe_psql("postgres", "SELECT count(*) FROM checkpoint_test"); +is($result, '2', 'data after checkpoint survives crash'); + +# ================================================================ +# Test 6: Multiple crashes in sequence with file operations +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS checkpoint_test; +CREATE TABLE multi_crash (id int); +INSERT INTO multi_crash VALUES (1); +)); + +$node->stop('immediate'); +$node->start; + +$node->safe_psql( + "postgres", qq( +INSERT INTO multi_crash VALUES (2); +CREATE TABLE multi_crash_2 (id int); +INSERT INTO multi_crash_2 VALUES (10); +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT count(*) FROM multi_crash"); +is($result, '2', 'multi_crash table correct after double crash'); + +$result = $node->safe_psql("postgres", "SELECT id FROM multi_crash_2"); +is($result, '10', 'multi_crash_2 table correct after double crash'); + +# ================================================================ +# Test 7: CREATE TABLESPACE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE IF EXISTS multi_crash; +DROP TABLE IF EXISTS multi_crash_2; +SET allow_in_place_tablespaces = on; +CREATE TABLESPACE fileops_crash_ts LOCATION ''; +CREATE TABLE ts_crash_table (id int) TABLESPACE fileops_crash_ts; +INSERT INTO ts_crash_table VALUES (99); +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", "SELECT id FROM ts_crash_table"); +is($result, '99', 'table in tablespace survives crash'); + +$result = $node->safe_psql("postgres", + "SELECT spcname FROM pg_tablespace WHERE spcname = 'fileops_crash_ts'"); +is($result, 'fileops_crash_ts', 'tablespace survives crash'); + +# ================================================================ +# Test 8: DROP TABLESPACE completes after crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +DROP TABLE ts_crash_table; +DROP TABLESPACE fileops_crash_ts; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", + "SELECT count(*) FROM pg_tablespace WHERE spcname = 'fileops_crash_ts'"); +is($result, '0', 'dropped tablespace is gone after crash'); + +# ================================================================ +# Test 9: CREATE DATABASE survives crash +# ================================================================ + +$node->safe_psql( + "postgres", qq( +CREATE DATABASE fileops_crash_db; +)); + +$node->stop('immediate'); +$node->start; + +$result = $node->safe_psql("postgres", + "SELECT datname FROM pg_database WHERE datname = 'fileops_crash_db'"); +is($result, 'fileops_crash_db', 'CREATE DATABASE survives crash'); + +$node->safe_psql("postgres", "DROP DATABASE fileops_crash_db"); + +# ================================================================ +# Test 10: Standby crash during FILEOPS replay +# ================================================================ + +# Set up primary + standby +my $primary = PostgreSQL::Test::Cluster->new('fileops_primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + "postgresql.conf", qq( +autovacuum = off +)); +$primary->start; +$primary->backup('backup'); + +my $standby = PostgreSQL::Test::Cluster->new('fileops_standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; + +# Create table on primary and wait for standby to catch up +$primary->safe_psql( + "postgres", qq( +CREATE TABLE standby_test (id int); +INSERT INTO standby_test VALUES (1); +)); + +$primary->wait_for_catchup($standby); + +# Verify on standby +$result = $standby->safe_psql("postgres", "SELECT id FROM standby_test"); +is($result, '1', 'CREATE TABLE replicated to standby'); + +# Crash the standby +$standby->stop('immediate'); +$standby->start; + +# Add more data on primary +$primary->safe_psql( + "postgres", qq( +INSERT INTO standby_test VALUES (2); +)); + +$primary->wait_for_catchup($standby); + +$result = + $standby->safe_psql("postgres", "SELECT count(*) FROM standby_test"); +is($result, '2', 'standby recovers and catches up after crash'); + +# Clean up primary/standby +$standby->stop; +$primary->stop; + +# Clean up original node +$node->stop; + +# ================================================================ +# Test 11/12: Crash right after an immediate op's WAL is flushed, before +# its transaction commits (FileOpsCreate, FileOpsChmod). +# +# An immediate op makes its filesystem change first and then flushes its +# WAL and UNDO records. The injection points pause the backend just after +# that flush, inside the still-open implicit transaction, and the postmaster +# is then hard-killed. Redo replays the change, and crash recovery applies +# the UNDO of the never-committed transaction, so the end state must be as +# if the transaction never ran: no orphaned file, no leftover mode change. +# ================================================================ + +if ($ENV{enable_injection_points} eq 'yes') +{ + my $inode = PostgreSQL::Test::Cluster->new('fileops_wal_ordering'); + $inode->init; + $inode->append_conf( + 'postgresql.conf', qq( +shared_preload_libraries = 'injection_points' +autovacuum = off +log_min_messages = debug2 +wal_level = replica +)); + $inode->start; + + if ( $inode->check_extension('injection_points') + && $inode->check_extension('test_fileops')) + { + $inode->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + $inode->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + + my $idatadir = $inode->data_dir; + + # --- Test 11: FileOpsCreate ----------------------------------- + $inode->safe_psql('postgres', + q{SELECT injection_points_attach('fileops-create-after-wal-flush', 'wait')} + ); + + my $create_path = "$idatadir/wal_ordering_create.dat"; + my $bgpsql = $inode->background_psql('postgres'); + $bgpsql->query_until( + qr/starting_create/, + qq(\\echo starting_create +SELECT test_fileops_create('$create_path', 384); +\\q +)); + + $inode->wait_for_event('client backend', + 'fileops-create-after-wal-flush'); + + # The file exists and its WAL is durable; the transaction is open. + ok(-e $create_path, + 'Test 11: file exists at the injection point (WAL durable)'); + + # Hard-kill the postmaster from exactly this window. The wrapping + # implicit transaction is still in progress (blocked on the + # injection point), so it never committed. + $inode->stop('immediate'); + $inode->start; + + # Recovery redoes the physical CREATE from the durable WAL record + # unconditionally, then determines the transaction never committed + # and applies its UNDO record to reverse the create. Net effect: + # the file must NOT exist -- exactly as if the transaction had + # never run, with no orphaned state left behind by the crash. + ok( !-e $create_path, + 'Test 11: FileOpsCreate of a not-committed transaction is ' + . 'reversed after a crash'); + + # Server is healthy after the reconciliation. + my $result11 = $inode->safe_psql('postgres', 'SELECT 1'); + is($result11, '1', 'Test 11: server operational after recovery'); + + $bgpsql->{run}->finish; + + # --- Test 12: FileOpsChmod ------------------------------------- + $inode->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('wal_ordering_chmod.dat')} + ); + my $chmod_path = "$idatadir/wal_ordering_chmod.dat"; + $inode->safe_psql('postgres', + qq{SELECT test_fileops_chmod('$chmod_path', 420)}); # 0644 + my $mode_before = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + # Windows has no POSIX file modes: chmod only toggles read-only. + my $posix_modes = !$windows_os; + is($mode_before, '420', 'Test 12: baseline mode is 0644') + if $posix_modes; + + $inode->safe_psql('postgres', + q{SELECT injection_points_attach('fileops-chmod-after-wal-flush', 'wait')} + ); + + $bgpsql = $inode->background_psql('postgres'); + $bgpsql->query_until( + qr/starting_chmod/, + qq(\\echo starting_chmod +SELECT test_fileops_chmod('$chmod_path', 448); -- 0700 +\\q +)); + + $inode->wait_for_event('client backend', + 'fileops-chmod-after-wal-flush'); + + # chmod() has run and its WAL is durable; the transaction is open. + my $mode_at_injection = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + is($mode_at_injection, '448', + 'Test 12: mode changed on disk at the injection point (WAL durable)' + ) if $posix_modes; + + # Hard-kill, same as Test 11. The chmod's wrapping implicit + # transaction never committed. + $inode->stop('immediate'); + $inode->start; + + # Recovery redoes the physical CHMOD unconditionally, then reverses + # it via UNDO because the transaction never committed. Net effect: + # mode must be back to the pre-transaction baseline (0644), not the + # attempted 0700, and not some undefined intermediate state. + my $mode_after = $inode->safe_psql('postgres', + qq{SELECT test_fileops_get_mode('$chmod_path')}); + is($mode_after, '420', + 'Test 12: FileOpsChmod of a not-committed transaction is ' + . 'reversed after a crash') + if $posix_modes; + + # Server is healthy after the reconciliation. + my $result12 = $inode->safe_psql('postgres', 'SELECT 1'); + is($result12, '1', 'Test 12: server operational after recovery'); + + $bgpsql->{run}->finish; + } + else + { + note + 'skipping Test 11/12 (injection_points or test_fileops extension not installed)'; + } + + $inode->stop; +} +else +{ + note 'skipping Test 11/12 (injection points not supported by this build)'; +} + +done_testing(); diff --git a/src/test/recovery/t/064_fileops_2pc.pl b/src/test/recovery/t/064_fileops_2pc.pl new file mode 100644 index 0000000000000..b88c8dd59103b --- /dev/null +++ b/src/test/recovery/t/064_fileops_2pc.pl @@ -0,0 +1,149 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Test two-phase commit (PREPARE TRANSACTION / COMMIT PREPARED / ROLLBACK +# PREPARED) for transactions that generated cluster-wide FILEOPS UNDO. +# +# FILEOPS uses the undo-in-common-WAL mechanism. At PREPARE the permanent +# UNDO chain-head LSN is durably saved in the 2PC state (xl_xact_prepare / +# gxact->undo_batch_lsn), and its WAL is retained while the xact stays +# prepared. ROLLBACK PREPARED (even from a different backend, post-crash) +# applies that chain, which reverses the filesystem change. COMMIT PREPARED +# keeps it. +# +# Vehicle: the test_fileops extension's SQL wrappers for FileOpsChmod, which +# are legal inside a transaction block that can then be PREPARE'd. +# +# NB: FILEOPS only writes UNDO WAL when XLogIsNeeded() (wal_level >= replica). +# At wal_level = minimal it relies on the backend-private pending-op list, +# which is discarded at PREPARE and cannot survive 2PC. This test therefore +# runs at wal_level = replica, which is what any FILEOPS + 2PC deployment +# requires. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Every test here is a chmod reversal; Windows has no POSIX file modes. +plan skip_all => 'POSIX permission semantics required' if $windows_os; + +my $node = PostgreSQL::Test::Cluster->new('fileops_2pc'); +$node->init; +$node->append_conf( + "postgresql.conf", qq( +autovacuum = off +max_prepared_transactions = 10 +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; +my $file = "$datadir/twopc.dat"; + +# Poll until the file mode equals $want, or fail. +sub wait_for_mode +{ + my ($want, $desc) = @_; + my $ok = $node->poll_query_until('postgres', + qq{SELECT test_fileops_get_mode('$file') = $want}); + is($ok, '1', $desc); +} + +# 0644 = 420, 0700 = 448, 0600 = 384 (decimal) +$node->safe_psql('postgres', + qq{SELECT test_fileops_create_tempfile('twopc.dat')}); +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); +my $baseline = + $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($baseline, '420', 'baseline mode is 0644'); + +# ================================================================ +# Test 1: ROLLBACK PREPARED reverses the chmod +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_rb'; +)); +my $m = + $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 1: chmod applied and still visible while prepared'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_rb'}); +wait_for_mode(420, 'Test 1: ROLLBACK PREPARED reverses chmod (0700 -> 0644)'); + +# ================================================================ +# Test 2: COMMIT PREPARED keeps the chmod +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_cp'; +)); +$node->safe_psql('postgres', q{COMMIT PREPARED 'fo_cp'}); +$m = $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 2: COMMIT PREPARED keeps chmod (0700)'); + +# Reset to baseline for the crash test. +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); + +# ================================================================ +# Test 3: WAL retention across checkpoints while prepared +# ================================================================ +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 384); -- 0600 +PREPARE TRANSACTION 'fo_ckpt'; +)); +# Churn WAL and checkpoint; the prepared xact's UNDO batch must be retained. +$node->safe_psql('postgres', 'CHECKPOINT'); +for my $i (1 .. 20) +{ + $node->safe_psql('postgres', + "CREATE TABLE junk_$i(x int); " + . "INSERT INTO junk_$i SELECT g FROM generate_series(1,5000) g; " + . "DROP TABLE junk_$i;"); +} +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_ckpt'}); +wait_for_mode(420, + 'Test 3: ROLLBACK PREPARED still reverses after WAL churn + checkpoints'); + +# ================================================================ +# Test 4: crash recovery -- prepared UNDO xact survives, then reverts +# ================================================================ +$node->safe_psql('postgres', qq{SELECT test_fileops_chmod('$file', 420)}); +$node->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_chmod('$file', 448); -- 0700 +PREPARE TRANSACTION 'fo_crash'; +)); +$node->stop('immediate'); +$node->start; + +my $prep = $node->safe_psql('postgres', q{SELECT gid FROM pg_prepared_xacts}); +is($prep, 'fo_crash', 'Test 4: prepared xact survives crash recovery'); +$m = $node->safe_psql('postgres', qq{SELECT test_fileops_get_mode('$file')}); +is($m, '448', 'Test 4: chmod still applied (prepared, not yet rolled back)'); + +$node->safe_psql('postgres', q{ROLLBACK PREPARED 'fo_crash'}); +wait_for_mode(420, + 'Test 4: ROLLBACK PREPARED after crash+restart reverses chmod'); + +# Server healthy at the end. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after all 2PC UNDO scenarios'); + +$node->stop; +done_testing(); diff --git a/src/test/recovery/t/065_fileops_standby_divergence.pl b/src/test/recovery/t/065_fileops_standby_divergence.pl new file mode 100644 index 0000000000000..65a07fb873e04 --- /dev/null +++ b/src/test/recovery/t/065_fileops_standby_divergence.pl @@ -0,0 +1,124 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# Regression test: physical-standby reproduction of FILEOPS deferred +# DESTRUCTIVE operations (DELETE, RENAME, RMDIR). +# +# These three ops execute on the primary at commit inside +# FileOpsDoPendingOps(). Each is logged at commit-EXECUTION time with a +# real, idempotent redo handler, so a physical standby (which only replays +# WAL, never running FileOpsDoPendingOps) reproduces them and its filesystem +# converges with the primary's. +# +# This test sets up a streaming physical standby, performs a committed +# deferred DELETE, RENAME, and RMDIR on the primary, waits for catchup, and +# asserts the standby's filesystem MATCHES the primary's. It MUST FAIL on the +# pre-fix (no-op redo) code -- see the header note in fileops.c and the +# "prove non-vacuity" comment below. +# +# All FILEOPS paths are RELATIVE to the data directory, matching every core +# FILEOPS caller (dbcommands.c, tablespace.c): the backend's cwd is PGDATA, so +# a relative path resolves to the right file on both the primary and a standby +# whose absolute data_dir differs. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('fileops_div_primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + "postgresql.conf", qq( +autovacuum = off +wal_level = replica +)); +$primary->start; + +if (!$primary->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $primary_dir = $primary->data_dir; + +# Fixture, all committed and WAL-logged so the standby has them before we +# start mutating. Use FileOps CREATE (WAL-logged, standby-reproduced) and +# FileOps MKDIR (WAL-logged) rather than raw open(), so the fixture itself +# replicates. Paths are relative to PGDATA. +$primary->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_create('div_delete_me.dat', 384); -- 0600 +SELECT test_fileops_create('div_rename_src.dat', 384); +SELECT test_fileops_mkdir('div_rmdir_me', 448); -- 0700 +COMMIT; +)); + +# Take the base backup AFTER the fixture exists so init_from_backup gives the +# standby the fixture files, then stream the mutations. +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('fileops_div_standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; + +my $standby_dir = $standby->data_dir; + +$primary->wait_for_catchup($standby); + +# Sanity: the fixture replicated to the standby. +ok(-e "$standby_dir/div_delete_me.dat", 'fixture: delete target on standby'); +ok(-e "$standby_dir/div_rename_src.dat", 'fixture: rename source on standby'); +ok(-d "$standby_dir/div_rmdir_me", 'fixture: rmdir target on standby'); + +# Perform the three deferred destructive ops in a committed transaction on the +# primary. Each goes through the FileOps deferred path (at_commit=true) and, +# with the fix, is WAL-logged at commit-execution. +$primary->safe_psql( + 'postgres', qq( +BEGIN; +SELECT test_fileops_delete('div_delete_me.dat'); +SELECT test_fileops_rename('div_rename_src.dat', 'div_rename_dst.dat'); +SELECT test_fileops_rmdir('div_rmdir_me'); +COMMIT; +)); + +# Confirm the primary applied them (the op ran in FileOpsDoPendingOps). +ok(!-e "$primary_dir/div_delete_me.dat", 'primary: delete applied'); +ok(!-e "$primary_dir/div_rename_src.dat", 'primary: rename source gone'); +ok(-e "$primary_dir/div_rename_dst.dat", 'primary: rename dest present'); +ok(!-d "$primary_dir/div_rmdir_me", 'primary: rmdir applied'); + +$primary->wait_for_catchup($standby); + +# The diff the reviewer asked for: the standby's filesystem must MATCH the +# primary's. Pre-fix (no-op redo) the standby still has the original file / +# dir and lacks the renamed file -- these four assertions all fail, which is +# the non-vacuity proof. Post-fix they pass. +ok(!-e "$standby_dir/div_delete_me.dat", + 'deferred DELETE reproduced on standby (file gone)'); +ok(!-e "$standby_dir/div_rename_src.dat", + 'deferred RENAME reproduced on standby (old name gone)'); +ok(-e "$standby_dir/div_rename_dst.dat", + 'deferred RENAME reproduced on standby (new name present)'); +ok(!-d "$standby_dir/div_rmdir_me", + 'deferred RMDIR reproduced on standby (dir gone)'); + +# Idempotent redo across a standby restart: replaying the same records again +# after a crash must not error or diverge. +$standby->stop('immediate'); +$standby->start; +$primary->safe_psql('postgres', 'SELECT txid_current()'); +$primary->wait_for_catchup($standby); +ok(!-e "$standby_dir/div_delete_me.dat", + 'standby end state stable after restart (delete)'); +ok(-e "$standby_dir/div_rename_dst.dat", + 'standby end state stable after restart (rename)'); +ok(!-d "$standby_dir/div_rmdir_me", + 'standby end state stable after restart (rmdir)'); + +$standby->stop; +$primary->stop; + +done_testing(); diff --git a/src/test/recovery/t/066_fileops_crash_undo.pl b/src/test/recovery/t/066_fileops_crash_undo.pl new file mode 100644 index 0000000000000..f305b2814bdad --- /dev/null +++ b/src/test/recovery/t/066_fileops_crash_undo.pl @@ -0,0 +1,137 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group +# +# FILEOPS crash-recovery revert via the cluster-wide UNDO engine (common WAL). +# +# FILEOPS is a consumer of the cluster-wide UNDO engine: its structural +# operations (file existence, permissions, directory structure) are +# WAL-durable via RM_FILEOPS_ID forward records (fileops_redo), and each +# reversible op also writes an UNDO record into the common WAL +# (PrepareXactUndoData(UNDO_RMID_FILEOPS,...) / InsertXactUndoData) in the +# same critical section as its forward record. Filesystem structural +# operations are cluster-wide changes whose durability/revert belong in the +# common WAL. This test proves that model is crash-safe. +# +# A transaction produces FILEOPS create/chmod (immediate, WAL-flushed), then the +# server is crashed (immediate stop) BEFORE the transaction commits. On +# recovery: +# - fileops_redo() REDOES the forward create/chmod unconditionally, and +# - UndoRecoveryRollBack() (end of recovery) applies the never-committed +# transaction's FILEOPS UNDO chain via the RM_FILEOPS_ID rm_undo +# callbacks (fileops_undo.c), reversing every op: +# - the created file is unlinked (CREATE revert), and +# - the chmod'd file's mode is restored (CHMOD revert, before-state in WAL). +# +# FILEOPS only writes WAL at wal_level >= replica (XLogIsNeeded()), so the test +# runs at wal_level = replica, matching any FILEOPS deployment. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('fileops_crash'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +autovacuum = off +wal_level = replica +)); +$node->start; + +if (!$node->check_extension('test_fileops')) +{ + plan skip_all => 'Extension test_fileops not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $datadir = $node->data_dir; +my $create_path = "$datadir/fileops_crash_create.dat"; +my $chmod_path = "$datadir/fileops_crash_chmod.dat"; + +# --------------------------------------------------------------------------- +# Set up a pre-existing file to chmod (this part is committed). 0644 = 420. +# --------------------------------------------------------------------------- +$node->safe_psql('postgres', + q{SELECT test_fileops_create_tempfile('fileops_crash_chmod.dat')}); +$node->safe_psql('postgres', + qq{SELECT test_fileops_chmod('$chmod_path', 420)}); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '420', + 'baseline mode of pre-existing file is 0644'); +} + +# --------------------------------------------------------------------------- +# Open a transaction that produces FILEOPS UNDO (create a new file and +# chmod the existing one) and DO NOT commit it. The FILEOPS forward operations +# execute and are WAL-flushed immediately (the file exists on disk, the mode is +# changed), but the wrapping transaction stays in progress. +# --------------------------------------------------------------------------- +my $bg = $node->background_psql('postgres'); +$bg->query_until( + qr/fileops_ready/, qq( +BEGIN; +SELECT test_fileops_create('$create_path', 384); -- 0600, new file +SELECT test_fileops_chmod('$chmod_path', 448); -- 0700, existing file +\\echo fileops_ready +)); + +# The forward ops are visible on disk while the xact is in progress. +ok(-e $create_path, + 'created file exists on disk before crash (in-progress xact)'); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '448', + 'chmod applied on disk before crash (in-progress xact)'); +} + +# --------------------------------------------------------------------------- +# Crash immediately: the transaction never commits. Stop the server before +# closing the client: closing it first would let the backend roll the +# transaction back online, which is a different case. The background psql +# connection dies with the server; abandon it (do not try to drain/finish it, +# which would hang waiting on the dead connection). +# --------------------------------------------------------------------------- +$node->stop('immediate'); +$bg->{run}->kill_kill; +$node->start; + +# --------------------------------------------------------------------------- +# Recovery redoes the forward ops from WAL, then UndoRecoveryRollBack() reverses +# the never-committed transaction's ops from its common-WAL UNDO chain: +# - the created file is unlinked, and +# - the chmod is restored to the original 0644. +# --------------------------------------------------------------------------- +ok( !-e $create_path, + 'FILEOPS COMMON-WAL CRASH RECOVERY: created file was REVERTED (unlinked) after crash' +); +SKIP: +{ + skip 'POSIX permission semantics required', 1 if $windows_os; + is( $node->safe_psql( + 'postgres', qq{SELECT test_fileops_get_mode('$chmod_path')}), + '420', + 'FILEOPS COMMON-WAL CRASH RECOVERY: chmod was REVERTED (0700 -> 0644) after crash' + ); +} + +# Recovery must be clean. +my $rlog = slurp_file($node->logfile); +unlike( + $rlog, + qr/PANIC|inconsistent|corrupt/i, + 'clean crash recovery, no PANIC/corruption'); + +# Server healthy afterwards. +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'server operational after crash-recovery undo'); + +$node->stop; +done_testing(); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index bdaea778ee5bb..6e627beebb0fd 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -836,6 +836,19 @@ ExtensionVersionInfo FDWCollateState FD_SET FILE +FileOpsSyncEntry +FileOpsTwoPhaseRecord +FileopsUndoChmod +FileopsUndoChown +FileopsUndoCreate +FileopsUndoLink +FileopsUndoMkdir +FileopsUndoRemovexattr +FileopsUndoRename +FileopsUndoSetxattr +FileopsUndoSymlink +FileopsUndoTruncate +FileopsUndoWrite FILETIME FPI FSMAddress @@ -1912,6 +1925,8 @@ PACE_HEADER PACL PATH PCtxtHandle +PendingFileOp +PendingFileOpType PERL_CONTEXT PERL_SI PFN @@ -3507,6 +3522,19 @@ XactUndoContext XactUndoData XactUndoInflightSlot XactUndoSubTransactionState +xl_fileops_chmod +xl_fileops_chown +xl_fileops_create +xl_fileops_delete +xl_fileops_link +xl_fileops_mkdir +xl_fileops_removexattr +xl_fileops_rename +xl_fileops_rmdir +xl_fileops_setxattr +xl_fileops_symlink +xl_fileops_truncate +xl_fileops_write xl_undo_batch xl_undo_inflight xl_undo_inflight_xact From 0af85aebe85cb30dd9a58bb7f7d9422279e76122 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 14 Sep 2026 13:03:40 +0000 Subject: [PATCH 5/5] Route filesystem mutations through FILEOPS Route the server's transactional DDL/storage callers that mutate the filesystem outside the buffer manager through the FILEOPS resource manager added in the previous commit, so an abort or crash recovery recursively reverts a partial filesystem change instead of leaving orphan trees behind. - copydir() gains a register_for_abort_cleanup argument: forward-path callers register the destination tree with FILEOPS (FileOpsRmtree at abort), while WAL-replay callers pass false because the redo handler is itself the recovery action. - CREATE DATABASE (STRATEGY=FILE_COPY) and ALTER DATABASE SET TABLESPACE copy their directory trees with abort cleanup enabled, closing the long-standing crash-asymmetry where a rolled-back copy left an orphan tree. - CREATE/DROP TABLESPACE route their directory create/remove through FileOps* so tablespace symlink and directory changes are transactional. Author: Greg Burd Reviewed-by: shihao zhong Reviewed-by: Manuel Reyes Bravo Discussion: https://postgr.es/m/7BNP_kRdfRAWyq4swht7lEjtTjEyIPgbDLkR7PXvp9qgNPfXb_fbg4k8ceAd6j-4genQldy9CY9j9KthZUI0SducvaQsxSDoa51e0GdEtQo=@burd.me Changes since v9: - CREATE DATABASE (both strategies) and the copydir() destination create their directory with FileOpsMkdirTree(), so an abort or a crash part way through the copy removes the whole tree. v9 only rmdir()ed it, which failed on the copied files, and FILE_COPY's cleanup was kept in memory only (Shihao; recovery/066, with injection points createdb-wal-log-after-copy-relation and createdb-file-copy-after-copydir) - test_fileops/010 checks that abort redo replays every kind of inverse, including a CREATE DATABASE that fails part way Changes since v10: - recovery/066_fileops_createdb_crash is now 067 --- src/backend/commands/dbcommands.c | 152 +++++++++---- src/backend/commands/tablespace.c | 175 +++++++++++---- src/backend/storage/file/copydir.c | 23 +- src/include/storage/copydir.h | 14 +- src/test/modules/test_fileops/meson.build | 1 + .../t/007_fileops_commit_crash.pl | 18 ++ .../test_fileops/t/010_fileops_abort_redo.pl | 202 ++++++++++++++++++ src/test/recovery/meson.build | 1 + .../recovery/t/067_fileops_createdb_crash.pl | 113 ++++++++++ 9 files changed, 602 insertions(+), 97 deletions(-) create mode 100644 src/test/modules/test_fileops/t/010_fileops_abort_redo.pl create mode 100644 src/test/recovery/t/067_fileops_createdb_crash.pl diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 15b93512e4bf5..3b0a8c4e1d5c4 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -57,6 +57,7 @@ #include "replication/slot.h" #include "storage/copydir.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/ipc.h" #include "storage/lmgr.h" #include "storage/md.h" @@ -222,6 +223,8 @@ CreateDatabaseUsingWalLog(Oid src_dboid, Oid dst_dboid, CreateAndCopyRelationData(srcrlocator, dstrlocator, relinfo->permanent, relinfo->mainfork_page_std); + INJECTION_POINT("createdb-wal-log-after-copy-relation", NULL); + /* Release the relation locks. */ UnlockRelationId(&srcrelid, AccessShareLock); UnlockRelationId(&dstrelid, AccessShareLock); @@ -488,56 +491,95 @@ CreateDirAndVersionFile(char *dbpath, Oid dbid, Oid tsid, bool isRedo) sprintf(buf, "%s\n", PG_MAJORVERSION); nbytes = strlen(PG_MAJORVERSION) + 1; - /* Create database directory. */ - if (MakePGDirectory(dbpath) < 0) + /* + * Create database directory. The copy fills it through smgr, so an abort + * or crash before commit must remove the whole tree. + */ + if (!isRedo) { - /* Failure other than already exists or not in WAL replay? */ - if (errno != EEXIST || !isRedo) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create directory \"%s\": %m", dbpath))); + FileOpsMkdirTree(dbpath, pg_dir_create_mode, dbid); + } + else + { + if (MakePGDirectory(dbpath) < 0) + { + /* Failure other than already exists or not in WAL replay? */ + if (errno != EEXIST) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", dbpath))); + } } /* - * Create PG_VERSION file in the database path. If the file already - * exists and we are in WAL replay then try again to open it in write - * mode. + * Create PG_VERSION file in the database path. */ snprintf(versionfile, sizeof(versionfile), "%s/%s", dbpath, "PG_VERSION"); - fd = OpenTransientFile(versionfile, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY); - if (fd < 0 && errno == EEXIST && isRedo) - fd = OpenTransientFile(versionfile, O_WRONLY | O_TRUNC | PG_BINARY); + if (!isRedo) + { + /* + * Use FileOpsCreate + FileOpsWrite for transactional version file + * creation with crash-safe rollback support. + */ + fd = FileOpsCreate(versionfile, O_WRONLY | PG_BINARY, + pg_file_create_mode, true); + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_WRITE); + errno = 0; + if ((int) write(fd, buf, nbytes) != nbytes) + { + if (errno == 0) + errno = ENOSPC; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", versionfile))); + } + pgstat_report_wait_end(); - if (fd < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create file \"%s\": %m", versionfile))); + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_SYNC); + if (pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", versionfile))); + fsync_fname(dbpath, true); + pgstat_report_wait_end(); - /* Write PG_MAJORVERSION in the PG_VERSION file. */ - pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_WRITE); - errno = 0; - if (write(fd, buf, nbytes) != nbytes) - { - /* If write didn't set errno, assume problem is no disk space. */ - if (errno == 0) - errno = ENOSPC; - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not write to file \"%s\": %m", versionfile))); + CloseTransientFile(fd); } - pgstat_report_wait_end(); + else + { + /* WAL replay: use raw file operations */ + fd = OpenTransientFile(versionfile, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY); + if (fd < 0 && errno == EEXIST) + fd = OpenTransientFile(versionfile, O_WRONLY | O_TRUNC | PG_BINARY); - pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_SYNC); - if (pg_fsync(fd) != 0) - ereport(data_sync_elevel(ERROR), - (errcode_for_file_access(), - errmsg("could not fsync file \"%s\": %m", versionfile))); - fsync_fname(dbpath, true); - pgstat_report_wait_end(); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", versionfile))); - /* Close the version file. */ - CloseTransientFile(fd); + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_WRITE); + errno = 0; + if ((int) write(fd, buf, nbytes) != nbytes) + { + if (errno == 0) + errno = ENOSPC; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", versionfile))); + } + pgstat_report_wait_end(); + + pgstat_report_wait_start(WAIT_EVENT_VERSION_FILE_SYNC); + if (pg_fsync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", versionfile))); + fsync_fname(dbpath, true); + pgstat_report_wait_end(); + + CloseTransientFile(fd); + } /* If we are not in WAL replay then write the WAL. */ if (!isRedo) @@ -649,9 +691,13 @@ CreateDatabaseUsingFileCopy(Oid src_dboid, Oid dst_dboid, Oid src_tsid, /* * Copy this subdirectory to the new location * - * We don't need to copy subdirectories + * We don't need to copy subdirectories. Register the destination for + * delete-on-abort so a transaction abort recursively removes the + * partial copy. */ - copydir(srcpath, dstpath, false); + copydir(srcpath, dstpath, false, true); + + INJECTION_POINT("createdb-file-copy-after-copydir", NULL); /* Record the filesystem change in XLOG */ { @@ -2237,8 +2283,17 @@ movedb(const char *dbname, const char *tblspcname) FreeDir(dstdir); /* - * The directory exists but is empty. We must remove it before using + * The directory exists but is empty. We must remove it before using * the copydir function. + * + * NB: this rmdir is deliberately NOT routed through FILEOPS. The + * destination tablespace path is not yet referenced by any catalog + * row (the pg_database.dattablespace update happens later, and is + * what binds dst_dbpath to the database on commit), so an abort that + * leaves dst_dbpath missing does not orphan any catalog state -- the + * database remains in src_tblspcoid. copydir() below recreates + * dst_dbpath with FileOpsMkdirTree(), so an abort or a crash before + * commit removes the partial copy. */ if (rmdir(dst_dbpath) != 0) elog(ERROR, "could not remove directory \"%s\": %m", @@ -2261,9 +2316,13 @@ movedb(const char *dbname, const char *tblspcname) bool new_record_repl[Natts_pg_database] = {0}; /* - * Copy files from the old tablespace to the new one + * Copy files from the old tablespace to the new one. Register the + * destination for delete-on-abort so a transaction abort recursively + * removes the partial copy (closing the long-standing crash-asymmetry + * with the source tree, which is preserved until commit by movedb's + * post-commit cleanup). */ - copydir(src_dbpath, dst_dbpath, false); + copydir(src_dbpath, dst_dbpath, false, true); /* * Record the filesystem change in XLOG @@ -3423,9 +3482,12 @@ dbase_redo(XLogReaderState *record) /* * Copy this subdirectory to the new location * - * We don't need to copy subdirectories + * We don't need to copy subdirectories. Pass false for + * register_for_abort_cleanup: this is a WAL replay path with no + * surrounding transaction; the redo handler is itself the recovery + * action. */ - copydir(src_path, dst_path, false); + copydir(src_path, dst_path, false, false); pfree(src_path); pfree(dst_path); diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c index 7c40176b7b9d2..0ba99f635afc3 100644 --- a/src/backend/commands/tablespace.c +++ b/src/backend/commands/tablespace.c @@ -70,6 +70,7 @@ #include "miscadmin.h" #include "postmaster/bgwriter.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "storage/lmgr.h" #include "storage/lwlock.h" #include "storage/procsignal.h" @@ -600,11 +601,18 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) if (in_place) { - if (MakePGDirectory(linkloc) < 0 && errno != EEXIST) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create directory \"%s\": %m", - linkloc))); + if (!InRecovery) + { + FileOpsMkdir(linkloc, pg_dir_create_mode); + } + else + { + if (MakePGDirectory(linkloc) < 0 && errno != EEXIST) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", + linkloc))); + } } location_with_version_dir = psprintf("%s/%s", in_place ? linkloc : location, @@ -615,20 +623,36 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) * it doesn't exist or has the wrong owner. Not needed for in-place mode, * because in that case we created the directory with the desired * permissions. + * + * Outside recovery, route through FileOpsChmod so a transaction abort + * restores the prior mode via the FILEOPS UNDO record. In recovery + * (where there is no surrounding transaction) issue chmod directly. On + * Windows, chmod is partial (only the read-only bit is meaningful); + * FileOpsChmod handles platform differences. */ - if (!in_place && chmod(location, pg_dir_create_mode) != 0) + if (!in_place) { - if (errno == ENOENT) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_FILE), - errmsg("directory \"%s\" does not exist", location), - InRecovery ? errhint("Create this directory for the tablespace before " - "restarting the server.") : 0)); + int rc; + + if (InRecovery) + rc = chmod(location, pg_dir_create_mode); else - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not set permissions on directory \"%s\": %m", - location))); + rc = FileOpsChmod(location, pg_dir_create_mode); + + if (rc != 0) + { + if (errno == ENOENT) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FILE), + errmsg("directory \"%s\" does not exist", location), + InRecovery ? errhint("Create this directory for the tablespace before " + "restarting the server.") : 0)); + else + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set permissions on directory \"%s\": %m", + location))); + } } /* @@ -645,6 +669,8 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) (errcode_for_file_access(), errmsg("could not stat directory \"%s\": %m", location_with_version_dir))); + else if (!InRecovery) + FileOpsMkdir(location_with_version_dir, pg_dir_create_mode); else if (MakePGDirectory(location_with_version_dir) < 0) ereport(ERROR, (errcode_for_file_access(), @@ -671,11 +697,16 @@ create_tablespace_directories(const char *location, const Oid tablespaceoid) /* * Create the symlink under PGDATA */ - if (!in_place && symlink(location, linkloc) < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create symbolic link \"%s\": %m", - linkloc))); + if (!in_place) + { + if (!InRecovery) + FileOpsSymlink(location, linkloc); + else if (symlink(location, linkloc) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create symbolic link \"%s\": %m", + linkloc))); + } pfree(linkloc); pfree(location_with_version_dir); @@ -773,27 +804,50 @@ destroy_tablespace_directories(Oid tablespaceoid, bool redo) return false; } - /* remove empty directory */ - if (rmdir(subfile) < 0) - ereport(redo ? LOG : ERROR, - (errcode_for_file_access(), - errmsg("could not remove directory \"%s\": %m", - subfile))); + /* + * Remove empty directory. Outside recovery, defer to commit time so + * a transaction abort leaves the per-DB tablespace subdirectory + * intact. In recovery, issue rmdir directly because the redo handler + * is itself the recovery action. + */ + if (redo) + { + if (rmdir(subfile) < 0) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + subfile))); + } + else + { + FileOpsRmdir(subfile, true /* at_commit */ ); + } pfree(subfile); } FreeDir(dirdesc); - /* remove version directory */ - if (rmdir(linkloc_with_version_dir) < 0) + /* + * Remove version directory. Same forward / redo split as above: defer to + * commit outside recovery so abort restores the version dir; issue rmdir + * directly during redo. + */ + if (redo) { - ereport(redo ? LOG : ERROR, - (errcode_for_file_access(), - errmsg("could not remove directory \"%s\": %m", - linkloc_with_version_dir))); - pfree(linkloc_with_version_dir); - return false; + if (rmdir(linkloc_with_version_dir) < 0) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + linkloc_with_version_dir))); + pfree(linkloc_with_version_dir); + return false; + } + } + else + { + FileOpsRmdir(linkloc_with_version_dir, true /* at_commit */ ); } /* @@ -819,26 +873,53 @@ destroy_tablespace_directories(Oid tablespaceoid, bool redo) } else if (S_ISDIR(st.st_mode)) { - if (rmdir(linkloc) < 0) + /* + * Junction-style entry (Windows) or stale directory (any platform). + * In redo, remove now; outside redo, defer to commit so abort + * restores the entry. + */ + if (redo) { - int saved_errno = errno; + if (rmdir(linkloc) < 0) + { + int saved_errno = errno; - ereport(redo ? LOG : (saved_errno == ENOENT ? WARNING : ERROR), - (errcode_for_file_access(), - errmsg("could not remove directory \"%s\": %m", - linkloc))); + ereport(saved_errno == ENOENT ? LOG : LOG, + (errcode_for_file_access(), + errmsg("could not remove directory \"%s\": %m", + linkloc))); + } + } + else + { + FileOpsRmdir(linkloc, true /* at_commit */ ); } } else if (S_ISLNK(st.st_mode)) { - if (unlink(linkloc) < 0) + /* + * Tablespace symlink under PGDATA/pg_tblspc/. Symmetric with + * FileOpsSymlink on the create path: defer the unlink to commit so + * abort restores the symlink, closing the long-standing + * crash-asymmetry between CREATE TABLESPACE and DROP TABLESPACE. In + * redo, issue unlink directly because the redo handler is itself the + * recovery action. + */ + if (redo) { - int saved_errno = errno; + if (unlink(linkloc) < 0) + { + int saved_errno = errno; - ereport(redo ? LOG : (saved_errno == ENOENT ? WARNING : ERROR), - (errcode_for_file_access(), - errmsg("could not remove symbolic link \"%s\": %m", - linkloc))); + ereport(saved_errno == ENOENT ? LOG : LOG, + (errcode_for_file_access(), + errmsg("could not remove symbolic link \"%s\": %m", + linkloc))); + } + } + else + { + FileOpsDelete(linkloc, true /* at_commit */ ); } } else diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c index ee42c796f773a..4eb46d3498a41 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -24,11 +24,14 @@ #include #include +#include "access/xlog.h" +#include "common/file_perm.h" #include "common/file_utils.h" #include "miscadmin.h" #include "pgstat.h" #include "storage/copydir.h" #include "storage/fd.h" +#include "storage/fileops.h" #include "utils/wait_event.h" /* GUCs */ @@ -44,16 +47,24 @@ static void clone_file(const char *fromfile, const char *tofile); * * This function uses the file_copy_method GUC. New uses of this function must * be documented in doc/src/sgml/config.sgml. + * + * If register_for_abort_cleanup is true and we are not in WAL replay, the + * destination is created with FileOpsMkdirTree(), so that a transaction + * abort, or a crash before commit, removes the partial copy. Recovery + * callers pass false because they are themselves the recovery action. */ void -copydir(const char *fromdir, const char *todir, bool recurse) +copydir(const char *fromdir, const char *todir, bool recurse, + bool register_for_abort_cleanup) { DIR *xldir; struct dirent *xlde; char fromfile[MAXPGPATH * 2]; char tofile[MAXPGPATH * 2]; - if (MakePGDirectory(todir) != 0) + if (register_for_abort_cleanup && !RecoveryInProgress()) + FileOpsMkdirTree(todir, pg_dir_create_mode, InvalidOid); + else if (MakePGDirectory(todir) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not create directory \"%s\": %m", todir))); @@ -78,9 +89,13 @@ copydir(const char *fromdir, const char *todir, bool recurse) if (xlde_type == PGFILETYPE_DIR) { - /* recurse to handle subdirectories */ + /* + * Recurse to handle subdirectories. Pass false for + * register_for_abort_cleanup: removing the top-level destination + * tree removes them too. + */ if (recurse) - copydir(fromfile, tofile, true); + copydir(fromfile, tofile, true, false); } else if (xlde_type == PGFILETYPE_REG) { diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 40849c2ef9947..b979ee4258b7d 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -22,7 +22,19 @@ typedef enum FileCopyMethod /* GUC parameters */ extern PGDLLIMPORT int file_copy_method; -extern void copydir(const char *fromdir, const char *todir, bool recurse); +/* + * copydir(): copy a directory tree. + * + * If register_for_abort_cleanup is true, the destination is created with + * FileOpsMkdirTree(). Forward-path callers in transactional commands + * (CREATE DATABASE STRATEGY=FILE_COPY, ALTER DATABASE SET TABLESPACE) should + * pass true so that a transaction abort, or a crash before commit, removes + * the partial copy. Recovery callers (XLOG_DBASE_CREATE_FILE_COPY redo) must + * pass false because there is no surrounding transaction context and the + * redo handler is itself the recovery action. + */ +extern void copydir(const char *fromdir, const char *todir, bool recurse, + bool register_for_abort_cleanup); extern void copy_file(const char *fromfile, const char *tofile); #endif /* COPYDIR_H */ diff --git a/src/test/modules/test_fileops/meson.build b/src/test/modules/test_fileops/meson.build index 3f0087fe6e429..e8f3210277e3b 100644 --- a/src/test/modules/test_fileops/meson.build +++ b/src/test/modules/test_fileops/meson.build @@ -40,6 +40,7 @@ tests += { 't/005_fileops_checkpoint_loser.pl', 't/007_fileops_commit_crash.pl', 't/008_fileops_standby_abort.pl', + 't/010_fileops_abort_redo.pl', 't/018_fileops_rename.pl', 't/023_fileops_inline_abort_ctx.pl', 't/024_fileops_recovery_apply_failure.pl', diff --git a/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl index 0ff3e80890f5a..f40dcd271b05b 100644 --- a/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl +++ b/src/test/modules/test_fileops/t/007_fileops_commit_crash.pl @@ -73,5 +73,23 @@ sub commit_and_crash 'committed FileOpsRmdir happened despite the crash'); } +# DROP TABLESPACE is safe: its XLOG_TBLSPC_DROP precedes the commit record +# and its redo removes the directories. (In-place, so they are under +# pg_tblspc.) +$node->safe_psql('postgres', + 'SET allow_in_place_tablespaces = on; CREATE TABLESPACE ts LOCATION \'\'' +); +my $tsoid = $node->safe_psql('postgres', + "SELECT oid FROM pg_tablespace WHERE spcname = 'ts'"); +ok(-d "$datadir/pg_tblspc/$tsoid", 'tablespace directory exists'); +commit_and_crash('DROP TABLESPACE ts;'); +is( $node->safe_psql( + 'postgres', "SELECT count(*) FROM pg_tablespace WHERE spcname = 'ts'" + ), + '0', + 'DROP TABLESPACE committed'); +ok(!-e "$datadir/pg_tblspc/$tsoid", + 'committed DROP TABLESPACE removed its directory despite the crash'); + $node->stop; done_testing(); diff --git a/src/test/modules/test_fileops/t/010_fileops_abort_redo.pl b/src/test/modules/test_fileops/t/010_fileops_abort_redo.pl new file mode 100644 index 0000000000000..11a33c94c589b --- /dev/null +++ b/src/test/modules/test_fileops/t/010_fileops_abort_redo.pl @@ -0,0 +1,202 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('abort_redo'); +$node->init; +$node->append_conf('postgresql.conf', + "wal_level = replica\nautovacuum = off\ncheckpoint_timeout = '1h'\nmax_prepared_transactions = 5\n" +); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +my $dir = $node->data_dir; +$node->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_create('base.dat', 384); +SELECT test_fileops_write('base.dat', 0, 'committed-base'); +COMMIT; +CHECKPOINT; +BEGIN; +SELECT test_fileops_create('abort.dat', 384); +SELECT test_fileops_write('abort.dat', 0, 'aborted'); +ROLLBACK; +BEGIN; SELECT test_fileops_mkdir('abortdir', 448); ROLLBACK; +BEGIN; SELECT test_fileops_chmod('base.dat', 448); ROLLBACK; +BEGIN; SELECT test_fileops_write('base.dat', 0, 'OVERWRITTEN!!!'); ROLLBACK; +BEGIN; SELECT test_fileops_truncate('base.dat', 3); ROLLBACK; +BEGIN; +SAVEPOINT s; +SELECT test_fileops_create('sub.dat', 384); +ROLLBACK TO s; +SELECT test_fileops_write('base.dat', 0, 'parent'); +ROLLBACK; +BEGIN; SELECT test_fileops_symlink('base.dat', 'abort.sym'); ROLLBACK; +BEGIN; SELECT test_fileops_link('base.dat', 'abort.link'); ROLLBACK; + +}); +my $xattrs = $node->safe_psql('postgres', + "SELECT test_fileops_setxattr('base.dat', 'user.abort_test', 'original')") + eq 't'; +if ($xattrs) +{ + $node->safe_psql( + 'postgres', q{ +BEGIN; +SELECT test_fileops_setxattr('base.dat', 'user.abort_test', 'changed'); +ROLLBACK; +BEGIN; +SELECT test_fileops_removexattr('base.dat', 'user.abort_test'); +ROLLBACK; +}); +} +my ($uid, $gid) = (stat("$dir/base.dat"))[ 4, 5 ]; +# An unprivileged test can chown to its own identity; a different uid needs +# a privileged harness. This still exercises CHOWN inverse serialization. +$node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_chown('base.dat', $uid, $gid); ROLLBACK;"); +for my $restart (0 .. 2) +{ + if ($restart) + { + $node->stop('immediate'); + $node->start; + } + ok(!-e "$dir/sub.dat", "$restart: subabort then parent abort"); + ok(!-l "$dir/abort.sym", "$restart: symlink inverse"); + ok(!-e "$dir/abort.link", "$restart: link inverse"); + ok(!-e "$dir/abort.dat", "$restart: aborted create absent"); + ok(!-e "$dir/abortdir", "$restart: aborted mkdir absent"); + SKIP: + { + skip 'POSIX permission semantics required', 1 if $windows_os; + is((stat("$dir/base.dat"))[2] & 07777, + 0600, "$restart: original mode"); + } + is((stat("$dir/base.dat"))[4], + $uid, "$restart: owner after chown inverse"); + is((stat("$dir/base.dat"))[5], + $gid, "$restart: group after chown inverse"); + SKIP: + { + skip 'extended attributes not supported', 1 unless $xattrs; + is( $node->safe_psql( + 'postgres', + "SELECT test_fileops_getxattr('base.dat', 'user.abort_test')" + ), + 'original', + "$restart: xattr inverses restore value"); + } + is(-s "$dir/base.dat", 14, "$restart: original size"); + is(slurp_file("$dir/base.dat"), + 'committed-base', "$restart: original bytes"); +} + +# Stop with a durable abort decision but no inverses applied yet. A +# concurrently requested checkpoint must wait, not omit the abort from redo. +if ($ENV{enable_injection_points} eq 'yes') +{ + $node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + for my $checkpoint (0, 1, 2) + { + my $finish = + $checkpoint == 2 + ? "PREPARE TRANSACTION 'window'; ROLLBACK PREPARED 'window';" + : 'ROLLBACK;'; + $node->safe_psql('postgres', 'CHECKPOINT'); + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-abort-after-flush', 'wait')" + ); + my $bg = $node->background_psql('postgres', on_error_stop => 0); + $bg->query_until( + qr/start/, qq(\\echo start +BEGIN; +SELECT test_fileops_create('mid.dat', 384); +SELECT test_fileops_write('mid.dat', 0, 'uncommitted'); +SELECT test_fileops_truncate('base.dat', 3); +$finish +)); + $node->wait_for_event('client backend', 'fileops-abort-after-flush'); + my $ckpt; + if ($checkpoint == 1) + { + $ckpt = $node->background_psql('postgres', on_error_stop => 0); + $ckpt->query_until(qr/checkpoint/, + "\\echo checkpoint\nCHECKPOINT;\n"); + $node->wait_for_event('checkpointer', 'CheckpointDelayStart'); + } + $node->stop('immediate'); + $bg->{run}->finish; + $ckpt->{run}->finish if $ckpt; + $node->start; + ok(!-e "$dir/mid.dat", + "$checkpoint: interrupted abort removed create"); + is(slurp_file("$dir/base.dat"), + 'committed-base', + "$checkpoint: interrupted abort restored shrink bytes"); + $node->stop('immediate'); + $node->start; + ok(!-e "$dir/mid.dat", + "$checkpoint: repeated recovery keeps create absent"); + } +} + +if ($ENV{enable_injection_points} eq 'yes') +{ + $node->safe_psql('postgres', 'CHECKPOINT'); + $node->safe_psql( + 'postgres', q{ +SELECT injection_points_attach('createdb-wal-log-after-copy-relation', 'error'); +SELECT injection_points_attach('fileops-abort-after-flush', 'wait'); +}); + my $db = $node->background_psql('postgres', on_error_stop => 0); + $db->query_until(qr/start/, + "\\echo start\nCREATE DATABASE aborted_tree STRATEGY WAL_LOG OID 90000;\n" + ); + $node->wait_for_event('client backend', 'fileops-abort-after-flush'); + # createdb's error cleanup removes the online directory already. Redo + # recreates it from forward WAL, so the embedded tree inverse must remove + # it again even though the original inode no longer exists. + ok(!-e "$dir/base/90000", 'createdb error cleanup removed online tree'); + $node->stop('immediate'); + $db->{run}->finish; + $node->start; + ok(!-e "$dir/base/90000", 'abort redo removes database tree'); +} + +# A WARNING is not a durable retry obligation: fixing permissions does not +# retry this inverse online. A crash before the next checkpoint replays it. +$node->safe_psql('postgres', 'CHECKPOINT'); +my $failed = $node->background_psql('postgres'); +$failed->query_safe('BEGIN'); +$failed->query_safe("SELECT test_fileops_truncate('base.dat', 3)"); +chmod(0400, "$dir/base.dat") or die "chmod: $!"; +my $log_offset = -s $node->logfile; +$failed->query('ROLLBACK'); +$failed->quit; +like( + slurp_file($node->logfile, $log_offset), + qr/WARNING:.*FILEOPS UNDO TRUNCATE: could not open/, + 'failed inverse reports a warning'); +chmod(0600, "$dir/base.dat") or die "chmod: $!"; +is(slurp_file("$dir/base.dat"), + 'com', 'failed inverse remains unapplied online'); +$node->stop('immediate'); +$node->start; +is(slurp_file("$dir/base.dat"), + 'committed-base', 'abort redo retries failed inverse after crash'); + +# Refuse excessive before-images before touching the file. +my ($ret, $out, $err) = $node->psql('postgres', + "SELECT test_fileops_write('base.dat', 0, repeat('x', 1048577))"); +like( + $err, + qr/ERROR:.*excessive transactional file write range/, + 'oversized write is an ordinary error'); +is(slurp_file("$dir/base.dat"), + 'committed-base', 'refused write changes nothing'); +$node->stop; +done_testing(); diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index c8aef86b59682..0c8444e39e329 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -75,6 +75,7 @@ tests += { 't/064_fileops_2pc.pl', 't/065_fileops_standby_divergence.pl', 't/066_fileops_crash_undo.pl', + 't/067_fileops_createdb_crash.pl', ], }, } diff --git a/src/test/recovery/t/067_fileops_createdb_crash.pl b/src/test/recovery/t/067_fileops_createdb_crash.pl new file mode 100644 index 0000000000000..108a932fc0288 --- /dev/null +++ b/src/test/recovery/t/067_fileops_createdb_crash.pl @@ -0,0 +1,113 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +# A CREATE DATABASE that crashes, or fails, part way through its copy must +# leave no database directory behind, for either strategy. The directory +# is created by FILEOPS but filled by smgr (WAL_LOG) or copydir (FILE_COPY), +# so its UNDO must remove the whole tree, not just rmdir it. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('createdb_crash'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +shared_preload_libraries = 'injection_points' +wal_level = replica +autovacuum = off +}); +$node->start; +if (!$node->check_extension('injection_points')) +{ + plan skip_all => 'Extension injection_points not installed'; +} +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); +$node->safe_psql('postgres', 'CREATE DATABASE tmpl'); +$node->safe_psql('tmpl', + "CREATE TABLE t AS SELECT g, repeat('x', 200) v FROM generate_series(1, 20000) g" +); + +my $base = $node->data_dir . '/base'; + +# Directories under base/ that belong to no database. +sub orphans +{ + my %known = map { $_ => 1 } split /\n/, + $node->safe_psql('postgres', 'SELECT oid FROM pg_database'); + opendir(my $dh, $base) or die "opendir $base: $!"; + my @o = grep { /^\d+$/ && !$known{$_} } readdir($dh); + closedir($dh); + return join(' ', sort @o); +} + +my %point = ( + wal_log => 'createdb-wal-log-after-copy-relation', + file_copy => 'createdb-file-copy-after-copydir'); + +foreach my $strategy ('wal_log', 'file_copy') +{ + my $ip = $point{$strategy}; + my %before = map { $_ => 1 } split / /, orphans(); + + # Crash while the new directory holds copied files. + $node->safe_psql('postgres', + "SELECT injection_points_attach('$ip', 'wait')"); + my $bg = $node->background_psql('postgres', on_error_stop => 0); + $bg->query_until( + qr/start/, qq(\\echo start +CREATE DATABASE c_$strategy TEMPLATE tmpl STRATEGY $strategy; +)); + $node->wait_for_event('client backend', $ip); + + my ($new) = grep { !$before{$_} } split / /, orphans(); + ok(defined $new, "$strategy: the new database directory exists"); + $new //= 'none'; + my @files = glob("$base/$new/*"); + cmp_ok(scalar(@files), '>', 1, + "$strategy: it holds copied files at the crash"); + + my $log_offset = -s $node->logfile; + $node->stop('immediate'); + $bg->{run}->finish; + $node->start; + + is(orphans(), '', "$strategy: no database directory left after a crash"); + my $log = slurp_file($node->logfile, $log_offset); + unlike( + $log, + qr/WARNING: FILEOPS UNDO/, + "$strategy: crash recovery applied the UNDO cleanly"); + + # The same, with the copy failing instead. + $node->safe_psql('postgres', + "SELECT injection_points_attach('$ip', 'error')"); + my ($ret, $out, $err) = $node->psql('postgres', + "CREATE DATABASE e_$strategy TEMPLATE tmpl STRATEGY $strategy"); + like($err, qr/ERROR/, "$strategy: CREATE DATABASE failed mid-copy"); + $node->safe_psql('postgres', "SELECT injection_points_detach('$ip')"); + is(orphans(), '', "$strategy: no database directory left after an error"); + + # A later CREATE DATABASE still works. + $node->safe_psql('postgres', + "CREATE DATABASE ok_$strategy TEMPLATE tmpl STRATEGY $strategy"); + is($node->safe_psql("ok_$strategy", 'SELECT count(*) FROM t'), + '20000', "$strategy: a new CREATE DATABASE succeeds"); +} + +# No failed rmdir of a non-empty directory anywhere in the log. +sleep(1); +unlike( + slurp_file($node->logfile), + qr/could not rmdir .*Directory not empty/, + 'no failed rmdir of a non-empty directory'); + +$node->stop; +done_testing();