jpargudo · Oct 30, 2026 · Oct 30, 2026 · Nov 5, 2026 · Nov 5, 2026 · Nov 5, 2026
@@ -1,8 +1,14 @@
 #! /usr/bin/env bash
 
-# Common functions and variables used by initiate.sh and complete.sh
+# Common functions and variables used by
+# prepare.sh, initiate.sh and complete.sh
 
+#staging/prod
 REPORTING_PROJECT_REF="ihmaxnjpcccasmrbkpvo"
+
+#localdev
+#REPORTING_PROJECT_REF="ditigwgjchkdxjxijzza"
+
 REPORTING_CREDENTIALS_FILE="/root/upgrade-reporting-credentials"
 
 REPORTING_ANON_KEY=""

@@ -133,12 +133,14 @@ cleanup() {
 	# can fail under set -e, or the restart below would be refused.
 	unblock_postgres_start || log "WARNING: failed to lift the postgres start block; check /run/systemd/system/postgresql.service.d"
 
+	# ship $LOG_FILE content to reporting project (see common.sh)
+	ship_logs "$LOG_FILE" || true
+
 	if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then
 		log "Copying pg_upgrade output to /var/log"
 		cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/ || true
 		chown -R postgres:postgres /var/log/pg_upgrade_output.d/
 		chmod -R 0750 /var/log/pg_upgrade_output.d/
-		ship_logs "$LOG_FILE" || true
 		tail -n +1 /var/log/pg_upgrade_output.d/*/* >/var/log/pg_upgrade_output.d/pg_upgrade.log || true
 		ship_logs "/var/log/pg_upgrade_output.d/pg_upgrade.log" || true
 	fi
@@ -199,7 +201,7 @@ EOF
 }
 
 # EXIT trap: any exit that did not go through cleanup() (set -u abort, TERM/INT)
-# still restores the source, since step 10 left it stopped and disabled. If
+# still restores the source, since step 9 left it stopped and disabled. If
 # cleanup() already ran (or died part way under set -e), only make sure the
 # start block is gone.
 on_exit() {
@@ -217,6 +219,7 @@ on_exit() {
 		fi
 	fi
 	unblock_postgres_start || log "WARNING: failed to lift the postgres start block; check /run/systemd/system/postgresql.service.d"
+
 	exit "$exit_code"
 }
 
@@ -586,21 +589,6 @@ EXTRA_NIX_CONF
 		export LD_LIBRARY_PATH="${PGLIBNEW}"
 	fi
 
-	log "9. Creating new data directory, initializing database"
-	chown -R postgres:postgres "$MOUNT_POINT/"
-	rm -rf "${PGDATANEW:?}/"
-
-	if [ "$IS_NIX_UPGRADE" = "true" ]; then
-		if [[ ${PGVERSION%%.*} -ge 16 ]]; then
-			LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
-		else
-			LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
-		fi
-	else
-		su -c "$PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
-
-	fi
-
 	# This line avoids the need to supply the supabase_admin password on the old
 	# instance, since pg_upgrade connects to the db as supabase_admin using unix
 	# sockets, which is gated behind scram-sha-256 per pg_hba.conf.j2. The new
@@ -650,13 +638,7 @@ EOF
 		UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $UPGRADE_COMMAND"
 	fi
 
-	if [[ ${PGVERSION%%.*} -ge 16 ]]; then
-		GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres
-	else
-		GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres
-	fi
-
-	log "10. Stopping postgres; running pg_upgrade"
+	log "9. Stopping SOURCE postgres and ensuring it says stopped"
 	# Extra work to ensure postgres is actually stopped
 	#  Mostly needed for PG12 projects with odd systemd unit behavior
 	if [ -z "$IS_CI" ]; then
@@ -672,9 +654,86 @@ EOF
 		CI_stop_postgres
 	fi
 
+	#CHECKSUM_FLAG="--data-checksums" if the SOURCE cluster has checksums
+	#CHECKSUM_FLAG=""                 if the SOURCE doesn't have them (yet)
+	#                                 in this case the following code adds them
+
+	if [[ $CHECKSUM_FLAG == "" ]]; then
+
+		# old db hasn't got checksum, we will enable it its overall size
+		# (${PGDATAOLD}) is <= 20 Gb
+
+		# 20 gb = 20 x 1024 x 1024 = 20971520
+		MAX_SIZE=$((20 * 1024 * 1024))
+
+		PGDATAOLD_SIZE=$(du -sk "${PGDATAOLD}" | cut -f1)
+
+		if [[ $PGDATAOLD_SIZE -le $MAX_SIZE ]]; then
+
+			log "9.1 adding checksums to SOURCE PG (<20 Gb)"
+
+			# pg_checksums -e does not verify existing pages. A run over MAX_DURATION fails the upgrade.
+
+			MAX_DURATION="10m"
+			KILL_AFTER="15s"
+			STATUS=0
+			timeout -k "${KILL_AFTER}" "${MAX_DURATION}" "${PGBINOLD}"/pg_checksums -e "${PGDATAOLD}" || STATUS=$?
+
+			case $STATUS in
+			0)
+				log "adding checksums SUCCEEDED."
+				;;
+			124 | 137)
+				log "adding checksums FAILED. Max time (${MAX_DURATION}) reached! Process killed."
+				exit 124
+				;;
+			*)
+				log "adding checksum FAILED with Error code ${STATUS}"
+				exit "$STATUS"
+				;;
+			esac
+
+			# we're here if everything went well with STATUS=0
+			# so now, checksums are enabled on the SOURCE!
+			CHECKSUM_FLAG="--data-checksums"
+
+		else
+
+			log "9.1 did *not* add checksum to SOURCE PG: size $PGDATAOLD_SIZE > 20 Gb!"
+
+		fi
+
+	else
+
+		log "9.1 checksums already present on SOURCE"
+
+	fi
+
+	log "10. Creating new data directory, initializing database"
+	chown -R postgres:postgres "$MOUNT_POINT/"
+	rm -rf "${PGDATANEW:?}/"
+
+	if [ "$IS_NIX_UPGRADE" = "true" ]; then
+		if [[ ${PGVERSION%%.*} -ge 16 ]]; then
+			LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
+		else
+			LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
+		fi
+	else
+		su -c "$PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
+	fi
+
+	log "11. pg_upgrade --check(ing) compatibility of old/new clusters"
+	if [[ ${PGVERSION%%.*} -ge 16 ]]; then
+		GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres
+	else
+		GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres
+	fi
+
 	start_pid_lock_watcher
 
-	# Start the old PostgreSQL instance with version-specific options
+	# Start the pg_upgrade from PGDATAOLD to PGDATANEW..
+	log "12. pg_upgrade PGDATAOLD to PGDATANEW"
 	if [[ ${PGVERSION%%.*} -ge 16 ]]; then
 		GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres
 	else
@@ -686,7 +745,7 @@ EOF
 	fi
 
 	# copying custom configurations
-	log "11. Copying custom configurations"
+	log "13. Copying custom configurations"
 	mkdir -p "$MOUNT_POINT/conf"
 	cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/"
 	# removing supautils config as to allow the latest one provided by the latest image to be used
@@ -697,12 +756,12 @@ EOF
 	rm -f "$MOUNT_POINT/conf/wal-g.conf"
 
 	# copy sql files generated by pg_upgrade
-	log "12. Copying sql files generated by pg_upgrade"
+	log "14. Copying sql files generated by pg_upgrade"
 	mkdir -p "$MOUNT_POINT/sql"
 	cp /tmp/pg_upgrade/*.sql "$MOUNT_POINT/sql/" || true
 	chown -R postgres:postgres "$MOUNT_POINT/sql/"
 
-	log "13. Cleaning up"
+	log "15. Cleaning up"
 	cleanup "complete"
 }
 

@@ -1,15 +1,100 @@
 #! /usr/bin/env bash
-## This script is runs in advance of the database version upgrade, on the newly
-## launched instance which will eventually be promoted to become the primary
-## database instance once the upgrade successfully completes, terminating the
-## previous (source) instance.
-## The following commands safely stop the Postgres service and unmount
-## the data disk off the newly launched instance, to be re-attached to the
-## source instance and run the upgrade there.
 
-set -euo pipefail
+# This script runs in advance of the database version upgrade, on the newly
+# launched instance which will eventually be promoted to become the primary
+# database instance once the upgrade successfully completes, terminating the
+# previous (source) instance.  The following commands safely stop the Postgres
+# service and unmount the data disk off the newly launched instance, to be
+# re-attached to the source instance and run the upgrade there.
 
+set -Eeuo pipefail
+
+SCRIPT_DIR=$(dirname -- "$0")
+
+# Each attempt overwrites the log. Failed attempts are shipped before the retry.
+LOG_FILE="/var/log/pg-upgrade-prepare.log"
+touch "$LOG_FILE"
+chmod 640 "$LOG_FILE"
+
+# From here on, everything written to stdout/stderr (log messages, but also
+# errors from systemctl, install, sync, etc.) is written to the log file.
+exec >"$LOG_FILE" 2>&1
+
+# Minimal fallback so that "log" exists even if common.sh cannot be sourced.
+# common.sh redefines it with the real implementation.
+log() { echo "$(date -u '+%Y-%m-%d %H:%M:%S UTC') $*"; }
+
+# Log any unexpected failure (set -e) with the line number and failing command,
+# so that no error can end the script silently.
+trap 'log "prepare.sh FAILED at line $LINENO: \"$BASH_COMMAND\" (exit code $?)"' ERR
+
+# Source the shared helpers (log and ship_logs functions)
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/common.sh" || {
+	log "cannot source $SCRIPT_DIR/common.sh"
+	exit 1
+}
+
+on_exit() {
+	local rc=$?
+	trap - EXIT ERR
+
+	[ "$rc" -eq 0 ] && log "prepare.sh ended with no errors."
+	ship_logs "$LOG_FILE" || log "Log upload failed (non-blocking)."
+	exit "$rc"
+}
+trap on_exit EXIT
+
+log "prepare.sh started"
+
+# check that /data is actually mounted
+mountpoint -q /data || {
+	log "/data is not mounted"
+	exit 1
+}
+
+# stop postgresql: "systemctl stop blocks until the unit has fully stopped"
+# so there is no need to check pg_isready afterwards...
 systemctl stop postgresql
 
-cp /etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key
-umount /data
+# ...but let's still verify that the service is explicitly stopped
+if systemctl is-active --quiet postgresql; then
+	log "postgresql service is still active!"
+	exit 1
+fi
+
+# ...and that there are no remaining postgres processes
+if pgrep -x postgres >/dev/null; then
+	log "postgres processes are still alive!"
+	exit 1
+fi
+
+# copy the pgsodium root key with proper owner and permissions
+install -m 600 -o postgres -g postgres \
+	/etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key
+
+# flush cached writes to persistent storage
+sync
+
+# Unmount /data. Capture the native umount error message and route it through
+# "log" so that it is timestamped like the rest of the log entries.
+if ! umount_out=$(umount /data 2>&1); then
+	log "Unable to unmount /data: $umount_out"
+	# List the processes whose cwd is under /data.
+	log "Processes using /data:"
+
+	# activate nullglob to avoid errors if /proc is not accessible
+	shopt -s nullglob
+
+	QUOTING_STYLE=literal stat -c '%U %N' /proc/[0-9]*/cwd 2>/dev/null |
+		sed -nE 's|^(\S+) /proc/([0-9]+)/cwd -> /data(/.*)?$|\1 \2|p' |
+		while read -r user pid; do
+			comm=$(cat "/proc/$pid/comm" 2>/dev/null) || continue
+			log "user: ${user} pid: ${pid} command: ${comm}"
+		done || true
+
+	exit 1
+fi
+
+# Reaching this point means success: the final message and the log upload
+# are handled by the EXIT trap (on_exit).
@@ -4,10 +4,15 @@
 # file copy. Sourced by initiate.sh and tests/pg_upgrade/test-start-guard.sh.
 # Expects log, IS_CI, PGDATAOLD and PGDATANEW from the caller.
 
-# From step 10 until cleanup, nothing may start the source postgres: pg_upgrade
-# copies the data files without a postmaster holding the directory, and a start
-# in that window (Salt's `service.running`, adminapi, Restart=always, ...)
-# yields a copy that is inconsistent with the dump. Two independent guards:
+# From step 9 until cleanup, nothing may start the source postgres:
+#
+# If it has to run, "pg_checksums -e" will operate on the source postgres, and
+# this one has to stay stopped during the whole operation
+#
+# pg_upgrade copies the data files without a postmaster holding the directory,
+# and a start in that window (Salt's `service.running`, adminapi,
+# Restart=always, ...) yields a copy that is inconsistent with the dump. Two
+# independent guards:
 #  1. a runtime systemd drop-in that makes `systemctl start` fail while this
 #     script or pg_upgrade is alive (a SIGKILLed script must not leave the
 #     source blocked until a reboot);