diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index f82cf5b63b..1cbe9f29df 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -1,8 +1,14 @@ #! /usr/bin/env bash -# Common functions and variables used by initiate.sh and complete.sh +# Common functions and variables used by +# prepare.sh, initiate.sh and complete.sh +#staging/prod REPORTING_PROJECT_REF="ihmaxnjpcccasmrbkpvo" + +#localdev +#REPORTING_PROJECT_REF="ditigwgjchkdxjxijzza" + REPORTING_CREDENTIALS_FILE="/root/upgrade-reporting-credentials" REPORTING_ANON_KEY="" diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 69cf423475..3b15259c74 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -133,12 +133,14 @@ cleanup() { # can fail under set -e, or the restart below would be refused. unblock_postgres_start || log "WARNING: failed to lift the postgres start block; check /run/systemd/system/postgresql.service.d" + # ship $LOG_FILE content to reporting project (see common.sh) + ship_logs "$LOG_FILE" || true + if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then log "Copying pg_upgrade output to /var/log" cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/ || true chown -R postgres:postgres /var/log/pg_upgrade_output.d/ chmod -R 0750 /var/log/pg_upgrade_output.d/ - ship_logs "$LOG_FILE" || true tail -n +1 /var/log/pg_upgrade_output.d/*/* >/var/log/pg_upgrade_output.d/pg_upgrade.log || true ship_logs "/var/log/pg_upgrade_output.d/pg_upgrade.log" || true fi @@ -199,7 +201,7 @@ EOF } # EXIT trap: any exit that did not go through cleanup() (set -u abort, TERM/INT) -# still restores the source, since step 10 left it stopped and disabled. If +# still restores the source, since step 9 left it stopped and disabled. If # cleanup() already ran (or died part way under set -e), only make sure the # start block is gone. on_exit() { @@ -217,6 +219,7 @@ on_exit() { fi fi unblock_postgres_start || log "WARNING: failed to lift the postgres start block; check /run/systemd/system/postgresql.service.d" + exit "$exit_code" } @@ -586,21 +589,6 @@ EXTRA_NIX_CONF export LD_LIBRARY_PATH="${PGLIBNEW}" fi - log "9. Creating new data directory, initializing database" - chown -R postgres:postgres "$MOUNT_POINT/" - rm -rf "${PGDATANEW:?}/" - - if [ "$IS_NIX_UPGRADE" = "true" ]; then - if [[ ${PGVERSION%%.*} -ge 16 ]]; then - LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres - else - LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres - fi - else - su -c "$PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres - - fi - # This line avoids the need to supply the supabase_admin password on the old # instance, since pg_upgrade connects to the db as supabase_admin using unix # sockets, which is gated behind scram-sha-256 per pg_hba.conf.j2. The new @@ -650,13 +638,7 @@ EOF UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $UPGRADE_COMMAND" fi - if [[ ${PGVERSION%%.*} -ge 16 ]]; then - GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres - else - GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres - fi - - log "10. Stopping postgres; running pg_upgrade" + log "9. Stopping SOURCE postgres and ensuring it says stopped" # Extra work to ensure postgres is actually stopped # Mostly needed for PG12 projects with odd systemd unit behavior if [ -z "$IS_CI" ]; then @@ -672,9 +654,86 @@ EOF CI_stop_postgres fi + #CHECKSUM_FLAG="--data-checksums" if the SOURCE cluster has checksums + #CHECKSUM_FLAG="" if the SOURCE doesn't have them (yet) + # in this case the following code adds them + + if [[ $CHECKSUM_FLAG == "" ]]; then + + # old db hasn't got checksum, we will enable it its overall size + # (${PGDATAOLD}) is <= 20 Gb + + # 20 gb = 20 x 1024 x 1024 = 20971520 + MAX_SIZE=$((20 * 1024 * 1024)) + + PGDATAOLD_SIZE=$(du -sk "${PGDATAOLD}" | cut -f1) + + if [[ $PGDATAOLD_SIZE -le $MAX_SIZE ]]; then + + log "9.1 adding checksums to SOURCE PG (<20 Gb)" + + # pg_checksums -e does not verify existing pages. A run over MAX_DURATION fails the upgrade. + + MAX_DURATION="10m" + KILL_AFTER="15s" + STATUS=0 + timeout -k "${KILL_AFTER}" "${MAX_DURATION}" "${PGBINOLD}"/pg_checksums -e "${PGDATAOLD}" || STATUS=$? + + case $STATUS in + 0) + log "adding checksums SUCCEEDED." + ;; + 124 | 137) + log "adding checksums FAILED. Max time (${MAX_DURATION}) reached! Process killed." + exit 124 + ;; + *) + log "adding checksum FAILED with Error code ${STATUS}" + exit "$STATUS" + ;; + esac + + # we're here if everything went well with STATUS=0 + # so now, checksums are enabled on the SOURCE! + CHECKSUM_FLAG="--data-checksums" + + else + + log "9.1 did *not* add checksum to SOURCE PG: size $PGDATAOLD_SIZE > 20 Gb!" + + fi + + else + + log "9.1 checksums already present on SOURCE" + + fi + + log "10. Creating new data directory, initializing database" + chown -R postgres:postgres "$MOUNT_POINT/" + rm -rf "${PGDATANEW:?}/" + + if [ "$IS_NIX_UPGRADE" = "true" ]; then + if [[ ${PGVERSION%%.*} -ge 16 ]]; then + LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + else + LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + fi + else + su -c "$PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + fi + + log "11. pg_upgrade --check(ing) compatibility of old/new clusters" + if [[ ${PGVERSION%%.*} -ge 16 ]]; then + GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres + else + GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres + fi + start_pid_lock_watcher - # Start the old PostgreSQL instance with version-specific options + # Start the pg_upgrade from PGDATAOLD to PGDATANEW.. + log "12. pg_upgrade PGDATAOLD to PGDATANEW" if [[ ${PGVERSION%%.*} -ge 16 ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres else @@ -686,7 +745,7 @@ EOF fi # copying custom configurations - log "11. Copying custom configurations" + log "13. Copying custom configurations" mkdir -p "$MOUNT_POINT/conf" cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/" # removing supautils config as to allow the latest one provided by the latest image to be used @@ -697,12 +756,12 @@ EOF rm -f "$MOUNT_POINT/conf/wal-g.conf" # copy sql files generated by pg_upgrade - log "12. Copying sql files generated by pg_upgrade" + log "14. Copying sql files generated by pg_upgrade" mkdir -p "$MOUNT_POINT/sql" cp /tmp/pg_upgrade/*.sql "$MOUNT_POINT/sql/" || true chown -R postgres:postgres "$MOUNT_POINT/sql/" - log "13. Cleaning up" + log "15. Cleaning up" cleanup "complete" } diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh index 7d7eb98903..87f16c2a9d 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh @@ -1,15 +1,100 @@ #! /usr/bin/env bash -## This script is runs in advance of the database version upgrade, on the newly -## launched instance which will eventually be promoted to become the primary -## database instance once the upgrade successfully completes, terminating the -## previous (source) instance. -## The following commands safely stop the Postgres service and unmount -## the data disk off the newly launched instance, to be re-attached to the -## source instance and run the upgrade there. -set -euo pipefail +# This script runs in advance of the database version upgrade, on the newly +# launched instance which will eventually be promoted to become the primary +# database instance once the upgrade successfully completes, terminating the +# previous (source) instance. The following commands safely stop the Postgres +# service and unmount the data disk off the newly launched instance, to be +# re-attached to the source instance and run the upgrade there. +set -Eeuo pipefail + +SCRIPT_DIR=$(dirname -- "$0") + +# Each attempt overwrites the log. Failed attempts are shipped before the retry. +LOG_FILE="/var/log/pg-upgrade-prepare.log" +touch "$LOG_FILE" +chmod 640 "$LOG_FILE" + +# From here on, everything written to stdout/stderr (log messages, but also +# errors from systemctl, install, sync, etc.) is written to the log file. +exec >"$LOG_FILE" 2>&1 + +# Minimal fallback so that "log" exists even if common.sh cannot be sourced. +# common.sh redefines it with the real implementation. +log() { echo "$(date -u '+%Y-%m-%d %H:%M:%S UTC') $*"; } + +# Log any unexpected failure (set -e) with the line number and failing command, +# so that no error can end the script silently. +trap 'log "prepare.sh FAILED at line $LINENO: \"$BASH_COMMAND\" (exit code $?)"' ERR + +# Source the shared helpers (log and ship_logs functions) +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/common.sh" || { + log "cannot source $SCRIPT_DIR/common.sh" + exit 1 +} + +on_exit() { + local rc=$? + trap - EXIT ERR + + [ "$rc" -eq 0 ] && log "prepare.sh ended with no errors." + ship_logs "$LOG_FILE" || log "Log upload failed (non-blocking)." + exit "$rc" +} +trap on_exit EXIT + +log "prepare.sh started" + +# check that /data is actually mounted +mountpoint -q /data || { + log "/data is not mounted" + exit 1 +} + +# stop postgresql: "systemctl stop blocks until the unit has fully stopped" +# so there is no need to check pg_isready afterwards... systemctl stop postgresql -cp /etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key -umount /data +# ...but let's still verify that the service is explicitly stopped +if systemctl is-active --quiet postgresql; then + log "postgresql service is still active!" + exit 1 +fi + +# ...and that there are no remaining postgres processes +if pgrep -x postgres >/dev/null; then + log "postgres processes are still alive!" + exit 1 +fi + +# copy the pgsodium root key with proper owner and permissions +install -m 600 -o postgres -g postgres \ + /etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key + +# flush cached writes to persistent storage +sync + +# Unmount /data. Capture the native umount error message and route it through +# "log" so that it is timestamped like the rest of the log entries. +if ! umount_out=$(umount /data 2>&1); then + log "Unable to unmount /data: $umount_out" + # List the processes whose cwd is under /data. + log "Processes using /data:" + + # activate nullglob to avoid errors if /proc is not accessible + shopt -s nullglob + + QUOTING_STYLE=literal stat -c '%U %N' /proc/[0-9]*/cwd 2>/dev/null | + sed -nE 's|^(\S+) /proc/([0-9]+)/cwd -> /data(/.*)?$|\1 \2|p' | + while read -r user pid; do + comm=$(cat "/proc/$pid/comm" 2>/dev/null) || continue + log "user: ${user} pid: ${pid} command: ${comm}" + done || true + + exit 1 +fi + +# Reaching this point means success: the final message and the log upload +# are handled by the EXIT trap (on_exit). diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh index 231a06040b..1537a5c736 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh @@ -4,10 +4,15 @@ # file copy. Sourced by initiate.sh and tests/pg_upgrade/test-start-guard.sh. # Expects log, IS_CI, PGDATAOLD and PGDATANEW from the caller. -# From step 10 until cleanup, nothing may start the source postgres: pg_upgrade -# copies the data files without a postmaster holding the directory, and a start -# in that window (Salt's `service.running`, adminapi, Restart=always, ...) -# yields a copy that is inconsistent with the dump. Two independent guards: +# From step 9 until cleanup, nothing may start the source postgres: +# +# If it has to run, "pg_checksums -e" will operate on the source postgres, and +# this one has to stay stopped during the whole operation +# +# pg_upgrade copies the data files without a postmaster holding the directory, +# and a start in that window (Salt's `service.running`, adminapi, +# Restart=always, ...) yields a copy that is inconsistent with the dump. Two +# independent guards: # 1. a runtime systemd drop-in that makes `systemctl start` fail while this # script or pg_upgrade is alive (a SIGKILLed script must not leave the # source blocked until a reboot);