jpa/MPG-1254-1283-1531 : initiate.sh rewritten + add checksums + ship logs no matter what

ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh

@@ -1,8 +1,14 @@
 #! /usr/bin/env bash
 
-# Common functions and variables used by initiate.sh and complete.sh
+# Common functions and variables used by
+# prepare.sh, initiate.sh and complete.sh
 
+#staging/prod
 REPORTING_PROJECT_REF="ihmaxnjpcccasmrbkpvo"
+
+#localdev
+#REPORTING_PROJECT_REF="ditigwgjchkdxjxijzza"
+
 REPORTING_CREDENTIALS_FILE="/root/upgrade-reporting-credentials"
 
 REPORTING_ANON_KEY=""

ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh

@@ -133,12 +133,14 @@ cleanup() {
 	# can fail under set -e, or the restart below would be refused.
 	unblock_postgres_start || log "WARNING: failed to lift the postgres start block; check /run/systemd/system/postgresql.service.d"
 
+	# ship $LOG_FILE content to reporting project (see common.sh)
+	ship_logs "$LOG_FILE" || true
+
 	if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then
 		log "Copying pg_upgrade output to /var/log"
 		cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/ || true
 		chown -R postgres:postgres /var/log/pg_upgrade_output.d/
 		chmod -R 0750 /var/log/pg_upgrade_output.d/
-		ship_logs "$LOG_FILE" || true
 		tail -n +1 /var/log/pg_upgrade_output.d/*/* >/var/log/pg_upgrade_output.d/pg_upgrade.log || true
 		ship_logs "/var/log/pg_upgrade_output.d/pg_upgrade.log" || true
 	fi
@@ -199,7 +201,7 @@ EOF
 }
 
 # EXIT trap: any exit that did not go through cleanup() (set -u abort, TERM/INT)
-# still restores the source, since step 10 left it stopped and disabled. If
+# still restores the source, since step 9 left it stopped and disabled. If
 # cleanup() already ran (or died part way under set -e), only make sure the
 # start block is gone.
 on_exit() {
@@ -217,6 +219,7 @@ on_exit() {
 		fi
 	fi
 	unblock_postgres_start || log "WARNING: failed to lift the postgres start block; check /run/systemd/system/postgresql.service.d"
+
 	exit "$exit_code"
 }
 
@@ -586,21 +589,6 @@ EXTRA_NIX_CONF
 		export LD_LIBRARY_PATH="${PGLIBNEW}"
 	fi
 
-	log "9. Creating new data directory, initializing database"
-	chown -R postgres:postgres "$MOUNT_POINT/"
-	rm -rf "${PGDATANEW:?}/"
-
-	if [ "$IS_NIX_UPGRADE" = "true" ]; then
-		if [[ ${PGVERSION%%.*} -ge 16 ]]; then
-			LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
-		else
-			LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
-		fi
-	else
-		su -c "$PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
-
-	fi
-
 	# This line avoids the need to supply the supabase_admin password on the old
 	# instance, since pg_upgrade connects to the db as supabase_admin using unix
 	# sockets, which is gated behind scram-sha-256 per pg_hba.conf.j2. The new
@@ -650,13 +638,7 @@ EOF
 		UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $UPGRADE_COMMAND"
 	fi
 
-	if [[ ${PGVERSION%%.*} -ge 16 ]]; then
-		GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres
-	else
-		GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres
-	fi
-
-	log "10. Stopping postgres; running pg_upgrade"
+	log "9. Stopping SOURCE postgres and ensuring it says stopped"
 	# Extra work to ensure postgres is actually stopped
 	#  Mostly needed for PG12 projects with odd systemd unit behavior
 	if [ -z "$IS_CI" ]; then
@@ -672,9 +654,94 @@ EOF
 		CI_stop_postgres
 	fi
 
+	#CHECKSUM_FLAG="--data-checksums" if the SOURCE cluster has checksums
+	#CHECKSUM_FLAG=""                 if the SOURCE doesn't have them (yet)
+	#                                 in this case the following code adds them
+
+	if [[ $CHECKSUM_FLAG == "" ]]; then
+
+		# old db hasn't got checksum, we will enable it its overall size
+		# (${PGDATAOLD}) is <= 20 Gb
+
+		# 20 gb = 20 x 1024 x 1024 = 20971520
+		MAX_SIZE=20971520
+
+		PGDATAOLD_SIZE=$(du -sk "${PGDATAOLD}" | cut -f1)
+
+		if [[ $PGDATAOLD_SIZE -le $MAX_SIZE ]]; then
+
+			log "9.1 adding checksums to SOURCE PG (<20 Gb)"
+
+			# pg_checksums -e does not verify existing pages. A run over MAX_DURATION fails the upgrade.
+
+			MAX_DURATION="10m"
+			KILL_AFTER="15s"
+
+			timeout -k "${KILL_AFTER}" "${MAX_DURATION}" "${PGBINOLD}"/pg_checksums -e "${PGDATAOLD}" || STATUS=$?
+
+			# if everything is fine, $STATUS hasn't been defined,
+			# so we initialize it with 0 because it's a success
+			STATUS=${STATUS:-0}
+
+			# we'll manage ERROR codes down there, to actually log anything
+			# properly, for later diagnostic reading the $LOG_FILE into project
+			# where it has been shipped ship_logs() in common.sh
+
+			case $STATUS in
+			0)
+				log "adding checksums SUCCEEDED."
+				;;
+			124 | 137)
+				log "adding checksums FAILED. Max time (${MAX_DURATION}) reached! Process killed."
+				exit 124
+				;;
+			*)
+				log "adding checksum FAILED with Error code ${STATUS}"
+				exit "$STATUS"
+				;;
+			esac
+
+			# we're here if everything went well with STATUS=0
+			# so now, checksums are enabled on the SOURCE!
+			CHECKSUM_FLAG="--data-checksums"
+
+		else
+
+			log "9.1 did *not* add checksum to SOURCE PG: size $PGDATAOLD_SIZE > 20 Gb!"
+
+		fi
+
+	else
+
+		log "9.1 checksums already present on SOURCE"
+
+	fi
+
+	log "10. Creating new data directory, initializing database"
+	chown -R postgres:postgres "$MOUNT_POINT/"
+	rm -rf "${PGDATANEW:?}/"
+
+	if [ "$IS_NIX_UPGRADE" = "true" ]; then
+		if [[ ${PGVERSION%%.*} -ge 16 ]]; then
+			LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
+		else
+			LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
+		fi
+	else
+		su -c "$PGBINNEW/initdb --allow-group-access $CHECKSUM_FLAG -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres
+	fi
+
+	log "11. pg_upgrade --check(ing) compatibility of old/new clusters"
+	if [[ ${PGVERSION%%.*} -ge 16 ]]; then
+		GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres
+	else
+		GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres
+	fi
+
 	start_pid_lock_watcher
 
-	# Start the old PostgreSQL instance with version-specific options
+	# Start the pg_upgrade from PGDATAOLD to PGDATANEW..
+	log "12. pg_upgrade PGDATAOLD to PGDATANEW"
 	if [[ ${PGVERSION%%.*} -ge 16 ]]; then
 		GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres
 	else
@@ -686,7 +753,7 @@ EOF
 	fi
 
 	# copying custom configurations
-	log "11. Copying custom configurations"
+	log "13. Copying custom configurations"
 	mkdir -p "$MOUNT_POINT/conf"
 	cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/"
 	# removing supautils config as to allow the latest one provided by the latest image to be used
@@ -697,12 +764,12 @@ EOF
 	rm -f "$MOUNT_POINT/conf/wal-g.conf"
 
 	# copy sql files generated by pg_upgrade
-	log "12. Copying sql files generated by pg_upgrade"
+	log "14. Copying sql files generated by pg_upgrade"
 	mkdir -p "$MOUNT_POINT/sql"
 	cp /tmp/pg_upgrade/*.sql "$MOUNT_POINT/sql/" || true
 	chown -R postgres:postgres "$MOUNT_POINT/sql/"
 
-	log "13. Cleaning up"
+	log "15. Cleaning up"
 	cleanup "complete"
 }
 

ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh

@@ -1,15 +1,135 @@
 #! /usr/bin/env bash
-## This script is runs in advance of the database version upgrade, on the newly
-## launched instance which will eventually be promoted to become the primary
-## database instance once the upgrade successfully completes, terminating the
-## previous (source) instance.
-## The following commands safely stop the Postgres service and unmount
-## the data disk off the newly launched instance, to be re-attached to the
-## source instance and run the upgrade there.
 
-set -euo pipefail
+# This script runs in advance of the database version upgrade, on the newly
+# launched instance which will eventually be promoted to become the primary
+# database instance once the upgrade successfully completes, terminating the
+# previous (source) instance.  The following commands safely stop the Postgres
+# service and unmount the data disk off the newly launched instance, to be
+# re-attached to the source instance and run the upgrade there.
 
+set -Eeuo pipefail
+
+SCRIPT_DIR=$(dirname -- "$0")
+
+# The log file is never truncated: when the script is retried by the caller,
+# each attempt appends to the same file. It is created right away so that it
+# exists even if a later step fails.
+LOG_FILE="/var/log/pg-upgrade-prepare.log"
+touch "$LOG_FILE"
+chmod 640 "$LOG_FILE"
+
+# From here on, everything written to stdout/stderr (log messages, but also
+# errors from systemctl, install, sync, etc.) is written to the log file.
+exec >"$LOG_FILE" 2>&1
+
+# Minimal fallback so that "log" exists even if common.sh cannot be sourced.
+# common.sh redefines it with the real implementation.
+log() { echo "$(date -u '+%Y-%m-%d %H:%M:%S UTC') $*"; }
+
+# Log any unexpected failure (set -e) with the line number and failing command,
+# so that no error can end the script silently.
+trap 'log "prepare.sh FAILED at line $LINENO: \"$BASH_COMMAND\" (exit code $?)"' ERR
+
+# Source the shared helpers (log and ship_logs functions)
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/common.sh" || {
+	log "cannot source $SCRIPT_DIR/common.sh"
+	exit 1
+}
+
+# ship_logs reads these variables, and "set -u" would abort the script on an
+# unset variable. Default them to empty (existing values are kept) so that
+# ship_logs can handle the "no reporting key" case itself.
+: "${REPORTING_ANON_KEY:=}"
+: "${REPORTING_PROJECT_REF:=}"
+
+# Runs on every exit (success, explicit "exit 1" or set -e failure).
+#   - success: final message + single upload of the log file
+#   - failure: nothing more is written (the error is already in the log) and
+#     nothing is uploaded: the script cannot know whether the caller will
+#     retry, so the caller handles the "gave up" case.
+# The script's exit code is always preserved for the caller.
+on_exit() {
+	local rc=$?
+	trap - EXIT ERR # avoid re-entering the traps from inside this handler
+
+	if [ "$rc" -eq 0 ]; then
+		log "prepare.sh ended with no errors."
+		ship_logs "$LOG_FILE" || log "Log upload failed (non-blocking)."
+	else
+		ship_logs "$LOG_FILE" || log "Log upload failed (non-blocking)."
+	fi
+	exit "$rc"
+}
+trap on_exit EXIT
+
+# Written at every attempt. It is also the marker used above to detect
+# retries, so do not change its wording.
+log "prepare.sh started"
+
+# check that /data is actually mounted
+mountpoint -q /data || {
+	log "/data is not mounted"
+	exit 1
+}
+
+# stop postgresql: "systemctl stop blocks until the unit has fully stopped"
+# so there is no need to check pg_isready afterwards...
 systemctl stop postgresql
 
-cp /etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key
-umount /data
+# ...but let's still verify that the service is explicitly stopped
+if systemctl is-active --quiet postgresql; then
+	log "postgresql service is still active!"
+	exit 1
+fi
+
+# ...and that there are no remaining postgres processes
+if pgrep -x postgres >/dev/null; then
+	log "postgres processes are still alive!"
+	exit 1
+fi
+
+# copy the pgsodium root key with proper owner and permissions
+install -m 600 -o postgres -g postgres \
+	/etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key
+
+# flush cached writes to persistent storage
+sync
+
+# Unmount /data. Capture the native umount error message and route it through
+# "log" so that it is timestamped like the rest of the log entries.
+if ! umount_out=$(umount /data 2>&1); then
+	log "Unable to unmount /data: $umount_out"
+	# List the processes still using /data ("|| true" so a fuser failure
+	# does not trigger the ERR trap and mask the real cause)
+	log "Processes using /data:"
+
+	# activate nullglob to avoid errors if /proc is not accessible
+	shopt -s nullglob
+
+	# go thru /proc on numeric PIDs
+	for pid_dir in /proc/[0-9]*/; do
+
+		# extract pid from dir name
+		pid="${pid_dir#/proc/}"
+		pid="${pid%/}"
+
+		# secure read from symbolink link cwd
+		if cwd=$(readlink -f "$pid_dir/cwd" 2>/dev/null); then
+			if [[ $cwd == "/data" || $cwd == "/data"/* ]]; then
+				# get user and name of the current command
+				stat_info=$(stat -c "%U" "$pid_dir" 2>/dev/null || true)
+				cmd_info=$(cat "$pid_dir/comm" 2>/dev/null || true)
+
+				if [[ -n $stat_info && -n $cmd_info ]]; then
+					log "user: ${stat_info} pid: ${pid} command: ${cmd_info}"
+				fi
+			fi
+		fi
+	done
+
+	exit 1
+fi
+
+# Reaching this point means success: the final message and the log upload
+# are handled by the EXIT trap (on_exit).

ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh

@@ -4,10 +4,15 @@
 # file copy. Sourced by initiate.sh and tests/pg_upgrade/test-start-guard.sh.
 # Expects log, IS_CI, PGDATAOLD and PGDATANEW from the caller.
 
-# From step 10 until cleanup, nothing may start the source postgres: pg_upgrade
-# copies the data files without a postmaster holding the directory, and a start
-# in that window (Salt's `service.running`, adminapi, Restart=always, ...)
-# yields a copy that is inconsistent with the dump. Two independent guards:
+# From step 9 until cleanup, nothing may start the source postgres:
+#
+# If it has to run, "pg_checksums -e" will operate on the source postgres, and
+# this one has to stay stopped during the whole operation
+#
+# pg_upgrade copies the data files without a postmaster holding the directory,
+# and a start in that window (Salt's `service.running`, adminapi,
+# Restart=always, ...) yields a copy that is inconsistent with the dump. Two
+# independent guards:
 #  1. a runtime systemd drop-in that makes `systemctl start` fail while this
 #     script or pg_upgrade is alive (a SIGKILLed script must not leave the
 #     source blocked until a reboot);