#!/bin/bash
# vlda-01 side of the alpha -> vlda-01 Gitea backup strategy
# (see SYNC-PLAN.md "Ongoing backup" section). Pulls whatever nightly
# gitea-dump archives exist on alpha that aren't already here yet - so a
# run after several days offline just catches up on all of them, not
# just the latest. Verifies each dump's sha256 checksum after pulling
# it, and only then deletes the alpha-side copy - a failed/incomplete
# verification leaves the remote copy alone for a retry next run.
#
# Uses SFTP (not real rsync) because the alpha-side account is
# restricted to `ForceCommand internal-sftp` - that blocks a real
# `rsync --server` invocation, so plain SFTP get/rm is the compatible
# transport here.
#
# Stored on the WSD2L840 Unassigned Device (not the array), which isn't
# guaranteed to already be mounted - this script mounts it itself first
# and refuses to proceed if that didn't actually result in a real
# mountpoint, rather than risk silently writing backups onto the array.
# Unmounts it again once done, but only if this run is the one that
# mounted it - never unmounts a disk something else already had mounted
# for its own reasons.

set -u
KEY=/root/.ssh/gitea-backup/id_ed25519
KNOWN_HOSTS=/root/.ssh/gitea-backup/known_hosts
REMOTE=giteabackup@alpha.jayfield.org
REMOTE_PORT=10022
UD_LABEL=WSD2L840
UD_MOUNTPOINT=/mnt/disks/WSD2L840
LOCAL_DIR="$UD_MOUNTPOINT/git"
KEEP=30

# WSD2L840 is an Unassigned Device, not part of the array - not
# guaranteed to already be mounted when this runs. Resolve by label
# (not a hardcoded /dev/sdX - USB/UD device paths aren't stable across
# reboots) and mount it via the Unassigned Devices plugin's own tool,
# which is safe to call even if it's already mounted.
UD_DEV=$(findfs LABEL="$UD_LABEL" 2>/dev/null)
if [ -z "$UD_DEV" ]; then
	echo "$(date -Is) ERROR: no device labeled $UD_LABEL found - is it plugged in?" >&2
	exit 1
fi

UD_ALREADY_MOUNTED=false
mountpoint -q "$UD_MOUNTPOINT" && UD_ALREADY_MOUNTED=true

/usr/local/sbin/rc.unassigned mount "$UD_DEV" >/dev/null 2>&1

if ! mountpoint -q "$UD_MOUNTPOINT"; then
	echo "$(date -Is) ERROR: $UD_MOUNTPOINT did not come up as a real mountpoint after mount attempt - refusing to write backups onto the array by mistake" >&2
	exit 1
fi

GET_BATCH=$(mktemp)
DEL_BATCH=$(mktemp)
cleanup() {
	rm -f "$GET_BATCH" "$DEL_BATCH"
	if [ "$UD_ALREADY_MOUNTED" = false ]; then
		# cd out first - our own cwd on the device would otherwise
		# make the fuser check below always find "something" using it
		cd / || true

		# rc.unassigned's umount always does a lazy (-l) unmount and
		# reports "success" no matter what - it does NOT refuse or
		# error out just because another process still has the
		# mountpoint open (verified live: a process with its cwd on
		# the device does not stop it, and does not get killed
		# either, unless this were a forced "stop array"-style
		# unmount, which a single-device call never is). So the only
		# way to actually detect and respect another user of the
		# disk is to check first, ourselves, and simply not attempt
		# the unmount at all if anything is found - no ERROR, just
		# leave it mounted for next time.
		if fuser -m "$UD_MOUNTPOINT" >/dev/null 2>&1; then
			echo "$(date -Is) $UD_MOUNTPOINT still in use by another process - leaving it mounted"
		else
			/usr/local/sbin/rc.unassigned umount "$UD_DEV" >/dev/null 2>&1
		fi
	fi
}
trap cleanup EXIT

mkdir -p "$LOCAL_DIR"

SFTP="sftp -P $REMOTE_PORT -i $KEY -o UserKnownHostsFile=$KNOWN_HOSTS -o BatchMode=yes"

REMOTE_LISTING=$($SFTP "$REMOTE" <<< 'ls -1 backups/' 2>/dev/null)
if [ -z "$REMOTE_LISTING" ]; then
	echo "$(date -Is) ERROR: could not list remote backups (host down or auth failed)" >&2
	exit 1
fi
REMOTE_ZIPS=$(echo "$REMOTE_LISTING" | grep '\.zip$')

TO_FETCH=0
while read -r fname; do
	[ -z "$fname" ] && continue
	base=$(basename "$fname")
	if [ ! -f "$LOCAL_DIR/$base" ]; then
		echo "get $fname $LOCAL_DIR/" >> "$GET_BATCH"
		TO_FETCH=$((TO_FETCH + 1))
		# grab the checksum alongside it if alpha actually has one
		# (older dumps from before checksums existed won't)
		if echo "$REMOTE_LISTING" | grep -qxF "${fname}.sha256"; then
			echo "get ${fname}.sha256 $LOCAL_DIR/" >> "$GET_BATCH"
		fi
	fi
done <<< "$REMOTE_ZIPS"

if [ "$TO_FETCH" -eq 0 ]; then
	echo "$(date -Is) up to date, nothing new to pull"
	exit 0
fi

if ! $SFTP -b "$GET_BATCH" "$REMOTE" 2>&1; then
	echo "$(date -Is) ERROR: sftp batch fetch failed partway through" >&2
	exit 1
fi

# verify each freshly-fetched dump before trusting it enough to delete
# the alpha-side copy
VERIFIED=0
UNVERIFIED=0
while read -r fname; do
	[ -z "$fname" ] && continue
	base=$(basename "$fname")
	local_zip="$LOCAL_DIR/$base"
	local_sum="$LOCAL_DIR/$base.sha256"
	[ -f "$local_zip" ] || continue   # wasn't newly fetched this run

	if [ ! -f "$local_sum" ]; then
		echo "$(date -Is) WARNING: no checksum available for $base (older dump, predates checksums?) - keeping local copy, leaving alpha's copy alone" >&2
		UNVERIFIED=$((UNVERIFIED + 1))
		continue
	fi

	if (cd "$LOCAL_DIR" && sha256sum -c "$(basename "$local_sum")") >/dev/null 2>&1; then
		echo "rm $fname" >> "$DEL_BATCH"
		echo "rm ${fname}.sha256" >> "$DEL_BATCH"
		VERIFIED=$((VERIFIED + 1))
	else
		echo "$(date -Is) ERROR: checksum mismatch for $base - deleting the bad local copy, leaving alpha's copy for a retry" >&2
		rm -f "$local_zip" "$local_sum"
		UNVERIFIED=$((UNVERIFIED + 1))
	fi
done <<< "$REMOTE_ZIPS"

if [ -s "$DEL_BATCH" ]; then
	if ! $SFTP -b "$DEL_BATCH" "$REMOTE" 2>&1; then
		echo "$(date -Is) WARNING: some verified files could not be deleted on alpha - harmless, alpha's own retention prune will clean them up eventually" >&2
	fi
fi

# prune to the last $KEEP archives (vlda-01 has far more headroom than
# alpha, so this is the real long-term history now that alpha deletes
# its own copies right after a verified pull)
cd "$LOCAL_DIR" || exit 1
ls -1t gitea-dump-*.zip 2>/dev/null | tail -n +$((KEEP + 1)) | while read -r old; do
	rm -f "$old" "${old}.sha256"
done

echo "$(date -Is) pulled $TO_FETCH new archive(s), $VERIFIED verified+deleted from alpha, $UNVERIFIED kept unverified, $(ls -1 gitea-dump-*.zip 2>/dev/null | wc -l) retained locally"