jens
495613018c
rc.unassigned umount always does a lazy unmount and reports success unconditionally, even with an open cwd on the mountpoint - it never actually refuses on its own. Check with fuser -m first instead, and just leave the disk mounted if anything is using it.
162 lines
6.1 KiB
Bash
162 lines
6.1 KiB
Bash
#!/bin/bash
|
|
# vlda-01 side of the alpha -> vlda-01 Gitea backup strategy
|
|
# (see SYNC-PLAN.md "Ongoing backup" section). Pulls whatever nightly
|
|
# gitea-dump archives exist on alpha that aren't already here yet - so a
|
|
# run after several days offline just catches up on all of them, not
|
|
# just the latest. Verifies each dump's sha256 checksum after pulling
|
|
# it, and only then deletes the alpha-side copy - a failed/incomplete
|
|
# verification leaves the remote copy alone for a retry next run.
|
|
#
|
|
# Uses SFTP (not real rsync) because the alpha-side account is
|
|
# restricted to `ForceCommand internal-sftp` - that blocks a real
|
|
# `rsync --server` invocation, so plain SFTP get/rm is the compatible
|
|
# transport here.
|
|
#
|
|
# Stored on the WSD2L840 Unassigned Device (not the array), which isn't
|
|
# guaranteed to already be mounted - this script mounts it itself first
|
|
# and refuses to proceed if that didn't actually result in a real
|
|
# mountpoint, rather than risk silently writing backups onto the array.
|
|
# Unmounts it again once done, but only if this run is the one that
|
|
# mounted it - never unmounts a disk something else already had mounted
|
|
# for its own reasons.
|
|
|
|
set -u
|
|
KEY=/root/.ssh/gitea-backup/id_ed25519
|
|
KNOWN_HOSTS=/root/.ssh/gitea-backup/known_hosts
|
|
REMOTE=giteabackup@alpha.jayfield.org
|
|
REMOTE_PORT=10022
|
|
UD_LABEL=WSD2L840
|
|
UD_MOUNTPOINT=/mnt/disks/WSD2L840
|
|
LOCAL_DIR="$UD_MOUNTPOINT/git"
|
|
KEEP=30
|
|
|
|
# WSD2L840 is an Unassigned Device, not part of the array - not
|
|
# guaranteed to already be mounted when this runs. Resolve by label
|
|
# (not a hardcoded /dev/sdX - USB/UD device paths aren't stable across
|
|
# reboots) and mount it via the Unassigned Devices plugin's own tool,
|
|
# which is safe to call even if it's already mounted.
|
|
UD_DEV=$(findfs LABEL="$UD_LABEL" 2>/dev/null)
|
|
if [ -z "$UD_DEV" ]; then
|
|
echo "$(date -Is) ERROR: no device labeled $UD_LABEL found - is it plugged in?" >&2
|
|
exit 1
|
|
fi
|
|
|
|
UD_ALREADY_MOUNTED=false
|
|
mountpoint -q "$UD_MOUNTPOINT" && UD_ALREADY_MOUNTED=true
|
|
|
|
/usr/local/sbin/rc.unassigned mount "$UD_DEV" >/dev/null 2>&1
|
|
|
|
if ! mountpoint -q "$UD_MOUNTPOINT"; then
|
|
echo "$(date -Is) ERROR: $UD_MOUNTPOINT did not come up as a real mountpoint after mount attempt - refusing to write backups onto the array by mistake" >&2
|
|
exit 1
|
|
fi
|
|
|
|
GET_BATCH=$(mktemp)
|
|
DEL_BATCH=$(mktemp)
|
|
cleanup() {
|
|
rm -f "$GET_BATCH" "$DEL_BATCH"
|
|
if [ "$UD_ALREADY_MOUNTED" = false ]; then
|
|
# cd out first - our own cwd on the device would otherwise
|
|
# make the fuser check below always find "something" using it
|
|
cd / || true
|
|
|
|
# rc.unassigned's umount always does a lazy (-l) unmount and
|
|
# reports "success" no matter what - it does NOT refuse or
|
|
# error out just because another process still has the
|
|
# mountpoint open (verified live: a process with its cwd on
|
|
# the device does not stop it, and does not get killed
|
|
# either, unless this were a forced "stop array"-style
|
|
# unmount, which a single-device call never is). So the only
|
|
# way to actually detect and respect another user of the
|
|
# disk is to check first, ourselves, and simply not attempt
|
|
# the unmount at all if anything is found - no ERROR, just
|
|
# leave it mounted for next time.
|
|
if fuser -m "$UD_MOUNTPOINT" >/dev/null 2>&1; then
|
|
echo "$(date -Is) $UD_MOUNTPOINT still in use by another process - leaving it mounted"
|
|
else
|
|
/usr/local/sbin/rc.unassigned umount "$UD_DEV" >/dev/null 2>&1
|
|
fi
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
mkdir -p "$LOCAL_DIR"
|
|
|
|
SFTP="sftp -P $REMOTE_PORT -i $KEY -o UserKnownHostsFile=$KNOWN_HOSTS -o BatchMode=yes"
|
|
|
|
REMOTE_LISTING=$($SFTP "$REMOTE" <<< 'ls -1 backups/' 2>/dev/null)
|
|
if [ -z "$REMOTE_LISTING" ]; then
|
|
echo "$(date -Is) ERROR: could not list remote backups (host down or auth failed)" >&2
|
|
exit 1
|
|
fi
|
|
REMOTE_ZIPS=$(echo "$REMOTE_LISTING" | grep '\.zip$')
|
|
|
|
TO_FETCH=0
|
|
while read -r fname; do
|
|
[ -z "$fname" ] && continue
|
|
base=$(basename "$fname")
|
|
if [ ! -f "$LOCAL_DIR/$base" ]; then
|
|
echo "get $fname $LOCAL_DIR/" >> "$GET_BATCH"
|
|
TO_FETCH=$((TO_FETCH + 1))
|
|
# grab the checksum alongside it if alpha actually has one
|
|
# (older dumps from before checksums existed won't)
|
|
if echo "$REMOTE_LISTING" | grep -qxF "${fname}.sha256"; then
|
|
echo "get ${fname}.sha256 $LOCAL_DIR/" >> "$GET_BATCH"
|
|
fi
|
|
fi
|
|
done <<< "$REMOTE_ZIPS"
|
|
|
|
if [ "$TO_FETCH" -eq 0 ]; then
|
|
echo "$(date -Is) up to date, nothing new to pull"
|
|
exit 0
|
|
fi
|
|
|
|
if ! $SFTP -b "$GET_BATCH" "$REMOTE" 2>&1; then
|
|
echo "$(date -Is) ERROR: sftp batch fetch failed partway through" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# verify each freshly-fetched dump before trusting it enough to delete
|
|
# the alpha-side copy
|
|
VERIFIED=0
|
|
UNVERIFIED=0
|
|
while read -r fname; do
|
|
[ -z "$fname" ] && continue
|
|
base=$(basename "$fname")
|
|
local_zip="$LOCAL_DIR/$base"
|
|
local_sum="$LOCAL_DIR/$base.sha256"
|
|
[ -f "$local_zip" ] || continue # wasn't newly fetched this run
|
|
|
|
if [ ! -f "$local_sum" ]; then
|
|
echo "$(date -Is) WARNING: no checksum available for $base (older dump, predates checksums?) - keeping local copy, leaving alpha's copy alone" >&2
|
|
UNVERIFIED=$((UNVERIFIED + 1))
|
|
continue
|
|
fi
|
|
|
|
if (cd "$LOCAL_DIR" && sha256sum -c "$(basename "$local_sum")") >/dev/null 2>&1; then
|
|
echo "rm $fname" >> "$DEL_BATCH"
|
|
echo "rm ${fname}.sha256" >> "$DEL_BATCH"
|
|
VERIFIED=$((VERIFIED + 1))
|
|
else
|
|
echo "$(date -Is) ERROR: checksum mismatch for $base - deleting the bad local copy, leaving alpha's copy for a retry" >&2
|
|
rm -f "$local_zip" "$local_sum"
|
|
UNVERIFIED=$((UNVERIFIED + 1))
|
|
fi
|
|
done <<< "$REMOTE_ZIPS"
|
|
|
|
if [ -s "$DEL_BATCH" ]; then
|
|
if ! $SFTP -b "$DEL_BATCH" "$REMOTE" 2>&1; then
|
|
echo "$(date -Is) WARNING: some verified files could not be deleted on alpha - harmless, alpha's own retention prune will clean them up eventually" >&2
|
|
fi
|
|
fi
|
|
|
|
# prune to the last $KEEP archives (vlda-01 has far more headroom than
|
|
# alpha, so this is the real long-term history now that alpha deletes
|
|
# its own copies right after a verified pull)
|
|
cd "$LOCAL_DIR" || exit 1
|
|
ls -1t gitea-dump-*.zip 2>/dev/null | tail -n +$((KEEP + 1)) | while read -r old; do
|
|
rm -f "$old" "${old}.sha256"
|
|
done
|
|
|
|
echo "$(date -Is) pulled $TO_FETCH new archive(s), $VERIFIED verified+deleted from alpha, $UNVERIFIED kept unverified, $(ls -1 gitea-dump-*.zip 2>/dev/null | wc -l) retained locally"
|