jensandClaude Sonnet 5
5b286644db
Previously the pull script mounted the Unassigned Device and left it mounted indefinitely. Now tracks whether it was already mounted before this run and only unmounts at the end if it wasn't - never disturbs a mount already in place for some other reason. Handled via an EXIT trap so it fires on any exit path, not just success. Found and fixed a real bug testing this: the script cd's into the mounted directory and never leaves, so the unmount was failing with "target is busy" (the shell's own cwd still being on the device) until the cleanup function explicitly cd's out first. Verified both directions live: already-mounted stays mounted after; not-mounted gets mounted, used, and cleanly unmounted again. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
147 lines
5.3 KiB
Bash
147 lines
5.3 KiB
Bash
#!/bin/bash
|
|
# vlda-01 side of the alpha -> vlda-01 Gitea backup strategy
|
|
# (see SYNC-PLAN.md "Ongoing backup" section). Pulls whatever nightly
|
|
# gitea-dump archives exist on alpha that aren't already here yet - so a
|
|
# run after several days offline just catches up on all of them, not
|
|
# just the latest. Verifies each dump's sha256 checksum after pulling
|
|
# it, and only then deletes the alpha-side copy - a failed/incomplete
|
|
# verification leaves the remote copy alone for a retry next run.
|
|
#
|
|
# Uses SFTP (not real rsync) because the alpha-side account is
|
|
# restricted to `ForceCommand internal-sftp` - that blocks a real
|
|
# `rsync --server` invocation, so plain SFTP get/rm is the compatible
|
|
# transport here.
|
|
#
|
|
# Stored on the WSD2L840 Unassigned Device (not the array), which isn't
|
|
# guaranteed to already be mounted - this script mounts it itself first
|
|
# and refuses to proceed if that didn't actually result in a real
|
|
# mountpoint, rather than risk silently writing backups onto the array.
|
|
# Unmounts it again once done, but only if this run is the one that
|
|
# mounted it - never unmounts a disk something else already had mounted
|
|
# for its own reasons.
|
|
|
|
set -u
|
|
KEY=/root/.ssh/gitea-backup/id_ed25519
|
|
KNOWN_HOSTS=/root/.ssh/gitea-backup/known_hosts
|
|
REMOTE=giteabackup@alpha.jayfield.org
|
|
REMOTE_PORT=10022
|
|
UD_LABEL=WSD2L840
|
|
UD_MOUNTPOINT=/mnt/disks/WSD2L840
|
|
LOCAL_DIR="$UD_MOUNTPOINT/git"
|
|
KEEP=30
|
|
|
|
# WSD2L840 is an Unassigned Device, not part of the array - not
|
|
# guaranteed to already be mounted when this runs. Resolve by label
|
|
# (not a hardcoded /dev/sdX - USB/UD device paths aren't stable across
|
|
# reboots) and mount it via the Unassigned Devices plugin's own tool,
|
|
# which is safe to call even if it's already mounted.
|
|
UD_DEV=$(findfs LABEL="$UD_LABEL" 2>/dev/null)
|
|
if [ -z "$UD_DEV" ]; then
|
|
echo "$(date -Is) ERROR: no device labeled $UD_LABEL found - is it plugged in?" >&2
|
|
exit 1
|
|
fi
|
|
|
|
UD_ALREADY_MOUNTED=false
|
|
mountpoint -q "$UD_MOUNTPOINT" && UD_ALREADY_MOUNTED=true
|
|
|
|
/usr/local/sbin/rc.unassigned mount "$UD_DEV" >/dev/null 2>&1
|
|
|
|
if ! mountpoint -q "$UD_MOUNTPOINT"; then
|
|
echo "$(date -Is) ERROR: $UD_MOUNTPOINT did not come up as a real mountpoint after mount attempt - refusing to write backups onto the array by mistake" >&2
|
|
exit 1
|
|
fi
|
|
|
|
GET_BATCH=$(mktemp)
|
|
DEL_BATCH=$(mktemp)
|
|
cleanup() {
|
|
rm -f "$GET_BATCH" "$DEL_BATCH"
|
|
if [ "$UD_ALREADY_MOUNTED" = false ]; then
|
|
# cd out first - if the shell's own cwd is still on the device
|
|
# (it is, after the "cd $LOCAL_DIR" below), umount fails with
|
|
# "target is busy"
|
|
cd / || true
|
|
/usr/local/sbin/rc.unassigned umount "$UD_DEV" >/dev/null 2>&1
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
mkdir -p "$LOCAL_DIR"
|
|
|
|
SFTP="sftp -P $REMOTE_PORT -i $KEY -o UserKnownHostsFile=$KNOWN_HOSTS -o BatchMode=yes"
|
|
|
|
REMOTE_LISTING=$($SFTP "$REMOTE" <<< 'ls -1 backups/' 2>/dev/null)
|
|
if [ -z "$REMOTE_LISTING" ]; then
|
|
echo "$(date -Is) ERROR: could not list remote backups (host down or auth failed)" >&2
|
|
exit 1
|
|
fi
|
|
REMOTE_ZIPS=$(echo "$REMOTE_LISTING" | grep '\.zip$')
|
|
|
|
TO_FETCH=0
|
|
while read -r fname; do
|
|
[ -z "$fname" ] && continue
|
|
base=$(basename "$fname")
|
|
if [ ! -f "$LOCAL_DIR/$base" ]; then
|
|
echo "get $fname $LOCAL_DIR/" >> "$GET_BATCH"
|
|
TO_FETCH=$((TO_FETCH + 1))
|
|
# grab the checksum alongside it if alpha actually has one
|
|
# (older dumps from before checksums existed won't)
|
|
if echo "$REMOTE_LISTING" | grep -qxF "${fname}.sha256"; then
|
|
echo "get ${fname}.sha256 $LOCAL_DIR/" >> "$GET_BATCH"
|
|
fi
|
|
fi
|
|
done <<< "$REMOTE_ZIPS"
|
|
|
|
if [ "$TO_FETCH" -eq 0 ]; then
|
|
echo "$(date -Is) up to date, nothing new to pull"
|
|
exit 0
|
|
fi
|
|
|
|
if ! $SFTP -b "$GET_BATCH" "$REMOTE" 2>&1; then
|
|
echo "$(date -Is) ERROR: sftp batch fetch failed partway through" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# verify each freshly-fetched dump before trusting it enough to delete
|
|
# the alpha-side copy
|
|
VERIFIED=0
|
|
UNVERIFIED=0
|
|
while read -r fname; do
|
|
[ -z "$fname" ] && continue
|
|
base=$(basename "$fname")
|
|
local_zip="$LOCAL_DIR/$base"
|
|
local_sum="$LOCAL_DIR/$base.sha256"
|
|
[ -f "$local_zip" ] || continue # wasn't newly fetched this run
|
|
|
|
if [ ! -f "$local_sum" ]; then
|
|
echo "$(date -Is) WARNING: no checksum available for $base (older dump, predates checksums?) - keeping local copy, leaving alpha's copy alone" >&2
|
|
UNVERIFIED=$((UNVERIFIED + 1))
|
|
continue
|
|
fi
|
|
|
|
if (cd "$LOCAL_DIR" && sha256sum -c "$(basename "$local_sum")") >/dev/null 2>&1; then
|
|
echo "rm $fname" >> "$DEL_BATCH"
|
|
echo "rm ${fname}.sha256" >> "$DEL_BATCH"
|
|
VERIFIED=$((VERIFIED + 1))
|
|
else
|
|
echo "$(date -Is) ERROR: checksum mismatch for $base - deleting the bad local copy, leaving alpha's copy for a retry" >&2
|
|
rm -f "$local_zip" "$local_sum"
|
|
UNVERIFIED=$((UNVERIFIED + 1))
|
|
fi
|
|
done <<< "$REMOTE_ZIPS"
|
|
|
|
if [ -s "$DEL_BATCH" ]; then
|
|
if ! $SFTP -b "$DEL_BATCH" "$REMOTE" 2>&1; then
|
|
echo "$(date -Is) WARNING: some verified files could not be deleted on alpha - harmless, alpha's own retention prune will clean them up eventually" >&2
|
|
fi
|
|
fi
|
|
|
|
# prune to the last $KEEP archives (vlda-01 has far more headroom than
|
|
# alpha, so this is the real long-term history now that alpha deletes
|
|
# its own copies right after a verified pull)
|
|
cd "$LOCAL_DIR" || exit 1
|
|
ls -1t gitea-dump-*.zip 2>/dev/null | tail -n +$((KEEP + 1)) | while read -r old; do
|
|
rm -f "$old" "${old}.sha256"
|
|
done
|
|
|
|
echo "$(date -Is) pulled $TO_FETCH new archive(s), $VERIFIED verified+deleted from alpha, $UNVERIFIED kept unverified, $(ls -1 gitea-dump-*.zip 2>/dev/null | wc -l) retained locally"
|