docs-alpha.jayfield.org/scripts/alpha/sanity-check.sh

#!/bin/bash
# alpha.jayfield.org post-change sanity check.
# Run manually after any significant change (apt upgrade, config change);
# runs automatically once per boot via sanity-check.service (covers OS
# upgrades, kernel updates, and anything else that ends in a reboot).

set -u

LOGDIR=/var/log/sanity-check
mkdir -p "$LOGDIR"
LOG="$LOGDIR/$(date +%Y%m%d-%H%M%S).log"
exec > >(tee -a "$LOG") 2>&1

PASS=0
FAIL=0
WARN=0

ok()   { printf '  OK    %s\n' "$1"; PASS=$((PASS+1)); }
bad()  { printf '  FAIL  %s\n' "$1"; FAIL=$((FAIL+1)); }
warn() { printf '  WARN  %s\n' "$1"; WARN=$((WARN+1)); }
section() { printf '\n== %s ==\n' "$1"; }

echo "alpha.jayfield.org sanity check - $(date -Is)"
. /etc/os-release
echo "OS: $PRETTY_NAME, kernel $(uname -r), uptime $(uptime -p)"

# ---------- System package consistency ----------
section "System package consistency"
if [ -z "$(sudo dpkg --audit 2>&1)" ]; then
	ok "dpkg --audit clean"
else
	bad "dpkg --audit found issues"
fi

if sudo apt-get check >/dev/null 2>&1; then
	ok "apt-get check clean"
else
	bad "apt-get check reported broken dependencies"
fi

UPGRADABLE=$(apt list --upgradable 2>/dev/null | grep -vc '^Listing')
if [ "$UPGRADABLE" -eq 0 ]; then
	ok "no pending package upgrades"
else
	warn "$UPGRADABLE package(s) still upgradable"
fi

NEEDRESTART_SVC=$(sudo needrestart -b 2>/dev/null | grep -c '^NEEDRESTART-SVC')
if [ "$NEEDRESTART_SVC" -le 1 ]; then
	ok "no outstanding service restarts needed (allowing the harmless per-login user@ unit)"
else
	warn "$NEEDRESTART_SVC services still flagged by needrestart"
fi

# ---------- Disk space ----------
section "Disk space"
df -h / /var /boot
ROOT_PCT=$(df --output=pcent / | tail -1 | tr -dc '0-9')
if [ "$ROOT_PCT" -lt 90 ]; then
	ok "root filesystem usage ${ROOT_PCT}%"
else
	warn "root filesystem usage ${ROOT_PCT}% - getting full"
fi

# ---------- SSH ----------
section "SSH hardening"
SSHD_T=$(sudo sshd -T 2>/dev/null)
echo "$SSHD_T" | grep -q '^permitrootlogin no' && ok "PermitRootLogin no" || bad "PermitRootLogin not disabled"
echo "$SSHD_T" | grep -q '^passwordauthentication no' && ok "PasswordAuthentication no" || bad "PasswordAuthentication not disabled"
systemctl is-active --quiet ssh && ok "ssh.service active" || bad "ssh.service not active"

# ---------- DNS / BIND ----------
section "DNS / BIND"
systemctl is-active --quiet named && ok "named active" || bad "named not active"

if [ "$(dig @127.0.0.1 jayfield.org +short)" = "89.58.8.149" ]; then
	ok "jayfield.org resolves to the expected IP"
else
	bad "jayfield.org did not resolve to the expected IP"
fi

DNSSEC_STATUS=$(dig @127.0.0.1 github.com +noall +comment 2>/dev/null | grep -oP 'status: \K[A-Z]+')
if [ "$DNSSEC_STATUS" = "NOERROR" ]; then
	ok "DNSSEC validation working (github.com -> NOERROR)"
else
	bad "DNSSEC validation broken (github.com -> ${DNSSEC_STATUS:-no response})"
fi

for h in uschi vpn; do
	IP=$(dig @127.0.0.1 "$h.dyndns.jayfield.org" A +short)
	if [ -n "$IP" ]; then
		ok "$h.dyndns.jayfield.org resolves ($IP)"
	else
		warn "$h.dyndns.jayfield.org has no A record"
	fi
done

# ---------- Mail stack ----------
section "Mail stack"
systemctl is-active --quiet postfix@- && ok "postfix active" || bad "postfix not active"
systemctl is-active --quiet dovecot && ok "dovecot active" || bad "dovecot not active"
systemctl is-active --quiet rspamd && ok "rspamd active" || bad "rspamd not active"

SMTP_BANNER=$(printf 'QUIT\r\n' | timeout 5 nc 127.0.0.1 25 2>/dev/null | head -1)
echo "$SMTP_BANNER" | grep -q '220.*Postfix' && ok "SMTP banner OK" || bad "SMTP banner missing/unexpected: $SMTP_BANNER"

IMAP_BANNER=$(printf 'a LOGOUT\r\n' | timeout 5 nc 127.0.0.1 143 2>/dev/null | head -1)
echo "$IMAP_BANNER" | grep -qi 'dovecot ready\|^\* OK' && ok "IMAP banner OK" || bad "IMAP banner missing/unexpected: $IMAP_BANNER"

for port in 11332 11333 11334; do
	if sudo ss -ltn | grep -q ":$port "; then
		ok "rspamd listening on $port"
	else
		bad "rspamd NOT listening on $port"
	fi
done

RSPAMD_TEST=$(printf 'Subject: sanity test\nFrom: test@example.com\nTo: jens@jayfield.org\n\nbody\n' | rspamc symbols 2>&1)
if echo "$RSPAMD_TEST" | grep -q 'Action:'; then
	ok "rspamd genuinely scanning (rspamc symbols responded)"
else
	bad "rspamd did not respond to a live scan test - check for a phantom 'active' status"
fi

if sudo ss -ltn | grep -q ':11332 '; then
	ok "postfix's configured milter target (11332) is listening"
else
	bad "nothing listening on postfix's milter port - mail is flowing unfiltered/unsigned"
fi

# ---------- MariaDB / vmail ----------
section "MariaDB / vmail"
systemctl is-active --quiet mariadb && ok "mariadb active" || bad "mariadb not active"

DOMAINS=$(sudo mysql -N -e "SELECT COUNT(*) FROM vmail.domains;" 2>/dev/null)
ACCOUNTS=$(sudo mysql -N -e "SELECT COUNT(*) FROM vmail.accounts;" 2>/dev/null)
[ -n "$DOMAINS" ] && [ "$DOMAINS" -ge 1 ] && ok "vmail.domains has $DOMAINS row(s)" || bad "vmail.domains query failed or returned zero rows"
[ -n "$ACCOUNTS" ] && [ "$ACCOUNTS" -ge 1 ] && ok "vmail.accounts has $ACCOUNTS row(s)" || bad "vmail.accounts query failed or returned zero rows"

if sudo postmap -q jayfield.org mysql:/etc/postfix/sql/domains.cf 2>/dev/null | grep -q jayfield.org; then
	ok "Postfix's MySQL domain lookup works"
else
	bad "Postfix's MySQL domain lookup failed"
fi

# ---------- fail2ban ----------
section "fail2ban"
systemctl is-active --quiet fail2ban && ok "fail2ban active" || bad "fail2ban not active"

JAIL_COUNT=$(sudo fail2ban-client status 2>/dev/null | grep -oP 'Number of jail:\s*\K[0-9]+')
if [ -n "$JAIL_COUNT" ] && [ "$JAIL_COUNT" -ge 15 ]; then
	ok "$JAIL_COUNT jails loaded"
else
	bad "only ${JAIL_COUNT:-0} jail(s) loaded (expected ~19)"
fi

SSHD_FT=$(sudo fail2ban-client get sshd findtime 2>/dev/null)
DOVECOT_FT=$(sudo fail2ban-client get dovecot findtime 2>/dev/null)
[ "$SSHD_FT" = "86400" ] && ok "sshd findtime hardening intact (86400s)" || bad "sshd findtime is ${SSHD_FT:-unknown}, expected 86400"
[ "$DOVECOT_FT" = "86400" ] && ok "dovecot findtime hardening intact (86400s)" || bad "dovecot findtime is ${DOVECOT_FT:-unknown}, expected 86400"

# ---------- Apache / vhosts ----------
section "Apache / vhosts"
systemctl is-active --quiet apache2 && ok "apache2 active" || bad "apache2 not active"
sudo apache2ctl configtest >/dev/null 2>&1 && ok "apache2ctl configtest clean" || bad "apache2ctl configtest failed"

check_url() {
	local url=$1 expected=$2 code
	code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "$url")
	if [ "$code" = "$expected" ]; then
		ok "$url -> $code"
	else
		bad "$url -> $code (expected $expected)"
	fi
}
check_url "https://jayfield.org/" 200
check_url "https://www.jayfield.org/" 200
check_url "https://dyndns.jayfield.org/" 401
check_url "https://mail.jayfield.org/" 403
check_url "https://cloud.jayfield.org/" 302
check_url "https://portainer.jayfield.org/" 200
check_url "https://git.jayfield.org/" 200

# ---------- Docker ----------
section "Docker"
systemctl is-active --quiet docker && ok "docker active" || bad "docker not active"

for c in portainer nextcloud nextcloud-redis nextcloud-db web gitea; do
	STATUS=$(sudo docker inspect -f '{{.State.Status}}' "$c" 2>/dev/null)
	if [ "$STATUS" = "running" ]; then
		ok "container $c running"
	else
		bad "container $c is '${STATUS:-missing}' (expected running)"
	fi
done

GITEA_REPO_COUNT=$(curl -s --max-time 5 'https://git.jayfield.org/api/v1/repos/search?limit=1' | python3 -c 'import json,sys; print(len(json.load(sys.stdin).get("data",[])))' 2>/dev/null)
if [ "$GITEA_REPO_COUNT" = "1" ]; then
	ok "Gitea API returns real repo data (not just a static page)"
else
	bad "Gitea API did not return expected repo data"
fi

# ---------- Summary ----------
section "Summary"
echo "PASS: $PASS   WARN: $WARN   FAIL: $FAIL"
echo "Full log: $LOG"

if command -v mail >/dev/null 2>&1; then
	SUBJECT="alpha.jayfield.org sanity check: $([ "$FAIL" -gt 0 ] && echo "$FAIL FAILURE(S)" || echo "OK")"
	mail -s "$SUBJECT" jens@jayfield.org < "$LOG"
fi

[ "$FAIL" -eq 0 ]