jens
45856f48f8
Informational only, doesn't affect PASS/WARN/FAIL - lists apache/bind/ postfix/dovecot/rspamd/redis/mariadb/fail2ban/docker/certbot/php7.4/8.0 plus each Docker container's actual running version (not just its image tag), queried from inside the container where the tag alone doesn't say much.
242 lines
9.9 KiB
Bash
242 lines
9.9 KiB
Bash
#!/bin/bash
|
|
# alpha.jayfield.org post-change sanity check.
|
|
# Run manually after any significant change (apt upgrade, config change);
|
|
# runs automatically once per boot via sanity-check.service (covers OS
|
|
# upgrades, kernel updates, and anything else that ends in a reboot).
|
|
|
|
set -u
|
|
|
|
LOGDIR=/var/log/sanity-check
|
|
mkdir -p "$LOGDIR"
|
|
LOG="$LOGDIR/$(date +%Y%m%d-%H%M%S).log"
|
|
exec > >(tee -a "$LOG") 2>&1
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
WARN=0
|
|
|
|
ok() { printf ' OK %s\n' "$1"; PASS=$((PASS+1)); }
|
|
bad() { printf ' FAIL %s\n' "$1"; FAIL=$((FAIL+1)); }
|
|
warn() { printf ' WARN %s\n' "$1"; WARN=$((WARN+1)); }
|
|
section() { printf '\n== %s ==\n' "$1"; }
|
|
|
|
echo "alpha.jayfield.org sanity check - $(date -Is)"
|
|
. /etc/os-release
|
|
echo "OS: $PRETTY_NAME, kernel $(uname -r), uptime $(uptime -p)"
|
|
|
|
# ---------- Versions ----------
|
|
# Informational only, not pass/fail - a quick reference for what's
|
|
# actually running without having to SSH in and check each service
|
|
# individually after an upgrade.
|
|
section "Versions"
|
|
echo "apache2: $(apache2 -v 2>/dev/null | head -1)"
|
|
echo "bind9: $(named -v 2>/dev/null | cut -d' ' -f1-2)"
|
|
echo "postfix: $(postconf mail_version 2>/dev/null)"
|
|
echo "dovecot: $(dovecot --version 2>/dev/null)"
|
|
echo "rspamd: $(rspamd --version 2>/dev/null | head -1)"
|
|
echo "redis-server: $(redis-server --version 2>/dev/null)"
|
|
echo "mariadb: $(mysql --version 2>/dev/null)"
|
|
echo "fail2ban: $(fail2ban-client --version 2>/dev/null)"
|
|
echo "docker: $(docker --version 2>/dev/null)"
|
|
echo "docker compose: $(docker compose version 2>/dev/null | head -1)"
|
|
echo "certbot: $(certbot --version 2>/dev/null)"
|
|
echo "php7.4: $(php7.4 -v 2>/dev/null | head -1)"
|
|
echo "php8.0: $(php8.0 -v 2>/dev/null | head -1)"
|
|
echo
|
|
echo "Docker containers:"
|
|
echo " portainer: $(sudo docker inspect portainer -f '{{.Config.Image}}' 2>/dev/null), app $(curl -sk --max-time 3 https://127.0.0.1:9443/api/status 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin).get("Version","?"))' 2>/dev/null || echo unknown)"
|
|
echo " web (nginx): $(sudo docker exec web nginx -v 2>&1)"
|
|
echo " nextcloud: $(sudo docker inspect nextcloud -f '{{.Config.Image}}' 2>/dev/null), app $(sudo docker exec -u www-data nextcloud php occ status 2>/dev/null | grep versionstring | awk '{print $3}')"
|
|
echo " nextcloud-db: $(sudo docker exec nextcloud-db mariadb --version 2>/dev/null)"
|
|
echo " nextcloud-redis: $(sudo docker exec nextcloud-redis redis-server --version 2>/dev/null)"
|
|
echo " gitea: $(sudo docker inspect gitea -f '{{.Config.Image}}' 2>/dev/null), API reports $(curl -s --max-time 3 https://git.jayfield.org/api/v1/version 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin).get("version","?"))' 2>/dev/null || echo unknown)"
|
|
|
|
# ---------- System package consistency ----------
|
|
section "System package consistency"
|
|
if [ -z "$(sudo dpkg --audit 2>&1)" ]; then
|
|
ok "dpkg --audit clean"
|
|
else
|
|
bad "dpkg --audit found issues"
|
|
fi
|
|
|
|
if sudo apt-get check >/dev/null 2>&1; then
|
|
ok "apt-get check clean"
|
|
else
|
|
bad "apt-get check reported broken dependencies"
|
|
fi
|
|
|
|
UPGRADABLE=$(apt list --upgradable 2>/dev/null | grep -vc '^Listing')
|
|
if [ "$UPGRADABLE" -eq 0 ]; then
|
|
ok "no pending package upgrades"
|
|
else
|
|
warn "$UPGRADABLE package(s) still upgradable"
|
|
fi
|
|
|
|
NEEDRESTART_SVC=$(sudo needrestart -b 2>/dev/null | grep -c '^NEEDRESTART-SVC')
|
|
if [ "$NEEDRESTART_SVC" -le 1 ]; then
|
|
ok "no outstanding service restarts needed (allowing the harmless per-login user@ unit)"
|
|
else
|
|
warn "$NEEDRESTART_SVC services still flagged by needrestart"
|
|
fi
|
|
|
|
# ---------- Disk space ----------
|
|
section "Disk space"
|
|
df -h / /var /boot
|
|
ROOT_PCT=$(df --output=pcent / | tail -1 | tr -dc '0-9')
|
|
if [ "$ROOT_PCT" -lt 90 ]; then
|
|
ok "root filesystem usage ${ROOT_PCT}%"
|
|
else
|
|
warn "root filesystem usage ${ROOT_PCT}% - getting full"
|
|
fi
|
|
|
|
# ---------- SSH ----------
|
|
section "SSH hardening"
|
|
SSHD_T=$(sudo sshd -T 2>/dev/null)
|
|
echo "$SSHD_T" | grep -q '^permitrootlogin no' && ok "PermitRootLogin no" || bad "PermitRootLogin not disabled"
|
|
echo "$SSHD_T" | grep -q '^passwordauthentication no' && ok "PasswordAuthentication no" || bad "PasswordAuthentication not disabled"
|
|
systemctl is-active --quiet ssh && ok "ssh.service active" || bad "ssh.service not active"
|
|
|
|
# ---------- DNS / BIND ----------
|
|
section "DNS / BIND"
|
|
systemctl is-active --quiet named && ok "named active" || bad "named not active"
|
|
|
|
if [ "$(dig @127.0.0.1 jayfield.org +short)" = "89.58.8.149" ]; then
|
|
ok "jayfield.org resolves to the expected IP"
|
|
else
|
|
bad "jayfield.org did not resolve to the expected IP"
|
|
fi
|
|
|
|
DNSSEC_STATUS=$(dig @127.0.0.1 github.com +noall +comment 2>/dev/null | grep -oP 'status: \K[A-Z]+')
|
|
if [ "$DNSSEC_STATUS" = "NOERROR" ]; then
|
|
ok "DNSSEC validation working (github.com -> NOERROR)"
|
|
else
|
|
bad "DNSSEC validation broken (github.com -> ${DNSSEC_STATUS:-no response})"
|
|
fi
|
|
|
|
for h in uschi vpn; do
|
|
IP=$(dig @127.0.0.1 "$h.dyndns.jayfield.org" A +short)
|
|
if [ -n "$IP" ]; then
|
|
ok "$h.dyndns.jayfield.org resolves ($IP)"
|
|
else
|
|
warn "$h.dyndns.jayfield.org has no A record"
|
|
fi
|
|
done
|
|
|
|
# ---------- Mail stack ----------
|
|
section "Mail stack"
|
|
systemctl is-active --quiet postfix@- && ok "postfix active" || bad "postfix not active"
|
|
systemctl is-active --quiet dovecot && ok "dovecot active" || bad "dovecot not active"
|
|
systemctl is-active --quiet rspamd && ok "rspamd active" || bad "rspamd not active"
|
|
|
|
SMTP_BANNER=$(printf 'QUIT\r\n' | timeout 5 nc 127.0.0.1 25 2>/dev/null | head -1)
|
|
echo "$SMTP_BANNER" | grep -q '220.*Postfix' && ok "SMTP banner OK" || bad "SMTP banner missing/unexpected: $SMTP_BANNER"
|
|
|
|
IMAP_BANNER=$(printf 'a LOGOUT\r\n' | timeout 5 nc 127.0.0.1 143 2>/dev/null | head -1)
|
|
echo "$IMAP_BANNER" | grep -qi 'dovecot ready\|^\* OK' && ok "IMAP banner OK" || bad "IMAP banner missing/unexpected: $IMAP_BANNER"
|
|
|
|
for port in 11332 11333 11334; do
|
|
if sudo ss -ltn | grep -q ":$port "; then
|
|
ok "rspamd listening on $port"
|
|
else
|
|
bad "rspamd NOT listening on $port"
|
|
fi
|
|
done
|
|
|
|
RSPAMD_TEST=$(printf 'Subject: sanity test\nFrom: test@example.com\nTo: jens@jayfield.org\n\nbody\n' | rspamc symbols 2>&1)
|
|
if echo "$RSPAMD_TEST" | grep -q 'Action:'; then
|
|
ok "rspamd genuinely scanning (rspamc symbols responded)"
|
|
else
|
|
bad "rspamd did not respond to a live scan test - check for a phantom 'active' status"
|
|
fi
|
|
|
|
if sudo ss -ltn | grep -q ':11332 '; then
|
|
ok "postfix's configured milter target (11332) is listening"
|
|
else
|
|
bad "nothing listening on postfix's milter port - mail is flowing unfiltered/unsigned"
|
|
fi
|
|
|
|
# ---------- MariaDB / vmail ----------
|
|
section "MariaDB / vmail"
|
|
systemctl is-active --quiet mariadb && ok "mariadb active" || bad "mariadb not active"
|
|
|
|
DOMAINS=$(sudo mysql -N -e "SELECT COUNT(*) FROM vmail.domains;" 2>/dev/null)
|
|
ACCOUNTS=$(sudo mysql -N -e "SELECT COUNT(*) FROM vmail.accounts;" 2>/dev/null)
|
|
[ -n "$DOMAINS" ] && [ "$DOMAINS" -ge 1 ] && ok "vmail.domains has $DOMAINS row(s)" || bad "vmail.domains query failed or returned zero rows"
|
|
[ -n "$ACCOUNTS" ] && [ "$ACCOUNTS" -ge 1 ] && ok "vmail.accounts has $ACCOUNTS row(s)" || bad "vmail.accounts query failed or returned zero rows"
|
|
|
|
if sudo postmap -q jayfield.org mysql:/etc/postfix/sql/domains.cf 2>/dev/null | grep -q jayfield.org; then
|
|
ok "Postfix's MySQL domain lookup works"
|
|
else
|
|
bad "Postfix's MySQL domain lookup failed"
|
|
fi
|
|
|
|
# ---------- fail2ban ----------
|
|
section "fail2ban"
|
|
systemctl is-active --quiet fail2ban && ok "fail2ban active" || bad "fail2ban not active"
|
|
|
|
JAIL_COUNT=$(sudo fail2ban-client status 2>/dev/null | grep -oP 'Number of jail:\s*\K[0-9]+')
|
|
if [ -n "$JAIL_COUNT" ] && [ "$JAIL_COUNT" -ge 15 ]; then
|
|
ok "$JAIL_COUNT jails loaded"
|
|
else
|
|
bad "only ${JAIL_COUNT:-0} jail(s) loaded (expected ~19)"
|
|
fi
|
|
|
|
SSHD_FT=$(sudo fail2ban-client get sshd findtime 2>/dev/null)
|
|
DOVECOT_FT=$(sudo fail2ban-client get dovecot findtime 2>/dev/null)
|
|
[ "$SSHD_FT" = "86400" ] && ok "sshd findtime hardening intact (86400s)" || bad "sshd findtime is ${SSHD_FT:-unknown}, expected 86400"
|
|
[ "$DOVECOT_FT" = "86400" ] && ok "dovecot findtime hardening intact (86400s)" || bad "dovecot findtime is ${DOVECOT_FT:-unknown}, expected 86400"
|
|
|
|
# ---------- Apache / vhosts ----------
|
|
section "Apache / vhosts"
|
|
systemctl is-active --quiet apache2 && ok "apache2 active" || bad "apache2 not active"
|
|
sudo apache2ctl configtest >/dev/null 2>&1 && ok "apache2ctl configtest clean" || bad "apache2ctl configtest failed"
|
|
|
|
check_url() {
|
|
local url=$1 expected=$2 code
|
|
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "$url")
|
|
if [ "$code" = "$expected" ]; then
|
|
ok "$url -> $code"
|
|
else
|
|
bad "$url -> $code (expected $expected)"
|
|
fi
|
|
}
|
|
check_url "https://jayfield.org/" 200
|
|
check_url "https://www.jayfield.org/" 200
|
|
check_url "https://dyndns.jayfield.org/" 401
|
|
check_url "https://mail.jayfield.org/" 403
|
|
check_url "https://cloud.jayfield.org/" 302
|
|
check_url "https://portainer.jayfield.org/" 200
|
|
check_url "https://git.jayfield.org/" 200
|
|
|
|
# ---------- Docker ----------
|
|
section "Docker"
|
|
systemctl is-active --quiet docker && ok "docker active" || bad "docker not active"
|
|
|
|
for c in portainer nextcloud nextcloud-redis nextcloud-db web gitea; do
|
|
STATUS=$(sudo docker inspect -f '{{.State.Status}}' "$c" 2>/dev/null)
|
|
if [ "$STATUS" = "running" ]; then
|
|
ok "container $c running"
|
|
else
|
|
bad "container $c is '${STATUS:-missing}' (expected running)"
|
|
fi
|
|
done
|
|
|
|
GITEA_REPO_COUNT=$(curl -s --max-time 5 'https://git.jayfield.org/api/v1/repos/search?limit=1' | python3 -c 'import json,sys; print(len(json.load(sys.stdin).get("data",[])))' 2>/dev/null)
|
|
if [ "$GITEA_REPO_COUNT" = "1" ]; then
|
|
ok "Gitea API returns real repo data (not just a static page)"
|
|
else
|
|
bad "Gitea API did not return expected repo data"
|
|
fi
|
|
|
|
# ---------- Summary ----------
|
|
section "Summary"
|
|
echo "PASS: $PASS WARN: $WARN FAIL: $FAIL"
|
|
echo "Full log: $LOG"
|
|
|
|
if command -v mail >/dev/null 2>&1; then
|
|
SUBJECT="alpha.jayfield.org sanity check: $([ "$FAIL" -gt 0 ] && echo "$FAIL FAILURE(S)" || echo "OK")"
|
|
mail -s "$SUBJECT" jens@jayfield.org < "$LOG"
|
|
fi
|
|
|
|
[ "$FAIL" -eq 0 ]
|