#!/bin/bash
# Mail an alert when a systemd unit fails. Wired up via OnFailure= on the restic
# units. Without this a failed nightly backup is completely silent -- which is how
# you end up trusting backups that have not run for weeks.
#
# Recipient is derived from the hostname (azkaban@canutethegreat.com), matching the
# existing per-host convention and the root alias already present in /etc/aliases.
# Override by setting BACKUP_MAILTO in /etc/default/pve-backup or /etc/default/guest-backup.
set -uo pipefail

UNIT="${1:?usage: unit-failure-mail <unit-name>}"

for _f in /etc/default/pve-backup /etc/default/guest-backup; do [ -r "$_f" ] && . "$_f"; done

HOST_SHORT="$(hostname -s)"
RCPT="${BACKUP_MAILTO:-${HOST_SHORT}@canutethegreat.com}"

{
    echo "systemd unit '${UNIT}' FAILED on ${HOST_SHORT} at $(date -Is)."
    echo
    echo "=== systemctl status ==="
    systemctl status --full --no-pager "$UNIT" 2>&1 | head -30
    echo
    echo "=== last 60 journal lines ==="
    journalctl -u "$UNIT" -n 60 --no-pager 2>&1
    echo
    echo "=== host memory / swap ==="
    free -m
    echo
    echo "=== disk space ==="
    df -h /
    if [ -r /proc/spl/kstat/zfs/arcstats ]; then
        echo
        echo "=== ZFS ARC ==="
        awk '/^(c_max|size)[[:space:]]/ {printf "%-8s %.0f MiB\n", $1, $3/1048576}' /proc/spl/kstat/zfs/arcstats
        echo
        echo "=== pool capacity ==="
        zpool list 2>/dev/null
    fi
} | mail -s "[${HOST_SHORT}] FAILED: ${UNIT}" "$RCPT"
