#!/bin/bash
# Mail an alert when a systemd unit fails. Wired up via OnFailure= on the restic
# units. Without this a failed nightly backup is completely silent -- which is how
# you end up trusting backups that have not run for weeks.
#
# Recipient is derived from the hostname (azkaban@canutethegreat.com), matching the
# existing per-host convention and the root alias already present in /etc/aliases.
# Override by setting BACKUP_MAILTO in /etc/default/pve-backup.
set -uo pipefail

UNIT="${1:?usage: unit-failure-mail <unit-name>}"

[ -r /etc/default/pve-backup ] && . /etc/default/pve-backup

HOST_SHORT="$(hostname -s)"
RCPT="${BACKUP_MAILTO:-${HOST_SHORT}@canutethegreat.com}"

{
    echo "systemd unit '${UNIT}' FAILED on ${HOST_SHORT} at $(date -Is)."
    echo
    echo "=== systemctl status ==="
    systemctl status --full --no-pager "$UNIT" 2>&1 | head -30
    echo
    echo "=== last 60 journal lines ==="
    journalctl -u "$UNIT" -n 60 --no-pager 2>&1
    echo
    echo "=== host memory / swap ==="
    free -m
    echo
    echo "=== ZFS ARC ==="
    awk '/^(c_max|size)[[:space:]]/ {printf "%-8s %.0f MiB\n", $1, $3/1048576}' \
        /proc/spl/kstat/zfs/arcstats 2>/dev/null
    echo
    echo "=== pool capacity ==="
    zpool list 2>/dev/null
} | mail -s "[${HOST_SHORT}] FAILED: ${UNIT}" "$RCPT"
