#!/bin/bash
# tmpool-recover: bring tmpool back after its USB enclosure dropped off the bus and re-enumerated.
# When the RTL9210 disconnects, ZFS (failmode=wait) SUSPENDS the pool; it stays suspended after the disk
# is back until someone runs `zpool clear` (2026-11-09: pre.sh hung on the suspended pool, host IO
# pressure stuck ~60%). udev runs us on (re)appearance and a 2-minute timer is the safety net, because
# the first clear can race the re-enumeration (the pool resumed and re-suspended within 1 s at 05:22:59).
# No-op (silent, instant) while the pool is not SUSPENDED. Gate before every clear: the disk must answer
# a direct read, host IO pressure must not be critical. Retries with backoff, then gives up (the hourly
# tmpool-health probe keeps flagging it). Overridable for tests: TMPOOL_* env vars.
POOL=${TMPOOL_POOL:-tmpool}
DEV=${TMPOOL_DEV:-/dev/disk/by-id/ata-PC811_SED_SK_hynix_2048GB_AME6N008111706O5D-part1}
CT=${TMPOOL_CT:-109}
TRIES=${TMPOOL_TRIES:-6}
BACKOFF=${TMPOOL_BACKOFF:-10}
PSI=${TMPOOL_PSI:-/proc/pressure/io}
LOCK=${TMPOOL_LOCK:-/run/tmpool-recover.lock}
exec 9>"$LOCK"; flock -n 9 || exit 0
tag() { logger -t tmpool-recover "$*"; echo "$*"; }
health() { timeout 20 zpool list -H -o health "$POOL" 2>/dev/null; }
[ "$(health)" = SUSPENDED ] || exit 0
tag "pool $POOL is SUSPENDED, starting recovery"
i=0
while [ $i -lt "$TRIES" ]; do
	i=$((i+1))
	if [ ! -e "$DEV" ]; then tag "attempt $i: $DEV absent (enclosure still gone)"; sleep "$BACKOFF"; continue; fi
	# the disk must really answer (not just have a node): 1 MiB direct read, bounded
	if ! timeout 30 dd if="$DEV" of=/dev/null bs=1M count=1 iflag=direct 2>/dev/null; then
		tag "attempt $i: $DEV does not answer reads yet"; sleep "$BACKOFF"; continue
	fi
	full=$(awk '/^full/{split($2,a,"=");print int(a[2])}' "$PSI" 2>/dev/null); full=${full:-0}
	if [ "$full" -ge 25 ] && [ "$i" -lt "$TRIES" ]; then
		# critical pressure is mostly the suspended pool's own hung writers; the final try clears regardless
		tag "attempt $i: io pressure full avg10=$full, waiting"; sleep "$BACKOFF"; continue
	fi
	timeout 60 zpool online "$POOL" "$DEV" >/dev/null 2>&1
	timeout 120 zpool clear "$POOL" >/dev/null 2>&1
	sleep 3
	h=$(health)
	if [ "$h" = ONLINE ] || [ "$h" = DEGRADED ]; then
		tag "recovered $POOL ($h) on attempt $i; $(timeout 20 zpool status "$POOL" 2>/dev/null | grep -E 'errors:' | head -1)"
		timeout 60 zfs mount -a 2>/dev/null
		if command -v pct >/dev/null 2>&1 && [ "$(pct status "$CT" 2>/dev/null)" = "status: stopped" ]; then
			pct start "$CT" && tag "started CT $CT"
		fi
		exit 0
	fi
	tag "attempt $i: pool still ${h:-unknown}"; sleep "$BACKOFF"
done
tag "GAVE UP: $POOL still not recovered after $TRIES attempts (check the enclosure/cable; tmpool-health flags it)"
exit 1
