#!/usr/bin/env bash
set -euo pipefail

# Liveness watchdog for the resident Pushy processes.
#
# systemd only knows whether the main PID still exists. A single-threaded
# runtime whose event loop has stopped turning — memory-pressure throttling in
# the cgroup, a blocking syscall, a native stall — keeps its PID and stays
# `active (running)` while answering nothing, and it will not even run its
# SIGTERM handler. This probes the per-instance diagnostic socket, which is
# served from that same event loop, so a reply proves the loop is alive.
#
# After `threshold` consecutive failed probes the unit is restarted. systemd
# escalates to SIGKILL once TimeoutStopSec expires, so a wedged process is
# recovered even though it cannot participate in its own shutdown.

curl_bin=${WATCHDOG_CURL_BIN:-curl}
systemctl_bin=${WATCHDOG_SYSTEMCTL_BIN:-systemctl}
state_dir=${WATCHDOG_STATE_DIR:-/run/pushy-watchdog}
uptime_file=${WATCHDOG_UPTIME_FILE:-/proc/uptime}
probe_timeout_seconds=${WATCHDOG_PROBE_TIMEOUT_SECONDS:-5}
failure_threshold=${WATCHDOG_FAILURE_THRESHOLD:-3}
# Long enough to cover a rollout restart plus first-request warmup, so the
# watchdog never races roll-api.sh or judges a still-booting process.
grace_seconds=${WATCHDOG_GRACE_SECONDS:-90}
# A unit that keeps wedging must not be restarted in a tight loop; the interval
# leaves room for the previous restart to settle and for a human to look.
restart_cooldown_seconds=${WATCHDOG_RESTART_COOLDOWN_SECONDS:-300}
targets=${WATCHDOG_TARGETS:-pushy-api.service=/run/pushy/metrics.sock pushy-worker.service=/run/pushy-worker/metrics.sock}

for name in probe_timeout_seconds failure_threshold grace_seconds restart_cooldown_seconds; do
  if [[ ! ${!name} =~ ^[0-9]+$ ]]; then
    echo "$name must be a non-negative integer" >&2
    exit 2
  fi
done
if ((failure_threshold < 1)); then
  echo "failure_threshold must be at least 1" >&2
  exit 2
fi

install -d -m 0750 "$state_dir"

monotonic_seconds() {
  awk '{ printf "%d", $1; exit }' "$uptime_file" 2>/dev/null || echo 0
}

unit_active_seconds() {
  local unit=$1
  local entered_usec
  entered_usec=$("$systemctl_bin" show -p ActiveEnterTimestampMonotonic --value "$unit" 2>/dev/null || echo 0)
  [[ $entered_usec =~ ^[0-9]+$ ]] || entered_usec=0
  if ((entered_usec == 0)); then
    # Unknown start time: treat as freshly started rather than risk restarting
    # a unit the watchdog cannot reason about.
    echo 0
    return
  fi
  echo $(($(monotonic_seconds) - entered_usec / 1000000))
}

read_counter() {
  local path=$1
  local value
  value=$(cat "$path" 2>/dev/null || echo 0)
  [[ $value =~ ^[0-9]+$ ]] || value=0
  echo "$value"
}

probe() {
  local socket_path=$1
  local response
  response=$("$curl_bin" --fail --silent --show-error \
    --max-time "$probe_timeout_seconds" \
    --unix-socket "$socket_path" http://localhost/health 2>/dev/null) || return 1
  [[ $response == *'"ok":true'* ]]
}

# Word splitting on $targets is intentional: it is a space-separated list.
for target in $targets; do
  unit=${target%%=*}
  socket_path=${target#*=}
  failure_file="$state_dir/${unit}.failures"
  restart_file="$state_dir/${unit}.last-restart"

  if ! "$systemctl_bin" is-active --quiet "$unit"; then
    rm -f "$failure_file"
    continue
  fi

  active_seconds=$(unit_active_seconds "$unit")
  if ((active_seconds < grace_seconds)); then
    rm -f "$failure_file"
    printf 'watchdog unit=%s status=warming active_seconds=%s\n' "$unit" "$active_seconds"
    continue
  fi

  if probe "$socket_path"; then
    if [[ -e $failure_file ]]; then
      printf 'watchdog unit=%s status=recovered\n' "$unit"
      rm -f "$failure_file"
    fi
    continue
  fi

  failures=$(($(read_counter "$failure_file") + 1))
  printf '%s\n' "$failures" >"$failure_file"
  printf 'watchdog unit=%s status=unresponsive failures=%s threshold=%s\n' \
    "$unit" "$failures" "$failure_threshold" >&2

  ((failures >= failure_threshold)) || continue

  now=$(monotonic_seconds)
  last_restart=$(read_counter "$restart_file")
  if ((last_restart > 0 && now - last_restart < restart_cooldown_seconds)); then
    printf 'watchdog unit=%s status=cooldown since_restart_seconds=%s\n' \
      "$unit" "$((now - last_restart))" >&2
    continue
  fi

  printf '%s\n' "$now" >"$restart_file"
  rm -f "$failure_file"
  printf 'watchdog unit=%s status=restarting after=%s failed probes\n' \
    "$unit" "$failures" >&2
  if "$systemctl_bin" restart "$unit"; then
    printf 'watchdog unit=%s status=restarted\n' "$unit" >&2
  else
    printf 'watchdog unit=%s status=restart-failed\n' "$unit" >&2
  fi
done

# Always succeed: an unresponsive service is reported through the journal and
# handled by the restart above. Failing the oneshot as well would leave the unit
# permanently `failed` and mark the whole system degraded for no added signal.
exit 0