#!/usr/bin/env bash
#
# omp-conductor fleet recovery — the OnFailure= playbook for the two fleet
# units (omp-conductor.service, herdr-fleet.service).
#
# Why it exists. Both units crash-loop under systemd's default start limit and
# then sit in `failed` indefinitely, in silence: the daemon cannot report its
# own crash loop, and the orchestrator cannot report a session host that is not
# hosting it (#485). Each unit carries OnFailure=omp-conductor-recover.service,
# so the moment one of them is given up on, systemd starts this unit — outside
# the failed unit's cgroup and process tree, so the restarts below can never
# take the recovery down with them.
#
# The playbook is fixed and bounded, because it runs at the moment the system
# is least healthy with no orchestrator alive to review it:
#
#   1. Collect evidence durably, before anything else. journal tail, unit
#      state, `omp-conductor doctor --json`, installed versions, and the
#      pre-upgrade snapshot list land in $RECOVER_DIR so the record survives
#      no matter what happens next.
#   2. Attempt at most ONE known-safe restorative action, chosen mechanically
#      from that evidence:
#        ROLLBACK     — a fresh pre-upgrade config snapshot exists (an upgrade
#                       just landed and the daemon died after it): restore the
#                       durable bytes `upgrade` keeps, then start the daemon
#                       unit once.
#        RECOVER_PANE — herdr-fleet.service failed: start it (reset-failed +
#                       start); its startup hook resumes the fleet pane by
#                       exact identity.
#        RESTART_PEER — the daemon failed with no upgrade evidence: restart
#                       the peer unit (herdr-fleet.service) once. The fleet
#                       units share state; the peer is the only unit the
#                       recovery may touch that cannot re-loop the failure
#                       that already exhausted the failed unit's start limit.
#      Nothing improvised: no diagnosis, no repo edits, no installed-tree
#      writes, no blind restarts of the failed unit. If the evidence points at
#      a code defect, the right output is the escalation — the normal fleet
#      loop repairs it once the fleet is back.
#   3. Escalate tier-2 through the durable outbox (`omp-conductor report
#      --kind tier2`), whether or not the action worked — a fleet that fell
#      over and got itself up is still news.
#   4. Never loop. Consecutive failed attempts are capped at
#      RECOVER_MAX_FAILED_ATTEMPTS via a durable counter (expired after
#      RECOVER_FAIL_WINDOW_S so a genuinely new episode gets fresh attempts),
#      the recovery unit itself carries StartLimitBurst=2 (rendered by
#      setup-host.ts), and this unit never carries OnFailure= — there is no
#      path where recovery re-triggers recovery.
#
# Runs as root (journalctl and systemctl need it). OMP_CONDUCTOR_HOME is set
# by the unit to the fleet's conductor home, so every omp-conductor call
# touches the fleet's own config and store. RECOVER_DRY_RUN=1 prints every
# decision and mutates nothing.

set -uo pipefail

PROG=omp-conductor-recover

# Every external command is overridable so tests can inject stubs; the unit
# renders an explicit PATH (this host's non-interactive PATH lacks `omp`,
# `herdr` and `omp-conductor`).
SYSTEMCTL=${SYSTEMCTL_BIN:-systemctl}
JOURNALCTL=${JOURNALCTL_BIN:-journalctl}
CONDUCTOR=${OMP_CONDUCTOR_BIN:-omp-conductor}
BUN=${BUN_BIN:-bun}

# The fleet's conductor home: where config.json, the sqlite store and the
# upgrade's durable snapshots live. The unit renders OMP_CONDUCTOR_HOME and
# RECOVER_STATE_DIR; a standalone run defaults to ~/.omp/conductor.
STATE_DIR=${RECOVER_STATE_DIR:-${OMP_CONDUCTOR_HOME:-$HOME/.omp/conductor}}
CONFIG_PATH=$STATE_DIR/config.json
BACKUP_ROOT=$STATE_DIR/backups/config
RECOVER_DIR=${RECOVER_DIR:-$STATE_DIR/recovery}

# The two fleet units, daemon first. Tests may narrow this list.
DAEMON_UNIT=${DAEMON_UNIT:-omp-conductor.service}
HERDR_UNIT=${HERDR_UNIT:-herdr-fleet.service}
FLEET_UNITS=${FLEET_UNITS:-$DAEMON_UNIT:$HERDR_UNIT}

# Bounds (see the header). The rendered unit's StartLimitIntervalSec should
# stay in agreement with RECOVER_FAIL_WINDOW_S.
MAX_FAILED_ATTEMPTS=${RECOVER_MAX_FAILED_ATTEMPTS:-2}
FAIL_WINDOW_S=${RECOVER_FAIL_WINDOW_S:-1800}
ROLLBACK_AGE_S=${RECOVER_ROLLBACK_AGE_S:-3600}
JOURNAL_LINES=${RECOVER_JOURNAL_LINES:-200}
VERIFY_POLL_S=${RECOVER_VERIFY_POLL_S:-2}
VERIFY_TRIES=${RECOVER_VERIFY_TRIES:-10}
PROJECT=${RECOVER_PROJECT:-}
# Recovery action scope and durable-report ownership are deliberately separate.
# `RECOVER_PROJECT` is only safe on a single-project host; the report still
# needs one configured project to own its outbox row on a shared host.
REPORT_PROJECT=${RECOVER_REPORT_PROJECT:-$PROJECT}
AFFECTED_PROJECTS=${RECOVER_PROJECTS:-$PROJECT}
REPORT_ROUTING_NOTE=
# The bound on the post-restore re-arm. `arm` with a claim-only proof performs
# no Telegram send or wait, so it completes in milliseconds; the timeout exists
# so a misbehaving conductor can never hang this bounded playbook (#613).
ARM_TIMEOUT_S=${RECOVER_ARM_TIMEOUT_S:-30}

# --------------------------------------------------------------------------
# output
# --------------------------------------------------------------------------

say() { printf '%s: %s\n' "$PROG" "$*"; }
log() { printf '%s: %s\n' "$PROG" "$*" >&2; }

dry_run() { [[ ${RECOVER_DRY_RUN:-0} == 1 ]]; }

stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
epoch_now() { date +%s; }

# --------------------------------------------------------------------------
# systemd access (the only live-host surface; tests inject stubs)
# --------------------------------------------------------------------------

# `is-failed` prints "failed" only for the failed state; an absent unit prints
# "inactive" and exits 3, which is not a failure.
unit_is_failed() { # <unit>
  [[ $("$SYSTEMCTL" is-failed "$1" 2>/dev/null | tr -d '[:space:]') == failed ]]
}

unit_says_failed() { # <unit> — "is-failed" verdict + exit code, for logs
  "$SYSTEMCTL" is-failed "$1" 2>/dev/null | tr -d '[:space:]'
}

# One bounded boot of a unit: clear the start limit, then start. Only ever
# used on the failed unit with fresh evidence behind it (the rollback leg) or
# on herdr-fleet (RECOVER_PANE), never as a blind daemon restart.
start_unit() { # <unit>
  say "systemd: reset-failed $1"
  "$SYSTEMCTL" reset-failed "$1" 2>/dev/null || true
  say "systemd: start $1"
  "$SYSTEMCTL" start "$1"
}

# Wait until the unit leaves the failed state, bounded by the poll window.
# Returns 0 when it did.
wait_no_longer_failed() { # <unit>
  local tries=$VERIFY_TRIES
  while (( tries > 0 )); do
    if ! unit_is_failed "$1"; then
      return 0
    fi
    sleep "$VERIFY_POLL_S"
    tries=$((tries - 1))
  done
  return 1
}

# --------------------------------------------------------------------------
# durable evidence (written before anything else)
# --------------------------------------------------------------------------

# Write the full evidence bundle for one run and print its path. The file is
# the record a later human or the ordinary fleet loop reads (#378 wants it
# provenance-carrying): who, when, what was installed, what the units said,
# what the doctor said, and the journal tail of the failed unit.
write_evidence() { # <failed-unit or empty>
  local unit=${1:-none} states="" snapshots="" doctor="" versions="" journal=""
  local u
  if ! mkdir -p "$RECOVER_DIR" 2>/dev/null; then
    log "cannot create recovery dir $RECOVER_DIR"
    return 2
  fi
  for u in ${FLEET_UNITS//:/ }; do
    states="$states $u=$(unit_says_failed "$u")"
  done
  snapshots=$(ls -t "$BACKUP_ROOT"/config.json.pre-upgrade-* 2>/dev/null | head -n 5 || true)
  doctor=$("$CONDUCTOR" doctor --json 2>&1 | head -c 4000 || true)
  versions=$("$CONDUCTOR" --version 2>&1 | head -n 1 || true)
  if [[ $unit != none ]]; then
    journal=$("$JOURNALCTL" -u "$unit" -n "$JOURNAL_LINES" --no-pager 2>&1 || true)
  fi
  local file="$RECOVER_DIR/evidence-$(stamp)-$unit.txt"
  {
    printf '%s\n' "=== omp-conductor fleet recovery evidence (#485) ==="
    printf 'failed unit:    %s\n' "$unit"
    printf 'triggered at:   %s (epoch %s)\n' "$(stamp)" "$(epoch_now)"
    printf 'recover script: %s\n' "$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")"
    printf 'units:         %s\n' "$states"
    printf 'installed:      %s\n' "$versions"
    printf '%s\n' 'pre-upgrade snapshots (newest first):'
    printf '%s\n' "${snapshots:-  (none)}"
    printf '%s\n' '--- doctor ---'
    printf '%s\n' "$doctor"
    printf '%s\n' '--- journal tail ---'
    printf '%s\n' "$journal"
    printf '%s\n' '--- decision ---'
  } >"$file" 2>/dev/null
  if [[ ! -s $file ]]; then
    log "could not write evidence to $file"
    return 2
  fi
  printf '%s\n' "$file"
}

append_evidence() { # <evidence path> <text>
  printf '%s\n' "$2" >>"$1" 2>/dev/null || true
}


# Resolve the host-global report owner from the live machine-written config.
# Bun is not an extra prerequisite: omp-conductor itself runs under Bun, and
# the recovery unit's canonical PATH already contains that executable. When a
# manual/degraded invocation cannot read the config, the script still attempts
# the legacy unscoped report and records that degradation in the evidence.
resolve_report_scope() {
  if [[ -n $REPORT_PROJECT && -n $AFFECTED_PROJECTS ]]; then
    return 0
  fi
  local lines first all
  if lines=$(
    "$BUN" -e '
      const path = Bun.argv.at(-1);
      const config = await Bun.file(path).json();
      const names = Array.isArray(config.projects)
        ? config.projects.map((project) => project?.name).filter((name) => typeof name === "string" && name.length > 0)
        : [];
      if (names.length === 0) process.exit(2);
      console.log(names[0]);
      console.log(names.join(","));
    ' "$CONFIG_PATH" 2>/dev/null
  ); then
    first=${lines%%$'\n'*}
    all=${lines#*$'\n'}
    REPORT_PROJECT=${REPORT_PROJECT:-$first}
    AFFECTED_PROJECTS=${AFFECTED_PROJECTS:-$all}
  fi
  if [[ -z $REPORT_PROJECT || -z $AFFECTED_PROJECTS ]]; then
    AFFECTED_PROJECTS=${AFFECTED_PROJECTS:-unknown}
    REPORT_ROUTING_NOTE="configured project resolution failed for $CONFIG_PATH; attempting the report without a project selector"
  fi
  return 0
}
# --------------------------------------------------------------------------
# escalation — always, through the durable outbox (#123, #288)
# --------------------------------------------------------------------------

# The single escalation path this repo has for "something must reach a human":
# `omp-conductor report --kind tier2` persists the row in the fleet's store
# (the durable outbox) before anything is sent, and the daemon owns delivery
# with bounded retries once it is back. This script only ever enqueues.
escalation_text() { # <failed-unit or empty> <action> <outcome> <evidence>
  local projects=${AFFECTED_PROJECTS:-unknown}
  if [[ ${1:-none} == none ]]; then
    printf '%s\n' \
      "fleet recovery (#485): a recovery trigger fired but no fleet unit was failing when it ran — the fleet got itself up or was started externally; affected-projects=$projects; evidence=$4"
    return
  fi
  printf '%s\n' \
    "fleet recovery (#485): $1 entered failed state; affected-projects=$projects; recovery=${2:-none}; outcome=${3}; evidence=${4}; next: read the evidence file (and its doctor output) before touching the fleet"
}

escalate() { # <failed-unit or empty> <action-name> <outcome> <evidence-path>
  local unit=${1:-none} text args
  resolve_report_scope
  if [[ -n $REPORT_ROUTING_NOTE ]]; then
    log "$REPORT_ROUTING_NOTE"
    append_evidence "$4" "report routing: $REPORT_ROUTING_NOTE"
  fi
  text=$(escalation_text "$unit" "$2" "$3" "$4")
  args=("$CONDUCTOR" report --kind tier2)
  [[ -n $REPORT_PROJECT ]] && args+=(--project "$REPORT_PROJECT")
  args+=(--text "$text")
  say "escalation: ${args[*]}"
  if dry_run; then
    return 0
  fi
  local out
  out=$("${args[@]}" 2>&1)
  local code=$?
  if (( code != 0 )); then
    log "escalation failed (exit $code): $out"
    return 1
  fi
  printf '%s: escalation: %s\n' "$PROG" "$out" >&2
  return 0
}

# --------------------------------------------------------------------------
# bounds — the durable counter ("a recovery that fails twice stops trying")
# --------------------------------------------------------------------------

MARKER="$RECOVER_DIR/attempts"

read_failed_attempts() {
  local count ts now
  [[ -f $MARKER ]] || { printf '0\n'; return; }
  count=$(sed -n '1p' "$MARKER" 2>/dev/null || true)
  ts=$(sed -n '2p' "$MARKER" 2>/dev/null || true)
  now=$(epoch_now)
  if [[ ! $count =~ ^[0-9]+$ || ! $ts =~ ^[0-9]+$ ]] || (( now - ts > FAIL_WINDOW_S )); then
    printf '0\n'
    return
  fi
  printf '%s\n' "$count"
}

bump_failed_attempts() {
  local next
  next=$(read_failed_attempts)
  mkdir -p "$RECOVER_DIR" 2>/dev/null || true
  printf '%s\n%s\n' "$((next + 1))" "$(epoch_now)" >"$MARKER"
}

clear_failed_attempts() {
  rm -f "$MARKER"
}

# --------------------------------------------------------------------------
# decisions (pure; the test suite drives these directly)
# --------------------------------------------------------------------------

# The newest pre-upgrade config snapshot recent enough to implicate the current
# failure. `upgrade` persists these durably before it touches anything
# (configBackupDir(), config.json.pre-upgrade-*). A fresh one plus a dead
# daemon is the one mechanical "the upgrade did it" signal we are allowed to
# act on — fixing the new version's mistakes is a normal daemon-side repair,
# but reinstalling the previous package pinned to a durable version is not
# something this bounded playbook may improvise.
fresh_rollback_snapshot() {
  local newest ts now
  newest=$(ls -t "$BACKUP_ROOT"/config.json.pre-upgrade-* 2>/dev/null | head -n 1 || true)
  [[ -n $newest ]] || return 1
  now=$(epoch_now)
  ts=$(stat -c %Y "$newest" 2>/dev/null || stat -f %m "$newest" 2>/dev/null || true)
  [[ -n $ts && $((now - ts)) -le $ROLLBACK_AGE_S ]] || return 1
  printf '%s\n' "$newest"
}

# The single action for one failed unit: `ROLLBACK|<snapshot>`,
# `RESTART_PEER|<herdr unit>`, `RECOVER_PANE|<herdr unit>` or `NONE|`.
# Mechanically selected, never improvised.
choose_action() { # <failed-unit>
  local unit=$1 snap
  if [[ $unit == "$DAEMON_UNIT" ]]; then
    if snap=$(fresh_rollback_snapshot); then
      printf 'ROLLBACK|%s\n' "$snap"
      return
    fi
    printf 'RESTART_PEER|%s\n' "$HERDR_UNIT"
    return
  fi
  if [[ $unit == "$HERDR_UNIT" ]]; then
    printf 'RECOVER_PANE|%s\n' "$HERDR_UNIT"
    return
  fi
  printf 'NONE|\n'
}

# --------------------------------------------------------------------------
# re-arm — the one follow-on after a successful restoration (#613)
# --------------------------------------------------------------------------

# Whether the scoped project opted into claim-only arming. The unit renders
# RECOVER_PROJECT only for an unambiguous single-project host (#510/#530), and
# "proof" is a value unique to the project's "arm" block, so two exact-key
# greps decide it on the machine-written config. Any unreadable, hand-formatted
# or legacy shape (no "arm" key at all) reads as challenge — the explicit human
# gate — which is the fail-safe direction: recovery never arms a project that
# did not opt into claim-only.
project_arms_claim_only() { # <config-path>
  local path=${1:-$CONFIG_PATH}
  [[ -f $path ]] || return 1
  grep -qE '"arm"[[:space:]]*:' "$path" || return 1
  grep -qE '"proof"[[:space:]]*:[[:space:]]*"claim-only"' "$path"
}

# Re-arm the scoped project through the ordinary `arm` path, unattended only
# when the project opted into claim-only proof: that proof performs no Telegram
# send or wait, so the command completes headless. A challenge project is never
# poked — its gate is a human answering the challenge — and the caller's
# escalation says so plainly instead. Prints exactly one outcome line (the
# progress line goes to stderr, so the captured return value is the outcome).
rearm_if_claim_only() {
  if [[ -z $PROJECT ]]; then
    printf 'no scoped project (RECOVER_PROJECT unset) — nothing to re-arm\n'
    return 0
  fi
  if ! project_arms_claim_only "$CONFIG_PATH"; then
    printf 'project %s keeps the challenge proof — arming stays an explicit human gate\n' "$PROJECT"
    return 0
  fi
  log "re-arming $PROJECT via the ordinary arm path (claim-only proof)"
  if dry_run; then
    printf 'dry run: would run %s arm --project %s (claim-only, bounded by %ss)\n' "$CONDUCTOR" "$PROJECT" "$ARM_TIMEOUT_S"
    return 0
  fi
  local out code
  out=$(timeout "$ARM_TIMEOUT_S" "$CONDUCTOR" arm --project "$PROJECT" 2>&1)
  code=$?
  if (( code == 0 )); then
    printf 're-armed %s via the claim-only proof\n' "$PROJECT"
    return 0
  fi
  printf 're-arm of %s failed (exit %s): %s\n' "$PROJECT" "$code" "$out"
  return 0
}

# --------------------------------------------------------------------------
# main
# --------------------------------------------------------------------------

run_recovery() {
  local failed="" u action_name="" action_arg="" ev="" outcome="" ok=1
  for u in ${FLEET_UNITS//:/ }; do
    if unit_is_failed "$u"; then
      failed=$u
      break
    fi
  done

  if dry_run; then
    resolve_report_scope
    say "plan: report owner — ${REPORT_PROJECT:-unscoped}; affected projects — $AFFECTED_PROJECTS"
    [[ -n $REPORT_ROUTING_NOTE ]] && say "plan: report routing degraded — $REPORT_ROUTING_NOTE"
    say "dry run: nothing will be written or executed"
    [[ -z $failed ]] && say "plan: no unit failed — escalate only"
    if [[ -n $failed ]]; then
      IFS='|' read -r action_name action_arg <<<"$(choose_action "$failed")"
      say "plan: failed unit $failed; action: $action_name"
    fi
    if [[ -z $PROJECT ]]; then
      say "plan: re-arm — none (RECOVER_PROJECT unset)"
    elif project_arms_claim_only "$CONFIG_PATH"; then
      say "plan: re-arm — $CONDUCTOR arm --project $PROJECT (claim-only proof, bounded by ${ARM_TIMEOUT_S}s)"
    else
      say "plan: re-arm — none ($PROJECT keeps the challenge proof; arming is an explicit human gate)"
    fi
    say "plan: escalation — $(escalation_text "$failed" "${action_name:-none}" "dry run" "DRY-RUN")"
    return 0
  fi

  ev=$(write_evidence "$failed")
  if [[ -z $ev ]]; then
    log "could not write evidence under $RECOVER_DIR — treating the recovery as failed"
    return 1
  fi

  local attempts
  if [[ -z $failed ]]; then
    # Nothing failed when recovery ran (the fleet got itself up, or an
    # external start beat this activation): always escalate, and a previously
    # exhausted episode counter gets a fresh start.
    say "no unit failed when this recovery ran — the fleet got itself up (or was started externally); nothing to attempt"
    append_evidence "$ev" "decision: NONE — no unit failed"
    if escalate none none "no unit failed when the recovery ran" "$ev"; then
      clear_failed_attempts
      return 0
    fi
    return 1
  fi

  attempts=$(read_failed_attempts)
  if [[ $attempts -ge $MAX_FAILED_ATTEMPTS ]]; then
    say "attempts exhausted: $attempts consecutive failed recoveries inside the ${FAIL_WINDOW_S}s window; escalating only"
    append_evidence "$ev" "decision: NONE (attempts exhausted, $attempts)"
    if escalate "$failed" none "attempts exhausted ($attempts); no further recovery" "$ev"; then
      return 0
    fi
    return 1
  fi

  say "failed unit: $failed"
  IFS='|' read -r action_name action_arg <<<"$(choose_action "$failed")"
  action_name=${action_name:-NONE}
  append_evidence "$ev" "decision: $action_name"
  if [[ $action_name == NONE ]]; then
    say "no safe mechanical action for $failed; escalating"
    if escalate "$failed" none "no known-safe action" "$ev"; then
      return 0
    fi
    return 1
  fi

  # Exactly one action, no matter how many units are down.
  case "$action_name" in
    ROLLBACK)
      if cp "$action_arg" "$CONFIG_PATH" && start_unit "$failed"; then
        if wait_no_longer_failed "$failed"; then
          ok=0
          outcome="restored pre-upgrade snapshot ${action_arg##*/} and started $failed"
        else
          outcome="restored pre-upgrade snapshot and started $failed, but it is still failing"
        fi
      else
        outcome="could not restore snapshot ${action_arg##*/} or start $failed"
      fi
      ;;
    RECOVER_PANE)
      if start_unit "$HERDR_UNIT"; then
        if wait_no_longer_failed "$HERDR_UNIT"; then
          ok=0
          outcome="restarted session host $HERDR_UNIT; its startup hook recovers the fleet pane"
        else
          outcome="$HERDR_UNIT started but is still failing"
        fi
      else
        outcome="could not start $HERDR_UNIT"
      fi
      ;;
    RESTART_PEER)
      if "$SYSTEMCTL" restart "$action_arg"; then
        outcome="restarted peer unit $action_arg"
      else
        outcome="could not restart peer unit $action_arg"
      fi
      ;;
  esac
  say "outcome: $outcome"
  append_evidence "$ev" "outcome: $outcome"

  if (( ok == 0 )); then
    append_evidence "$ev" "verify: $failed no longer failed"
    say "recovery succeeded: $failed no longer failed"
    clear_failed_attempts
  else
    append_evidence "$ev" "verify: $failed still failed"
    say "recovery failed: $failed still failed; attempts remaining until exhausted"
    bump_failed_attempts
  fi

  # The one follow-on a verified restoration earns (#613): re-arm the scoped
  # project through the ordinary `arm` command. Claim-only completes headless;
  # a challenge project stays an explicit human gate and the escalation below
  # says so plainly. Only after `verify` passed — writing an arm marker for a
  # fleet that is still down is the improvisation this playbook never does.
  local rearm_line="not attempted (recovery did not verify the fleet restored)"
  if (( ok == 0 )); then
    rearm_line=$(rearm_if_claim_only)
  fi
  say "re-arm: $rearm_line"
  append_evidence "$ev" "re-arm: $rearm_line"
  outcome="$outcome; re-arm: $rearm_line"

  if escalate "$failed" "$action_name" "$outcome" "$ev"; then
    if (( ok == 0 )); then
      return 0
    fi
    return 1
  fi
  return 1
}

main() {
  dry_run && say "dry run: printing decisions, mutating nothing"
  if run_recovery; then
    exit 0
  fi
  exit 1
}

if [[ ${BASH_SOURCE[0]} == "$0" ]]; then
  main
fi