#!/usr/bin/env bash
#
# omp-conductor fleet recovery — offline tests for omp/systemd/omp-conductor-recover.sh
#
# No network, no real systemctl/journalctl/omp-conductor, and never a touch on
# this host's units. Every external call goes through stub binaries on a
# fixture PATH; a fixture state dir drives the unit state machine, so what is
# under test is the playbook's decision: evidence written before anything,
# exactly one bounded action from the fixed set, always an escalation through
# the outbox, and no loop.
#
# Prints one line per case and exits non-zero if any of them fails.

set -uo pipefail

here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
recover="$here/omp-conductor-recover.sh"

failures=0
pass() { printf 'ok    %s\n' "$1"; }
fail() {
  printf 'FAIL  %s\n' "$1"
  failures=$((failures + 1))
}

check() { # <name> <expected substring> <actual>
  if [[ $3 == *"$2"* ]]; then
    pass "$1"
  else
    fail "$1"
    printf '      expected substring: %s\n' "$2"
    printf '      actual:            %s\n' "$3"
  fi
}

# --------------------------------------------------------------------------
# syntax
# --------------------------------------------------------------------------

syntax_errors=$(
  bash -n "$recover" 2>&1
  bash -n "$here/recover-unit-test.sh" 2>&1
)
if [[ -z $syntax_errors ]]; then
  pass 'syntax: recovery playbook and test suite'
else
  fail 'syntax: recovery playbook and test suite'
  printf '%s\n' "$syntax_errors"
fi

tmp=$(mktemp -d "${TMPDIR:-/tmp}/omp-recover-test.XXXXXX")
trap 'rm -rf "$tmp"' EXIT
bun_bin=$(command -v bun)
mkdir -p "$tmp/bin"

# --------------------------------------------------------------------------
# fixture stubs. The systemctl stub reads per-unit state files so tests can
# decide which unit is failed and whether `start` heals it; every call is
# logged so tests pin call ORDER (evidence before action) and the single
# action rule.
# --------------------------------------------------------------------------

cat >"$tmp/bin/systemctl" <<'STUB'
#!/usr/bin/env bash
echo "systemctl $*" >>"$STUB_CALLS"
unit="$2"
state_path="$STUB_UNITS/$unit"
state=$(cat "$state_path" 2>/dev/null || echo absent)
case "$1" in
  is-failed)
    [[ $state == failed ]] && echo failed || echo inactive ;;
  is-active)
    if [[ $state == failed ]]; then echo failed; elif [[ $state == absent ]]; then echo inactive; else echo active; fi ;;
  reset-failed) : ;;
  start)
    [[ $unit == "$STUB_HEAL_ON_START" ]] && echo active > "$state_path" ;;
  restart)
    echo active > "$state_path" ;;
  *)
    echo "unexpected systemctl $*" >&2 ;;
esac
STUB

cat >"$tmp/bin/journalctl" <<'STUB'
#!/usr/bin/env bash
echo "journalctl $*" >>"$STUB_CALLS"
echo "journal: fleet crashed here"
echo "journal: last line before death"
STUB

cat >"$tmp/bin/omp-conductor" <<'STUB'
#!/usr/bin/env bash
echo "omp-conductor $*" >>"$STUB_CALLS"
case "$1" in
  doctor)
    printf '%s\n' '{"project":"demo","status":"fail","findings":[{"id":"systemd-unit","status":"fail","summary":"installed differs"}]}'
    ;;
  --version)
    echo "0.15.11"
    ;;
  arm)
    echo "armed demo via fixture"
    ;;
  report)
    echo "report 485-abc queued for demo (tier2)"
    ;;
  *)
    exit 2 ;;
esac
STUB

chmod +x "$tmp/bin/"*

# --------------------------------------------------------------------------
# fixture helpers
# --------------------------------------------------------------------------

newcase() { # <name> -> case dir
  local d="$tmp/case-$1"
  mkdir -p "$d/state/backups/config" "$d/units"
  printf '%s\n' "$d"
}

unit_state() { # <case-dir> <unit> <state|absent>
  if [[ $3 == absent ]]; then
    rm -f "$1/units/$2"
  else
    printf '%s\n' "$3" >"$1/units/$2"
  fi
}

# A pre-upgrade snapshot `upgrade` durably keeps (configBackupDir()):
# config.json.pre-upgrade-<ts>. The test passes the age in seconds.
write_snapshot() { # <case-dir> <age-seconds>
  local d="$1" age="${2:-0}" now
  now=$(date +%s)
  local snap="$d/state/backups/config/config.json.pre-upgrade-$now"
  printf '%s\n' '{"preUpgrade":true}' >"$snap"
  if (( age > 0 )); then
    perl -e 'utime $ARGV[1], $ARGV[1], $ARGV[0] or die "utime: $!"' "$snap" "$(( now - age ))"
  fi
  printf '%s\n' "$snap"
}

marker() { # <case-dir> -> attempts count or 0
  local f="$1/state/recovery/attempts"
  if [[ -f $f ]]; then
    sed -n '1p' "$f"
  else
    echo 0
  fi
}

# The minimal machine-written config shape the re-arm decision greps: the arm
# block with the chosen proof, as `saveConfig` would write it.
write_project_config() { # <case-dir> <proof>
  printf '{\n  "version": 2,\n  "projects": [\n    { "name": "demo", "arm": { "proof": "%s" } }\n  ]\n}\n' "$2" >"$1/state/config.json"
}

write_multi_project_config() { # <case-dir>
  cat >"$1/state/config.json" <<'JSON'
{
  "version": 2,
  "projects": [
    { "name": "alpha" },
    { "name": "beta" }
  ]
}
JSON
}

run_recover() { # <case-dir> [env assignments…]
  local d="$1"
  shift
  (
    cd "$here" || exit 1
    env \
      PATH="$tmp/bin:/usr/bin:/bin" \
      SYSTEMCTL_BIN="$tmp/bin/systemctl" \
      JOURNALCTL_BIN="$tmp/bin/journalctl" \
      OMP_CONDUCTOR_BIN="$tmp/bin/omp-conductor" \
      RECOVER_STATE_DIR="$d/state" \
      RECOVER_PROJECT=demo \
      RECOVER_VERIFY_POLL_S=0 \
      RECOVER_VERIFY_TRIES=3 \
      STUB_CALLS="$d/calls" \
      STUB_UNITS="$d/units" \
      "$@" \
      bash "$recover" >"$d/out" 2>"$d/stderr"
  )
  echo $?
}

# `systemctl restart` never heals the FAILED unit in this fixture (it only
# marks the named unit active, which is what a real peer restart does to a
# healthy peer), so tests wanting daemon-side healing pass HEAL_ON_START=<unit>
# and the stub's `start` flips that unit back to active.
run_with_heal() { # <case-dir> <heal-on-start> [env assignments…]
  local d="$1" heal="$2"
  shift 2
  (
    cd "$here" || exit 1
    env \
      PATH="$tmp/bin:/usr/bin:/bin" \
      SYSTEMCTL_BIN="$tmp/bin/systemctl" \
      JOURNALCTL_BIN="$tmp/bin/journalctl" \
      OMP_CONDUCTOR_BIN="$tmp/bin/omp-conductor" \
      RECOVER_STATE_DIR="$d/state" \
      RECOVER_PROJECT=demo \
      RECOVER_VERIFY_POLL_S=0 \
      RECOVER_VERIFY_TRIES=3 \
      STUB_CALLS="$d/calls" \
      STUB_UNITS="$d/units" \
      STUB_HEAL_ON_START="$heal" \
      "$@" \
      bash "$recover" >"$d/out" 2>"$d/stderr"
  )
  echo $?
}

# --------------------------------------------------------------------------
# the decision functions, driven directly. The unit text holds the other half
# of the bounds (StartLimitBurst=2, no OnFailure=) — setup-host.test.ts pins it.
# --------------------------------------------------------------------------

SCAFFOLD=$(newcase decisions)
FLEET_UNITS=herdr-fleet.service
RECOVER_STATE_DIR="$SCAFFOLD/state"
. "$recover"

# RECOVER_PANE is the only action for a failed herdr-fleet unit: herdr's own
# startup hook recovers the fleet pane, so the pane leg is `start herdr`.
check 'herdr failure chooses RECOVER_PANE' \
  'RECOVER_PANE|herdr-fleet.service' "$(choose_action herdr-fleet.service)"

D2=$(newcase decisions-daemon)
RECOVER_STATE_DIR="$D2/state"
BACKUP_ROOT="$D2/state/backups/config"

# The daemon's action hinges on a fresh pre-upgrade snapshot: with one, the
# mechanical conclusion is "the upgrade did it" and the action is ROLLBACK.
write_snapshot "$D2" 0 >/dev/null
check 'a fresh pre-upgrade snapshot chooses ROLLBACK' \
  "ROLLBACK|$BACKUP_ROOT/config.json.pre-upgrade-" "$(choose_action omp-conductor.service)"

# Without one, the only safe motion is restarting the peer unit.
rm -f "$BACKUP_ROOT"/config.json.pre-upgrade-*
check 'no snapshot chooses RESTART_PEER' \
  'RESTART_PEER|herdr-fleet.service' "$(choose_action omp-conductor.service)"

# A snapshot too old to implicate THIS failure is not evidence.
write_snapshot "$D2" 7200 >/dev/null
check 'a stale snapshot does not choose ROLLBACK' \
  'RESTART_PEER|herdr-fleet.service' "$(choose_action omp-conductor.service)"
unset FLEET_UNITS RECOVER_STATE_DIR BACKUP_ROOT

# ---------------------------------------------------------------------------
# case A — RECOVER_PANE: herdr-fleet failed and the start heals it in time.
# Evidence must be written before a single systemd mutation is attempted.
# ---------------------------------------------------------------------------

d=$(newcase recover-pane)
unit_state "$d" herdr-fleet.service failed
unit_state "$d" omp-conductor.service active
code=$(run_with_heal "$d" herdr-fleet.service)
check 'RECOVER_PANE exits 0 when the unit heals' '0' "$code"
check 'RECOVER_PANE reports success and names the pane hook' \
  'startup hook recovers the fleet pane' "$(grep 'outcome:' "$d/out")"

# Order the acceptance pins: journal + doctor + version (evidence) strictly
# before the first systemd mutation (reset-failed/start).
journal_line=$(grep -n '^journalctl' "$d/calls" | head -n 1 | cut -d: -f1 | tr -d ' ')
reset_line=$(grep -n '^systemctl reset-failed' "$d/calls" | head -n 1 | cut -d: -f1 | tr -d ' ')
if [[ -n $journal_line && -n $reset_line && $journal_line -lt $reset_line ]]; then
  pass 'evidence collection precedes the single action'
else
  fail "evidence collection precedes the single action (journal=$journal_line reset=$reset_line)"
fi

# Exactly one restorative action: one reset-failed + one start, no restart.
starts=$(grep -c '^systemctl start ' "$d/calls" || true)
resets=$(grep -c '^systemctl reset-failed ' "$d/calls" || true)
restarts=$(grep -c '^systemctl restart ' "$d/calls" || true)
if [[ $starts == 1 && $resets == 1 && $restarts == 0 ]]; then
  pass 'exactly one restorative action (start herdr-fleet)'
else
  fail "exactly one restorative action — starts=$starts resets=$resets restarts=$restarts"
fi

check 'successful recovery still escalates tier-2 through the outbox' \
  'omp-conductor report --kind tier2' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
check 'the escalation names the durable evidence file' \
  "$d/state/recovery/evidence-" "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
# The evidence bundle itself carries the collected facts.
ev=$(ls "$d/state/recovery"/evidence-* | head -n 1)
check 'the evidence bundle holds unit state, doctor JSON, versions and journal' \
  '=== omp-conductor fleet recovery evidence' "$(head -n 1 "$ev")"
check 'the evidence bundle names the failed unit' \
  'herdr-fleet.service' "$(grep '^failed unit:' "$ev")"
check 'the evidence bundle carries the doctor output' \
  '"systemd-unit"' "$(grep 'installed differs' "$ev")"
check 'the evidence bundle carries the journal tail' \
  'journal: last line before death' "$(grep 'journal:' "$ev")"

[[ $(marker "$d") == 0 ]]
pass 'the attempts counter is cleared after a successful recovery'

# ---------------------------------------------------------------------------
# case ROLLBACK: daemon failed AND a fresh pre-upgrade snapshot exists.
# ---------------------------------------------------------------------------

d=$(newcase rollback)
write_snapshot "$d" 5 >/dev/null
printf '%s\n' '{"live":true}' >"$d/state/config.json"
unit_state "$d" omp-conductor.service failed
unit_state "$d" herdr-fleet.service active
code=$(run_with_heal "$d" omp-conductor.service)
check 'ROLLBACK exits 0 when the daemon heals' '0' "$code"
check 'ROLLBACK restores the pre-upgrade bytes over config.json' \
  '{"preUpgrade":true}' "$(cat "$d/state/config.json")"
check 'ROLLBACK starts the daemon unit' \
  'systemctl start omp-conductor.service' "$(grep '^systemctl start ' "$d/calls" | head -n 1)"
check 'ROLLBACK escalates as well' \
  'omp-conductor report --kind tier2' "$(grep '^omp-conductor report' "$d/calls" | tail -n 1)"
check 'the pre-upgrade snapshot survives the rollback for the operator' \
  '{"preUpgrade":true}' "$(cat "$d/state/backups/config"/config.json.pre-upgrade-*)"

# ---------------------------------------------------------------------------
# case — the post-restore re-arm (#613): a verified restoration re-arms a
# claim-only project through the ordinary arm command; a challenge project is
# never poked and the escalation says the human gate is explicit.
# ---------------------------------------------------------------------------

d=$(newcase rearm-claim-only)
write_project_config "$d" claim-only
unit_state "$d" herdr-fleet.service failed
unit_state "$d" omp-conductor.service active
code=$(run_with_heal "$d" herdr-fleet.service)
check 'claim-only re-arm exits 0 when the unit heals' '0' "$code"
check 'claim-only recovery re-arms via the ordinary arm path' \
  'omp-conductor arm --project demo' "$(grep '^omp-conductor arm' "$d/calls")"
check 'the escalation names the claim-only re-arm' \
  're-armed demo via the claim-only proof' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
check 'the evidence bundle records the re-arm' \
  're-arm: re-armed demo via the claim-only proof' "$(cat "$d/state/recovery"/evidence-*)"

d=$(newcase rearm-challenge)
write_project_config "$d" challenge
unit_state "$d" herdr-fleet.service failed
unit_state "$d" omp-conductor.service active
code=$(run_with_heal "$d" herdr-fleet.service)
check 'a challenge project exits 0 when the unit heals' '0' "$code"
check 'a challenge project is never poked with arm' \
  '' "$(grep '^omp-conductor arm' "$d/calls" || true)"
check 'the escalation says the challenge gate is explicit' \
  'arming stays an explicit human gate' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"

# A project whose config predates the key reads as challenge: no "arm" block.
d=$(newcase rearm-legacy-config)
unit_state "$d" herdr-fleet.service failed
unit_state "$d" omp-conductor.service active
code=$(run_with_heal "$d" herdr-fleet.service)
check 'a legacy config without arm.proof is never poked with arm' \
  '' "$(grep '^omp-conductor arm' "$d/calls" || true)"
check 'a legacy config keeps the human gate in the escalation' \
  'arming stays an explicit human gate' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"

# RESTART_PEER never verifies a restoration, so even a claim-only project must
# not be re-armed while the daemon is still down.
d=$(newcase rearm-unverified)
write_project_config "$d" claim-only
unit_state "$d" omp-conductor.service failed
unit_state "$d" herdr-fleet.service active
code=$(run_recover "$d")
check 'an unverified recovery never re-arms' \
  're-arm: not attempted' "$(grep 're-arm:' "$d/out" | head -n 1)"
check 'an unverified recovery calls no arm' \
  '' "$(grep '^omp-conductor arm' "$d/calls" || true)"

# ---------------------------------------------------------------------------
# case RESTART_PEER and the bound: attempts cap at two, then escalate-only.
# A daemon crash with no upgrade evidence must never get a blind daemon start.
# ---------------------------------------------------------------------------

d=$(newcase restart-peer)
unit_state "$d" omp-conductor.service failed
unit_state "$d" herdr-fleet.service active

code=$(run_recover "$d")
check 'RESTART_PEER restarts exactly the peer unit and nothing else' \
  'systemctl restart herdr-fleet.service' "$(cat "$d/calls")"
check 'RESTART_PEER never starts the failed daemon' \
  '' "$(grep '^systemctl start omp-conductor' "$d/calls" || true)"
check 'a peer restart that leaves the daemon failed exits 1' '1' "$code"
check 'the failed attempt is recorded' '1' "$(marker "$d")"

# A second trigger in the same episode is one more attempt, not a loop…
code=$(run_recover "$d")
check 'a second attempt is allowed below the cap and also exits 1' '1' "$code"
check 'two failed attempts are recorded' '2' "$(marker "$d")"

# …and the third trigger only escalates: no mutating systemd call at all
# (is-failed probes for evidence are reads, not actions).
mutating() { grep -c '^systemctl \(start\|restart\|reset-failed\) ' "$1" || true; }
before=$(mutating "$d/calls")
code=$(run_recover "$d")
after=$(mutating "$d/calls")
check 'attempts exhausted — the run says so' \
  'attempts exhausted' "$(cat "$d/out")"
check 'an exhausted run performs no systemd action' \
  "$before" "$after"
check 'the escalation is still queued after exhaustion' \
  'omp-conductor report --kind tier2' "$(grep '^omp-conductor report' "$d/calls" | tail -n 1)"

# ---------------------------------------------------------------------------
# case — nothing failed when recovery ran (the "got itself up" case): always
# escalate, and a previously exhausted episode counter gets a fresh start.
# ---------------------------------------------------------------------------

d=$(newcase nothing-failed)
mkdir -p "$d/state/recovery"
printf '3\n%s\n' "$(date +%s)" >"$d/state/recovery/attempts"
unit_state "$d" omp-conductor.service active
unit_state "$d" herdr-fleet.service active
code=$(run_recover "$d")
check 'a trigger with nothing failed escalates (fleet came back)' \
  'no unit failed' "$(cat "$d/out")"
check 'a trigger with nothing failed exits 0' '0' "$code"
check 'a trigger with nothing failed clears the old counter' '0' "$(marker "$d")"

# ---------------------------------------------------------------------------
# case — a host-global recovery on a multi-project config still has one
# deterministic outbox owner, while its body names every affected project.
# ---------------------------------------------------------------------------

d=$(newcase multi-project-report)
write_multi_project_config "$d"
unit_state "$d" omp-conductor.service active
unit_state "$d" herdr-fleet.service active
code=$(run_recover "$d" \
  RECOVER_PROJECT= \
  RECOVER_REPORT_PROJECT= \
  RECOVER_PROJECTS= \
  BUN_BIN="$bun_bin")
report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
check 'multi-project recovery exits after queuing its report' '0' "$code"
check 'multi-project recovery uses the first configured outbox owner' \
  '--project alpha' "$report_call"
check 'multi-project recovery names every affected project' \
  'affected-projects=alpha,beta' "$report_call"
check 'multi-project recovery queues exactly one report' \
  '1' "$(grep -c '^omp-conductor report' "$d/calls")"

d=$(newcase multi-project-recovery-action)
write_multi_project_config "$d"
unit_state "$d" omp-conductor.service active
unit_state "$d" herdr-fleet.service failed
code=$(run_with_heal "$d" herdr-fleet.service \
  RECOVER_PROJECT= \
  RECOVER_REPORT_PROJECT= \
  RECOVER_PROJECTS= \
  BUN_BIN="$bun_bin")
report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
check 'verified multi-project recovery action exits cleanly' '0' "$code"
check 'verified multi-project recovery action queues one scoped report' \
  '1' "$(grep -c '^omp-conductor report' "$d/calls")"
check 'verified multi-project recovery action uses configured owner' \
  '--project alpha' "$report_call"
check 'verified multi-project recovery action names every project' \
  'affected-projects=alpha,beta' "$report_call"

d=$(newcase multi-project-dry-run)
write_multi_project_config "$d"
unit_state "$d" omp-conductor.service active
unit_state "$d" herdr-fleet.service failed
code=$(run_recover "$d" \
  RECOVER_PROJECT= \
  RECOVER_REPORT_PROJECT= \
  RECOVER_PROJECTS= \
  RECOVER_DRY_RUN=1 \
  BUN_BIN="$bun_bin")
check 'multi-project dry run exits cleanly' '0' "$code"
check 'multi-project dry run names its report owner' \
  'plan: report owner — alpha' "$(cat "$d/out")"
check 'multi-project dry run names every affected project' \
  'affected projects — alpha,beta' "$(cat "$d/out")"

d=$(newcase report-routing-fallback)
unit_state "$d" omp-conductor.service active
unit_state "$d" herdr-fleet.service active
code=$(run_recover "$d" \
  RECOVER_PROJECT= \
  RECOVER_REPORT_PROJECT= \
  RECOVER_PROJECTS= \
  BUN_BIN="$bun_bin")
report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
check 'unreadable config routing still attempts a durable report' \
  'omp-conductor report --kind tier2 --text' "$report_call"
if [[ $report_call == *"--project "* ]]; then
  fail "routing fallback does not guess an unconfigured project — $report_call"
else
  pass 'routing fallback does not guess an unconfigured project'
fi
check 'routing fallback remains a successful recovered run' '0' "$code"
check 'routing fallback is preserved in durable evidence' \
  'attempting the report without a project selector' "$(cat "$d/state/recovery"/evidence-*)"

# ---------------------------------------------------------------------------
# case DRY-RUN — the operator's rehearsal mode prints decisions and mutates
# nothing, not even the recovery dir or the fixture call log.
# ---------------------------------------------------------------------------

d=$(newcase dry-run)
unit_state "$d" herdr-fleet.service failed
unit_state "$d" omp-conductor.service active
(
  cd "$here" || exit 1
  env \
    PATH="$tmp/bin:/usr/bin:/bin" \
    SYSTEMCTL_BIN="$tmp/bin/systemctl" \
    JOURNALCTL_BIN="$tmp/bin/journalctl" \
    OMP_CONDUCTOR_BIN="$tmp/bin/omp-conductor" \
    RECOVER_STATE_DIR="$d/state" \
    RECOVER_PROJECT=demo \
    RECOVER_DRY_RUN=1 \
    STUB_CALLS="$d/calls" \
    STUB_UNITS="$d/units" \
    bash "$recover" >"$d/out" 2>"$d/stderr"
)
check 'a dry run names the action and the escalation' \
  'action: RECOVER_PANE' "$(cat "$d/out")"
check 'a dry run performs no systemctl call' \
  '' "$(cat "$d/calls" 2>/dev/null || true)"
check 'a dry run writes no evidence or counter' \
  '' "$(ls "$d/state/recovery" 2>/dev/null || true)"

# ---------------------------------------------------------------------------
# case — escalation failure fails the recovery (bounded), evidence still there.
# ---------------------------------------------------------------------------

d=$(newcase escalation-fail)
mkdir -p "$d/bin"
cat >"$d/bin/omp-conductor" <<'STUB'
#!/usr/bin/env bash
echo "omp-conductor $*" >>"$STUB_CALLS"
case "$1" in
  doctor) echo '{"status":"pass"}'; exit 0 ;;
  --version) echo "0.15.11" ;;
  report) exit 3 ;;
esac
STUB
chmod +x "$d/bin/omp-conductor"
unit_state "$d" herdr-fleet.service failed
unit_state "$d" omp-conductor.service absent
(
  cd "$here" || exit 1
  env \
    PATH="$tmp/bin:/usr/bin:/bin" \
    SYSTEMCTL_BIN="$tmp/bin/systemctl" \
    JOURNALCTL_BIN="$tmp/bin/journalctl" \
    OMP_CONDUCTOR_BIN="$d/bin/omp-conductor" \
    RECOVER_STATE_DIR="$d/state" \
    RECOVER_PROJECT=demo \
    RECOVER_VERIFY_POLL_S=0 \
    RECOVER_VERIFY_TRIES=2 \
    STUB_CALLS="$d/calls" \
    STUB_UNITS="$d/units" \
    bash "$recover" >"$d/out" 2>"$d/stderr"
)
code=$?
check 'a failed escalation fails the recovery run' '1' "$code"
check 'the evidence file exists even when the escalation failed' \
  '=== omp-conductor fleet recovery evidence (#485) ===' "$(head -n 1 "$d/state/recovery"/evidence-*)"

# ---------------------------------------------------------------------------

if (( failures > 0 )); then
  printf 'FAIL: %s test(s) failed\n' "$failures"
  exit 1
fi
echo 'all tests passed'
exit 0