#!/bin/bash
# ralph-watchdog.sh — supervise Ralph loops and self-heal failures.
#
# Default behavior:
# - Poll health every 5 minutes
# - Keep one Codex loop and one Claude loop alive
# - Reconcile orphaned/stale runs
# - Reset active tasks that no longer have a running run
# - Restart loops after error bursts
#
# Usage:
#   ./scripts/ralph-watchdog.sh
#   ./scripts/ralph-watchdog.sh --once
#   ./scripts/ralph-watchdog.sh --interval 120
#   ./scripts/ralph-watchdog.sh --no-claude
#   ./scripts/ralph-watchdog.sh --no-start

set -euo pipefail

PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
DB_PATH="$PROJECT_DIR/.tx/tasks.db"
LOG_FILE="$PROJECT_DIR/.tx/ralph-watchdog.log"

POLL_SECONDS=${POLL_SECONDS:-300}
RUN_STALE_SECONDS=${RUN_STALE_SECONDS:-5400}
TRANSCRIPT_IDLE_SECONDS=${TRANSCRIPT_IDLE_SECONDS:-600}
HEARTBEAT_LAG_SECONDS=${HEARTBEAT_LAG_SECONDS:-180}
CLAUDE_STALL_GRACE_SECONDS=${CLAUDE_STALL_GRACE_SECONDS:-900}
ERROR_BURST_WINDOW_MINUTES=${ERROR_BURST_WINDOW_MINUTES:-20}
ERROR_BURST_THRESHOLD=${ERROR_BURST_THRESHOLD:-4}
ERROR_BURST_GRACE_SECONDS=${ERROR_BURST_GRACE_SECONDS:-600}
RESTART_COOLDOWN_SECONDS=${RESTART_COOLDOWN_SECONDS:-900}

CODEX_ENABLED=true
CLAUDE_ENABLED=true
AUTO_START=true
RUN_ONCE=false

CODEX_PREFIX=${CODEX_PREFIX:-ralph-codex-live}
CLAUDE_PREFIX=${CLAUDE_PREFIX:-ralph-claude-live}

MAX_ITERATIONS=${MAX_ITERATIONS:-1000000}
MAX_HOURS=${MAX_HOURS:-24}
TASK_TIMEOUT=${TASK_TIMEOUT:-1800}
VERIFY_TIMEOUT=${VERIFY_TIMEOUT:-180}
LEARNINGS_TIMEOUT=${LEARNINGS_TIMEOUT:-180}
CLAIM_LEASE_MINUTES=${CLAIM_LEASE_MINUTES:-30}
CLAIM_RENEW_INTERVAL=${CLAIM_RENEW_INTERVAL:-300}
HEARTBEAT_INTERVAL=${HEARTBEAT_INTERVAL:-30}
IDLE_ROUNDS=${IDLE_ROUNDS:-300}
AUTO_COMMIT=${AUTO_COMMIT:-true}
REVIEW_ENABLED=${REVIEW_ENABLED:-false}
WATCHDOG_PID_FILE="$PROJECT_DIR/.tx/ralph-watchdog.pid"

while [[ $# -gt 0 ]]; do
  case "$1" in
    --interval) POLL_SECONDS="$2"; shift 2 ;;
    --run-stale-seconds) RUN_STALE_SECONDS="$2"; shift 2 ;;
    --transcript-idle-seconds) TRANSCRIPT_IDLE_SECONDS="$2"; shift 2 ;;
    --heartbeat-lag-seconds) HEARTBEAT_LAG_SECONDS="$2"; shift 2 ;;
    --claude-stall-grace-seconds) CLAUDE_STALL_GRACE_SECONDS="$2"; shift 2 ;;
    --error-window-minutes) ERROR_BURST_WINDOW_MINUTES="$2"; shift 2 ;;
    --error-threshold) ERROR_BURST_THRESHOLD="$2"; shift 2 ;;
    --error-burst-grace-seconds) ERROR_BURST_GRACE_SECONDS="$2"; shift 2 ;;
    --restart-cooldown-seconds) RESTART_COOLDOWN_SECONDS="$2"; shift 2 ;;
    --heartbeat-interval) HEARTBEAT_INTERVAL="$2"; shift 2 ;;
    --idle-rounds) IDLE_ROUNDS="$2"; shift 2 ;;
    --codex-prefix) CODEX_PREFIX="$2"; shift 2 ;;
    --claude-prefix) CLAUDE_PREFIX="$2"; shift 2 ;;
    --no-codex) CODEX_ENABLED=false; shift ;;
    --no-claude) CLAUDE_ENABLED=false; shift ;;
    --no-start) AUTO_START=false; shift ;;
    --once) RUN_ONCE=true; shift ;;
    *) echo "Unknown option: $1" >&2; exit 1 ;;
  esac
done

if ! [[ "$POLL_SECONDS" =~ ^[0-9]+$ ]] || [ "$POLL_SECONDS" -lt 1 ]; then
  echo "Invalid --interval value: $POLL_SECONDS" >&2
  exit 1
fi

if ! [[ "$RUN_STALE_SECONDS" =~ ^[0-9]+$ ]] || [ "$RUN_STALE_SECONDS" -lt 60 ]; then
  echo "Invalid --run-stale-seconds value: $RUN_STALE_SECONDS" >&2
  exit 1
fi

if ! [[ "$TRANSCRIPT_IDLE_SECONDS" =~ ^[0-9]+$ ]] || [ "$TRANSCRIPT_IDLE_SECONDS" -lt 60 ]; then
  echo "Invalid --transcript-idle-seconds value: $TRANSCRIPT_IDLE_SECONDS" >&2
  exit 1
fi

if ! [[ "$HEARTBEAT_LAG_SECONDS" =~ ^[0-9]+$ ]] || [ "$HEARTBEAT_LAG_SECONDS" -lt 1 ]; then
  echo "Invalid --heartbeat-lag-seconds value: $HEARTBEAT_LAG_SECONDS" >&2
  exit 1
fi

if ! [[ "$CLAUDE_STALL_GRACE_SECONDS" =~ ^[0-9]+$ ]] || [ "$CLAUDE_STALL_GRACE_SECONDS" -lt 0 ]; then
  echo "Invalid --claude-stall-grace-seconds value: $CLAUDE_STALL_GRACE_SECONDS" >&2
  exit 1
fi

if ! [[ "$ERROR_BURST_WINDOW_MINUTES" =~ ^[0-9]+$ ]] || [ "$ERROR_BURST_WINDOW_MINUTES" -lt 1 ]; then
  echo "Invalid --error-window-minutes value: $ERROR_BURST_WINDOW_MINUTES" >&2
  exit 1
fi

if ! [[ "$ERROR_BURST_THRESHOLD" =~ ^[0-9]+$ ]] || [ "$ERROR_BURST_THRESHOLD" -lt 1 ]; then
  echo "Invalid --error-threshold value: $ERROR_BURST_THRESHOLD" >&2
  exit 1
fi

if ! [[ "$ERROR_BURST_GRACE_SECONDS" =~ ^[0-9]+$ ]] || [ "$ERROR_BURST_GRACE_SECONDS" -lt 1 ]; then
  echo "Invalid --error-burst-grace-seconds value: $ERROR_BURST_GRACE_SECONDS" >&2
  exit 1
fi

if ! [[ "$RESTART_COOLDOWN_SECONDS" =~ ^[0-9]+$ ]] || [ "$RESTART_COOLDOWN_SECONDS" -lt 1 ]; then
  echo "Invalid --restart-cooldown-seconds value: $RESTART_COOLDOWN_SECONDS" >&2
  exit 1
fi

if ! [[ "$HEARTBEAT_INTERVAL" =~ ^[0-9]+$ ]] || [ "$HEARTBEAT_INTERVAL" -lt 1 ]; then
  echo "Invalid --heartbeat-interval value: $HEARTBEAT_INTERVAL" >&2
  exit 1
fi

if ! [[ "$IDLE_ROUNDS" =~ ^[0-9]+$ ]] || [ "$IDLE_ROUNDS" -lt 1 ]; then
  echo "Invalid --idle-rounds value: $IDLE_ROUNDS" >&2
  exit 1
fi

if [ "$CODEX_ENABLED" = false ] && [ "$CLAUDE_ENABLED" = false ]; then
  echo "Both runtimes disabled (--no-codex and --no-claude). Nothing to supervise." >&2
  exit 1
fi

mkdir -p "$PROJECT_DIR/.tx"

if [ ! -f "$DB_PATH" ]; then
  echo "Database not found: $DB_PATH (run tx init first)" >&2
  exit 1
fi

tx() {
  bun "$PROJECT_DIR/apps/cli/src/cli.ts" "$@"
}

log() {
  local msg=""
  msg="[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $1"
  echo "$msg" | tee -a "$LOG_FILE"
}

interruptible_sleep_seconds() {
  local seconds="$1"
  [ -z "$seconds" ] && return 0
  if ! [[ "$seconds" =~ ^[0-9]+$ ]] || [ "$seconds" -le 0 ]; then
    return 0
  fi

  # Keep sleep chunks small so INT/TERM traps can be handled promptly.
  while [ "$seconds" -gt 0 ]; do
    sleep 1
    seconds=$((seconds - 1))
  done
}

pid_is_live() {
  local pid="$1"
  if [ -z "$pid" ]; then
    return 1
  fi
  if ! [[ "$pid" =~ ^[0-9]+$ ]]; then
    return 1
  fi
  kill -0 "$pid" 2>/dev/null
}

pid_command() {
  local pid="$1"
  ps -p "$pid" -o command= 2>/dev/null || true
}

duration_to_seconds() {
  local value="$1"
  local days=0
  local hours=0
  local mins=0
  local secs=0

  [ -z "$value" ] && { echo "0"; return; }
  value=$(echo "$value" | tr -d '[:space:]')
  [ -z "$value" ] && { echo "0"; return; }

  if [[ "$value" == *-* ]]; then
    days="${value%%-*}"
    value="${value#*-}"
  fi

  case "$(echo "$value" | awk -F: '{print NF}')" in
    3)
      hours="${value%%:*}"
      value="${value#*:}"
      mins="${value%%:*}"
      secs="${value#*:}"
      ;;
    2)
      mins="${value%%:*}"
      secs="${value#*:}"
      ;;
    1)
      secs="$value"
      ;;
    *)
      echo "0"
      return
      ;;
  esac

  [[ "$days" =~ ^[0-9]+$ ]] || days=0
  [[ "$hours" =~ ^[0-9]+$ ]] || hours=0
  [[ "$mins" =~ ^[0-9]+$ ]] || mins=0
  [[ "$secs" =~ ^[0-9]+$ ]] || secs=0

  echo $((days * 86400 + hours * 3600 + mins * 60 + secs))
}

pid_elapsed_seconds() {
  local pid="$1"
  local raw=""

  if ! pid_is_live "$pid"; then
    echo "0"
    return
  fi

  raw=$(ps -p "$pid" -o etime= 2>/dev/null || echo "")
  duration_to_seconds "$raw"
}

watchdog_pid_is_live() {
  local pid="$1"
  local cmd=""
  if ! pid_is_live "$pid"; then
    return 1
  fi

  cmd=$(pid_command "$pid")
  [ -n "$cmd" ] || return 1
  echo "$cmd" | grep -q "ralph-watchdog.sh"
}

loop_pid_is_live() {
  local pid="$1"
  local cmd=""
  if ! pid_is_live "$pid"; then
    return 1
  fi

  cmd=$(pid_command "$pid")
  [ -n "$cmd" ] || return 1
  echo "$cmd" | grep -Eq '(^|[ /])ralph\.sh([[:space:]]|$)'
}

command_has_arg_pair() {
  local command_line="$1"
  local flag="$2"
  local value="$3"

  [ -n "$value" ] || return 1

  case " $command_line " in
    *" $flag $value "*) return 0 ;;
    *) return 1 ;;
  esac
}

expected_prefix_for_runtime() {
  local runtime="$1"
  case "$runtime" in
    codex) echo "$CODEX_PREFIX" ;;
    claude) echo "$CLAUDE_PREFIX" ;;
    *) echo "" ;;
  esac
}

run_pid_is_owned_by_tx_runtime() {
  local pid="$1"
  local runtime="$2"
  local worker="$3"
  local cmd=""
  local expected_prefix=""
  local worker_prefix=""

  if ! pid_is_live "$pid"; then
    return 1
  fi

  [ -n "$runtime" ] || return 1
  expected_prefix=$(expected_prefix_for_runtime "$runtime")
  [ -n "$expected_prefix" ] || return 1

  cmd=$(pid_command "$pid")
  [ -n "$cmd" ] || return 1

  echo "$cmd" | grep -F -- "$PROJECT_DIR/scripts/ralph.sh" >/dev/null || return 1
  command_has_arg_pair "$cmd" "--runtime" "$runtime" || return 1
  command_has_arg_pair "$cmd" "--worker-prefix" "$expected_prefix" || return 1

  if [ -n "$worker" ]; then
    worker_prefix="${worker%-main}"
    if [ -n "$worker_prefix" ] && [ "$worker_prefix" != "$worker" ]; then
      command_has_arg_pair "$cmd" "--worker-prefix" "$worker_prefix" || return 1
    fi
  fi

  return 0
}

is_worker_managed_run() {
  local runtime="$1"
  local worker="$2"

  case "$runtime" in
    codex|claude|custom) return 0 ;;
  esac

  if [ -n "$worker" ]; then
    return 0
  fi

  return 1
}

read_watchdog_pid_file() {
  local pid=""

  if [ ! -f "$WATCHDOG_PID_FILE" ]; then
    echo ""
    return
  fi

  pid=$(cat "$WATCHDOG_PID_FILE" 2>/dev/null | tr -d '[:space:]')
  if [[ "$pid" =~ ^[0-9]+$ ]]; then
    echo "$pid"
  else
    echo ""
  fi
}

acquire_watchdog_lock() {
  local attempts=0
  local max_attempts=8

  while [ "$attempts" -lt "$max_attempts" ]; do
    if ( set -o noclobber; printf '%s\n' "$$" > "$WATCHDOG_PID_FILE" ) 2>/dev/null; then
      return 0
    fi

    local lock_pid=""
    lock_pid=$(read_watchdog_pid_file)
    if watchdog_pid_is_live "$lock_pid"; then
      log "Another watchdog is already running (pid=$lock_pid). Exiting."
      exit 0
    fi

    if [ -n "$lock_pid" ]; then
      if pid_is_live "$lock_pid"; then
        log "Watchdog pid file points to non-watchdog process (pid=$lock_pid); replacing stale lock"
      else
        log "Removing stale watchdog pid file (pid=$lock_pid)"
      fi
    fi

    # Race-safe stale cleanup: only remove when contents match what we observed.
    local latest_pid=""
    latest_pid=$(read_watchdog_pid_file)
    if [ "$latest_pid" = "$lock_pid" ]; then
      rm -f "$WATCHDOG_PID_FILE" 2>/dev/null || true
    fi

    attempts=$((attempts + 1))
    sleep 0.02
  done

  local final_pid=""
  final_pid=$(read_watchdog_pid_file)
  if watchdog_pid_is_live "$final_pid"; then
    log "Another watchdog is already running (pid=$final_pid). Exiting."
    exit 0
  fi

  log "Unable to acquire watchdog lock after retries; exiting."
  exit 1
}

remove_owned_watchdog_pid_file() {
  if [ ! -f "$WATCHDOG_PID_FILE" ]; then
    return 0
  fi

  local owner_pid=""
  owner_pid=$(read_watchdog_pid_file)
  if [ "$owner_pid" = "$$" ]; then
    rm -f "$WATCHDOG_PID_FILE"
  fi
}

release_watchdog_lock() {
  remove_owned_watchdog_pid_file
}

handle_watchdog_shutdown_signal() {
  local signal="$1"
  local exit_code="$2"

  trap - INT TERM
  log "Received SIG${signal}; releasing watchdog lock and exiting"
  release_watchdog_lock
  exit "$exit_code"
}

restart_stamp_file_for() {
  local runtime="$1"
  local prefix="$2"
  local key=""
  key=$(lock_key_for "$runtime" "$prefix")
  printf '%s/.tx/ralph-watchdog-restart-%s.stamp' "$PROJECT_DIR" "$key"
}

sql_escape() {
  echo "${1//\'/\'\'}"
}

lock_key_for() {
  local runtime="$1"
  local prefix="$2"
  printf '%s' "${runtime}-${prefix}" | tr -c 'a-zA-Z0-9._-' '-'
}

pid_file_for() {
  local runtime="$1"
  local prefix="$2"
  local key=""
  key=$(lock_key_for "$runtime" "$prefix")
  printf '%s/.tx/ralph-%s.pid' "$PROJECT_DIR" "$key"
}

loop_pid() {
  local runtime="$1"
  local prefix="$2"
  local pid_file=""
  pid_file=$(pid_file_for "$runtime" "$prefix")

  if [ ! -f "$pid_file" ]; then
    return 1
  fi

  local pid=""
  pid=$(cat "$pid_file" 2>/dev/null || true)
  if [ -z "$pid" ]; then
    return 1
  fi

  printf '%s' "$pid"
}

terminate_pid_tree() {
  local pid="$1"
  local signal="${2:-TERM}"

  [ -z "$pid" ] && return 0

  if command -v pgrep >/dev/null 2>&1; then
    local children=""
    children=$(pgrep -P "$pid" 2>/dev/null || true)
    if [ -n "$children" ]; then
      while IFS= read -r child_pid; do
        [ -z "$child_pid" ] && continue
        terminate_pid_tree "$child_pid" "$signal"
      done <<< "$children"
    fi
  fi

  kill "-$signal" "$pid" 2>/dev/null || true
}

expire_active_claims_for_task() {
  local task_id="$1"
  [ -z "$task_id" ] && return

  sqlite3 "$DB_PATH" \
    "UPDATE task_claims
     SET status='expired'
     WHERE task_id='$(sql_escape "$task_id")'
       AND status='active';" \
    >/dev/null 2>&1 || true
}

reset_task_and_expire_claims() {
  local task_id="$1"
  [ -z "$task_id" ] && return

  expire_active_claims_for_task "$task_id"
  tx reset "$task_id" >/dev/null 2>&1 || true
}

start_loop() {
  local runtime="$1"
  local prefix="$2"
  local out_log="$PROJECT_DIR/.tx/ralph-${runtime}-${prefix}.supervised.log"

  local cmd=(
    "$PROJECT_DIR/scripts/ralph.sh"
    --runtime "$runtime"
    --workers 1
    --max "$MAX_ITERATIONS"
    --max-hours "$MAX_HOURS"
    --worker-prefix "$prefix"
    --task-timeout "$TASK_TIMEOUT"
    --verify-timeout "$VERIFY_TIMEOUT"
    --learnings-timeout "$LEARNINGS_TIMEOUT"
    --claim-lease "$CLAIM_LEASE_MINUTES"
    --claim-renew-interval "$CLAIM_RENEW_INTERVAL"
    --heartbeat-interval "$HEARTBEAT_INTERVAL"
    --claude-stall-seconds "$TRANSCRIPT_IDLE_SECONDS"
    --claude-stall-grace-seconds "$CLAUDE_STALL_GRACE_SECONDS"
    --idle-rounds "$IDLE_ROUNDS"
  )

  if [ "$AUTO_COMMIT" != true ]; then
    cmd+=(--no-commit)
  fi

  if [ "$REVIEW_ENABLED" != true ]; then
    cmd+=(--no-review)
  fi

  if [ "$AUTO_START" != true ]; then
    log "[dry] AUTO_START=false; would start runtime=$runtime prefix=$prefix"
    return 0
  fi

  RALPH_IGNORE_HUP=1 nohup "${cmd[@]}" >>"$out_log" 2>&1 < /dev/null &
  local launcher_pid="$!"
  sleep 1

  local pid=""
  pid=$(loop_pid "$runtime" "$prefix" 2>/dev/null || true)
  if [ -n "$pid" ] && loop_pid_is_live "$pid"; then
    log "Started runtime=$runtime prefix=$prefix pid=$pid"
  else
    log "Start attempted runtime=$runtime prefix=$prefix launcher_pid=$launcher_pid"
  fi
}

restart_loop() {
  local runtime="$1"
  local prefix="$2"
  local reason="$3"
  local pid=""
  pid=$(loop_pid "$runtime" "$prefix" 2>/dev/null || true)

  if [ -n "$pid" ] && loop_pid_is_live "$pid"; then
    log "Restarting runtime=$runtime prefix=$prefix pid=$pid reason=$reason"
    terminate_pid_tree "$pid" TERM
    sleep 2
    if loop_pid_is_live "$pid"; then
      terminate_pid_tree "$pid" KILL
    fi
  else
    if [ -n "$pid" ]; then
      log "Restart runtime=$runtime prefix=$prefix reason=$reason (stale pid=$pid)"
    else
      log "Restart runtime=$runtime prefix=$prefix reason=$reason (no live pid)"
    fi
  fi

  start_loop "$runtime" "$prefix"
}

restart_with_cooldown() {
  local runtime="$1"
  local prefix="$2"
  local reason="$3"
  local now=""
  now=$(date +%s)

  local stamp_file=""
  stamp_file=$(restart_stamp_file_for "$runtime" "$prefix")
  if [ -f "$stamp_file" ]; then
    local last_restart=""
    last_restart=$(cat "$stamp_file" 2>/dev/null || echo "0")
    if [[ "$last_restart" =~ ^[0-9]+$ ]]; then
      local elapsed=$((now - last_restart))
      if [ "$elapsed" -lt "$RESTART_COOLDOWN_SECONDS" ]; then
        log "Restart cooldown active runtime=$runtime prefix=$prefix elapsed=${elapsed}s threshold=${RESTART_COOLDOWN_SECONDS}s"
        return
      fi
    fi
  fi

  echo "$now" > "$stamp_file"
  restart_loop "$runtime" "$prefix" "$reason"
}

ensure_loop() {
  local runtime="$1"
  local prefix="$2"
  local enabled="$3"

  if [ "$enabled" != true ]; then
    return
  fi

  local pid=""
  pid=$(loop_pid "$runtime" "$prefix" 2>/dev/null || true)
  if [ -n "$pid" ] && loop_pid_is_live "$pid"; then
    return
  fi
  if [ -n "$pid" ]; then
    log "Detected stale loop pid runtime=$runtime prefix=$prefix pid=$pid; restarting"
    rm -f "$(pid_file_for "$runtime" "$prefix")"
  fi

  log "Loop missing runtime=$runtime prefix=$prefix; starting"
  start_loop "$runtime" "$prefix"
}

reconcile_running_runs() {
  local rows=""
  rows=$(sqlite3 "$DB_PATH" \
    "SELECT id, COALESCE(task_id,''), COALESCE(pid,0), CAST((julianday('now') - julianday(started_at)) * 86400 AS INTEGER), COALESCE(json_extract(metadata, '$.runtime'), ''), COALESCE(json_extract(metadata, '$.worker'), ''), COALESCE(agent, '')
     FROM runs
     WHERE status='running';" 2>/dev/null || echo "")

  [ -z "$rows" ] && return

  while IFS='|' read -r run_id task_id pid age runtime worker agent; do
    [ -z "$run_id" ] && continue
    local managed_run=false
    if is_worker_managed_run "$runtime" "$worker"; then
      managed_run=true
    fi

    if [ "$pid" -le 0 ]; then
      if [ "$managed_run" != true ]; then
        log "Skipping run=$run_id pid reconciliation (agent=${agent:-unknown} runtime=${runtime:-none} worker=${worker:-none})"
        continue
      fi
      sqlite3 "$DB_PATH" \
        "UPDATE runs SET status='cancelled', ended_at=strftime('%Y-%m-%dT%H:%M:%SZ', 'now'), exit_code=137, error_message='Watchdog: missing PID for running run' WHERE id='$(sql_escape "$run_id")';" \
        >/dev/null 2>&1 || true
      [ -n "$task_id" ] && reset_task_and_expire_claims "$task_id"
      log "Reconciled run=$run_id (missing pid)"
      continue
    fi

    if ! kill -0 "$pid" 2>/dev/null; then
      if [ "$managed_run" != true ]; then
        log "Skipping run=$run_id dead-pid reconciliation (agent=${agent:-unknown} pid=$pid runtime=${runtime:-none} worker=${worker:-none})"
        continue
      fi
      sqlite3 "$DB_PATH" \
        "UPDATE runs SET status='cancelled', ended_at=strftime('%Y-%m-%dT%H:%M:%SZ', 'now'), exit_code=137, error_message='Watchdog: process not alive' WHERE id='$(sql_escape "$run_id")';" \
        >/dev/null 2>&1 || true
      [ -n "$task_id" ] && reset_task_and_expire_claims "$task_id"
      log "Reconciled run=$run_id (dead pid=$pid)"
      continue
    fi

    if [ "$age" -gt "$RUN_STALE_SECONDS" ]; then
      if [ "$managed_run" != true ]; then
        log "Skipping stale-run pid kill for non-worker run=$run_id (agent=${agent:-unknown} age=${age}s)"
        continue
      fi
      if ! run_pid_is_owned_by_tx_runtime "$pid" "$runtime" "$worker"; then
        log "Stale run detected run=$run_id pid=$pid age=${age}s; ownership not confirmed, cancelling without kill"
        sqlite3 "$DB_PATH" \
          "UPDATE runs SET status='cancelled', ended_at=strftime('%Y-%m-%dT%H:%M:%SZ', 'now'), exit_code=137, error_message='Watchdog: stale running run cancelled without kill (ownership not confirmed for pid $pid, age ${age}s)' WHERE id='$(sql_escape "$run_id")';" \
          >/dev/null 2>&1 || true
        [ -n "$task_id" ] && reset_task_and_expire_claims "$task_id"
        continue
      fi

      log "Stale run detected run=$run_id pid=$pid age=${age}s; terminating"
      terminate_pid_tree "$pid" TERM
      sleep 2
      if kill -0 "$pid" 2>/dev/null; then
        terminate_pid_tree "$pid" KILL
      fi
      sqlite3 "$DB_PATH" \
        "UPDATE runs SET status='cancelled', ended_at=strftime('%Y-%m-%dT%H:%M:%SZ', 'now'), exit_code=137, error_message='Watchdog: stale running run killed (age ${age}s)' WHERE id='$(sql_escape "$run_id")';" \
        >/dev/null 2>&1 || true
      [ -n "$task_id" ] && reset_task_and_expire_claims "$task_id"
    fi
  done <<< "$rows"
}

reset_orphaned_active_tasks() {
  local tasks=""
  tasks=$(sqlite3 "$DB_PATH" \
    "SELECT t.id
     FROM tasks t
     WHERE t.status='active'
       AND NOT EXISTS (
         SELECT 1
         FROM runs r
         WHERE r.task_id=t.id
           AND r.status='running'
       );" 2>/dev/null || echo "")

  [ -z "$tasks" ] && return

  local count=0
  while IFS= read -r task_id; do
    [ -z "$task_id" ] && continue
    reset_task_and_expire_claims "$task_id"
    count=$((count + 1))
  done <<< "$tasks"

  if [ "$count" -gt 0 ]; then
    log "Reset $count orphaned active task(s)"
  fi
}

reap_stalled_runs_via_primitive() {
  local cmd=(
    trace stalled
    --reap
    --json
    --transcript-idle-seconds "$TRANSCRIPT_IDLE_SECONDS"
    --heartbeat-lag-seconds "$HEARTBEAT_LAG_SECONDS"
  )

  local output="[]"
  output=$(tx "${cmd[@]}" 2>/dev/null || echo "[]")

  local reaped_count="0"
  reaped_count=$(echo "$output" | jq 'length' 2>/dev/null || echo "0")
  if [ "$reaped_count" -gt 0 ]; then
    log "Reaped $reaped_count stalled run(s) via tx primitive (transcript_idle>${TRANSCRIPT_IDLE_SECONDS}s)"
  fi
}

check_error_burst_for_worker() {
  local runtime="$1"
  local prefix="$2"
  local enabled="$3"

  if [ "$enabled" != true ]; then
    return
  fi

  local worker="${prefix}-main"
  local ready_count="0"
  ready_count=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM tasks WHERE status='ready';" 2>/dev/null || echo "0")
  if [ "$ready_count" -eq 0 ]; then
    return
  fi

  local count="0"
  count=$(sqlite3 "$DB_PATH" \
    "SELECT COUNT(*)
     FROM runs
     WHERE status IN ('failed', 'cancelled')
       AND started_at >= strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-${ERROR_BURST_WINDOW_MINUTES} minutes')
       AND json_extract(metadata, '$.worker') = '$(sql_escape "$worker")';" 2>/dev/null || echo "0")

  if [ "$count" -ge "$ERROR_BURST_THRESHOLD" ]; then
    local loop_pid_val=""
    loop_pid_val=$(loop_pid "$runtime" "$prefix" 2>/dev/null || true)
    if [ -n "$loop_pid_val" ] && loop_pid_is_live "$loop_pid_val"; then
      local loop_age_seconds="0"
      loop_age_seconds=$(pid_elapsed_seconds "$loop_pid_val")
      if [ "$loop_age_seconds" -lt "$ERROR_BURST_GRACE_SECONDS" ]; then
        log "Error burst detected runtime=$runtime prefix=$prefix count=$count but loop uptime=${loop_age_seconds}s < grace=${ERROR_BURST_GRACE_SECONDS}s; skipping restart"
        return
      fi
    fi

    local worker_running_count="0"
    worker_running_count=$(sqlite3 "$DB_PATH" \
      "SELECT COUNT(*)
       FROM runs
       WHERE status = 'running'
         AND json_extract(metadata, '$.worker') = '$(sql_escape "$worker")';" 2>/dev/null || echo "0")

    if [ "$worker_running_count" -gt 0 ]; then
      log "Error burst detected runtime=$runtime prefix=$prefix count=$count but worker has active run; skipping restart"
      return
    fi

    restart_with_cooldown "$runtime" "$prefix" "error-burst count=${count} window=${ERROR_BURST_WINDOW_MINUTES}m"
  fi
}

acquire_watchdog_lock
trap 'release_watchdog_lock' EXIT
trap 'handle_watchdog_shutdown_signal INT 130' INT
trap 'handle_watchdog_shutdown_signal TERM 143' TERM
trap 'log "Received SIGHUP signal; ignoring to stay detached"' HUP

log "Watchdog started interval=${POLL_SECONDS}s codex=${CODEX_ENABLED} claude=${CLAUDE_ENABLED} auto_start=${AUTO_START} idle_rounds=${IDLE_ROUNDS}"
log "Stall thresholds: transcript_idle=${TRANSCRIPT_IDLE_SECONDS}s claude_grace=${CLAUDE_STALL_GRACE_SECONDS}s heartbeat_lag=${HEARTBEAT_LAG_SECONDS}s run_stale=${RUN_STALE_SECONDS}s"

while true; do
  ensure_loop "codex" "$CODEX_PREFIX" "$CODEX_ENABLED"
  ensure_loop "claude" "$CLAUDE_PREFIX" "$CLAUDE_ENABLED"

  reconcile_running_runs
  reset_orphaned_active_tasks
  reap_stalled_runs_via_primitive

  check_error_burst_for_worker "codex" "$CODEX_PREFIX" "$CODEX_ENABLED"
  check_error_burst_for_worker "claude" "$CLAUDE_PREFIX" "$CLAUDE_ENABLED"

  local_running="0"
  local_running=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM runs WHERE status='running';" 2>/dev/null || echo "0")
  log "Health check complete running_runs=$local_running"

  if [ "$RUN_ONCE" = true ]; then
    break
  fi

  interruptible_sleep_seconds "$POLL_SECONDS"
done

log "Watchdog exiting"