#!/usr/bin/env bash
# Standing RSS sampler — Task 600.
# Reads /proc/meminfo and per-claude.exe /proc/<pid>/status every 30s.
# Appends one structured line per cycle to LOG_PATH.
# Caps log at LOG_MAX_BYTES (default 10485760 = 10MB).

set -euo pipefail

LOG_PATH="${LOG_PATH:-/tmp/rss-sampler.log}"
LOG_MAX_BYTES="${LOG_MAX_BYTES:-10485760}"
INTERVAL="${INTERVAL:-30}"

# Task 1191 — sibling vnc.sh drives the standing CDP-liveness check each cycle.
VNC_SH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/vnc.sh"

# Task 1593 — standing per-service memory audit + approach-to-OOM warning. The
# muvin OOM (2026-07-13) emitted no application event until the box hard-froze
# on memory exhaustion; these lines reconcile available memory, swap used, and
# per-service RSS against a low-water threshold every cycle, so an approach to
# OOM is visible in the persistent log BEFORE any freeze. Field names are exact
# per the task. Any unavailable field prints `na`; the loop calls emit_mem_lines
# with `|| true`, so a missing unit or absent systemctl never kills the sampler.
# The pressure-state vars persist across iterations (the loop runs in this same
# shell). The test harness sed-extracts the BEGIN..END block and sources it, so
# this is the single source of truth for the emission logic.
mem_in_pressure=0
mem_pressure_since_ms=0
# Monotonic ms since boot from /proc/uptime, so ranSinceKillMs is unaffected by
# an NTP/chrony clock step during a pressure episode. Falls back to wall clock
# only where /proc/uptime is unreadable (never on the Pi). The test overrides it.
now_ms() {
  local up
  if [ -r /proc/uptime ]; then
    up=$(awk '{printf "%d", $1*1000}' /proc/uptime 2>/dev/null || echo "")
    [[ "$up" =~ ^[0-9]+$ ]] && { echo "$up"; return; }
  fi
  date +%s%3N 2>/dev/null || echo $(( $(date +%s) * 1000 ))
}
# 1593-mem-emit-BEGIN
emit_mem_lines() {
  local meminfo="${MEMINFO:-/proc/meminfo}"
  local mt=0 mf=0 ma=0 st=0 sf=0 k v _u
  while IFS=': ' read -r k v _u; do
    case "$k" in
      MemTotal)     mt="$v" ;;
      MemFree)      mf="$v" ;;
      MemAvailable) ma="$v" ;;
      SwapTotal)    st="$v" ;;
      SwapFree)     sf="$v" ;;
    esac
  done < "$meminfo"
  local mt_mb=$(( mt / 1024 )) ma_mb=$(( ma / 1024 )) mf_mb=$(( mf / 1024 )) swused_mb=$(( (st - sf) / 1024 ))
  # per-unit MemoryCurrent (bytes -> MB); `na` when systemctl or the unit is
  # absent, or when systemd reports `[not set]`. Read from systemd's structured
  # key=value interface, never CLI prose.
  _unit_mb() {
    local unit="$1" scope="$2" out val
    [ -z "$unit" ] && { echo na; return 0; }
    command -v systemctl >/dev/null 2>&1 || { echo na; return 0; }
    out=$(systemctl $scope show "$unit" -p MemoryCurrent 2>/dev/null || true)
    val=$(printf '%s\n' "$out" | sed -n 's/^MemoryCurrent=//p')
    if [[ "${val:-}" =~ ^[0-9]+$ ]]; then echo $(( val / 1048576 )); else echo na; fi
  }
  local ptys_mb neo_mb brand_mb edge_mb
  ptys_mb=$(_unit_mb "claude-ptys.slice" "--user")
  neo_mb=$(_unit_mb "${NEO4J_UNIT:-}" "")
  brand_mb=$(_unit_mb "${BRAND_SERVICE:-}" "--user")
  edge_mb=$(_unit_mb "${EDGE_SERVICE:-}" "--user")
  # vnc breakdown: sum VmRSS of the display stack (children of edge.service, so
  # edge= already includes them; vnc= is the breakdown).
  local vnc_mb=0 p rss
  for p in $( { pgrep -x Xtigervnc || true; pgrep -x websockify || true; pgrep -x chromium || true; pgrep -x chromium-browse || true; } 2>/dev/null ); do
    rss=$(grep -m1 '^VmRSS:' "/proc/$p/status" 2>/dev/null | awk '{print $2}' || echo 0)
    [[ "$rss" =~ ^[0-9]+$ ]] || rss=0
    vnc_mb=$(( vnc_mb + rss / 1024 ))
  done
  local ts; ts=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
  echo "[mem-sampler] op=mem-sample ts=${ts} MemAvailable=${ma_mb} MemFree=${mf_mb} SwapUsed=${swused_mb} ptys=${ptys_mb} neo4j=${neo_mb} brand=${brand_mb} edge=${edge_mb} vnc=${vnc_mb}" >> "$LOG_PATH"
  # topSlice/topRss: the unit with the highest numeric reading this cycle.
  local top_slice=na top_rss=0 pair nm vl
  for pair in "ptys:${ptys_mb}" "neo4j:${neo_mb}" "brand:${brand_mb}" "edge:${edge_mb}" "vnc:${vnc_mb}"; do
    nm=${pair%%:*}; vl=${pair##*:}
    if [[ "$vl" =~ ^[0-9]+$ ]] && [ "$vl" -gt "$top_rss" ]; then top_rss="$vl"; top_slice="$nm"; fi
  done
  # Pressure threshold as a percentage of MemTotal, kept above earlyoom's 10%
  # SIGTERM line so the warning is logged BEFORE the kill on every fleet RAM
  # size (8-16 GB): a fixed MB value cannot stay above a percentage kill line
  # across both. Default 20%. A non-numeric env value falls back to the default
  # rather than silently disabling detection.
  local pct="${MEM_PRESSURE_PCT:-20}"
  [[ "$pct" =~ ^[0-9]+$ ]] || pct=20
  local threshold_mb=$(( mt_mb * pct / 100 ))
  local now_v; now_v=$(now_ms)
  # Gate on a real reading: a failed/partial /proc/meminfo read leaves ma_mb=0,
  # which must not be mistaken for genuine exhaustion (a live box is never at 0
  # MemAvailable). On a zero/failed read, neither fire pressure nor clear an
  # existing pressure latch.
  if [ "$ma_mb" -gt 0 ] && [ "$threshold_mb" -gt 0 ]; then
    if [ "$ma_mb" -lt "$threshold_mb" ]; then
      echo "[mem-sampler] op=mem-pressure MemAvailable=${ma_mb} threshold=${threshold_mb} topSlice=${top_slice} topRss=${top_rss}" >> "$LOG_PATH"
      if [ "$mem_in_pressure" -eq 0 ]; then mem_in_pressure=1; mem_pressure_since_ms="$now_v"; fi
    else
      if [ "$mem_in_pressure" -eq 1 ]; then
        local ran=$(( now_v - mem_pressure_since_ms ))
        [ "$ran" -lt 0 ] && ran=0
        # Emit the MEASURED post-condition (MemAvailable recovered + elapsed),
        # never "sent a kill". killedSlice attribution is the operator's cross-
        # check (journalctl -t earlyoom + the service's own `Started` line), per
        # the troubleshooting doc — the sampler does not parse earlyoom prose.
        echo "[mem-sampler] op=mem-recovered MemAvailable=${ma_mb} killedSlice=na ranSinceKillMs=${ran}" >> "$LOG_PATH"
        mem_in_pressure=0
      fi
    fi
  fi
}
# 1593-mem-emit-END

while true; do
  # Read /proc/meminfo for MemTotal and MemAvailable (in kB)
  mem_total_kb=0
  mem_available_kb=0
  while IFS=': ' read -r key val _unit; do
    case "$key" in
      MemTotal)     mem_total_kb="$val" ;;
      MemAvailable) mem_available_kb="$val" ;;
    esac
  done < /proc/meminfo
  mem_total_mb=$(( mem_total_kb / 1024 ))
  mem_available_mb=$(( mem_available_kb / 1024 ))

  # Find all claude.exe PIDs
  mapfile -t pids < <(pgrep -x "claude.exe" 2>/dev/null || true)
  claude_count="${#pids[@]}"
  aggregate_rss_mb=0
  pid_pairs=""

  for pid in "${pids[@]}"; do
    rss_kb=0
    if [[ -r "/proc/${pid}/status" ]]; then
      rss_kb=$(grep -m1 '^VmRSS:' "/proc/${pid}/status" 2>/dev/null | awk '{print $2}' || echo 0)
    fi
    rss_mb=$(( rss_kb / 1024 ))
    aggregate_rss_mb=$(( aggregate_rss_mb + rss_mb ))
    pid_pairs="${pid_pairs:+$pid_pairs,}${pid}:${rss_mb}"
  done

  ts=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
  line="${ts}  MemTotalMB=${mem_total_mb} MemAvailableMB=${mem_available_mb}  claude_count=${claude_count}  aggregate_rss_mb=${aggregate_rss_mb}  pids=${pid_pairs:-none}"
  echo "$line" >> "$LOG_PATH"

  # Task 1176 — standing slice-pressure check. The OOM loop emits no
  # application event until the kill, so sample the claude-ptys.slice
  # cohort's live memory against its MemoryHigh ceiling each cycle. A box
  # trending toward the cap is then visible BEFORE the kernel intervenes;
  # the success post-condition is the brand server's NRestarts staying flat
  # while this ratio may rise — proving the slice, not the server, absorbed
  # the pressure. Read both values from systemd (a structured key=value
  # interface, not CLI prose); skip the line on any host where systemctl is
  # absent, the slice is unknown, or MemoryHigh is unset/infinity.
  if command -v systemctl >/dev/null 2>&1; then
    slice_props=$(systemctl --user show claude-ptys.slice -p MemoryCurrent -p MemoryHigh -p ControlGroup 2>/dev/null || true)
    slice_current=$(printf '%s\n' "$slice_props" | sed -n 's/^MemoryCurrent=//p')
    slice_high=$(printf '%s\n' "$slice_props" | sed -n 's/^MemoryHigh=//p')
    slice_cgroup=$(printf '%s\n' "$slice_props" | sed -n 's/^ControlGroup=//p')
    if [[ "$slice_current" =~ ^[0-9]+$ && "$slice_high" =~ ^[0-9]+$ && "$slice_high" -gt 0 ]]; then
      slice_current_mb=$(( slice_current / 1048576 ))
      slice_high_mb=$(( slice_high / 1048576 ))
      slice_ratio=$(( slice_current * 100 / slice_high ))
      # Task 1297 — memory.events `high` is the cgroup's own count of times
      # this slice crossed MemoryHigh (a throttle event). The concern this
      # answers: a MemoryHigh throttle could itself slow the brand server
      # enough to miss its systemd watchdog heartbeat — this makes that
      # correlation checkable from logs instead of guessed.
      high_events=0
      if [[ -n "$slice_cgroup" && -r "/sys/fs/cgroup${slice_cgroup}/memory.events" ]]; then
        high_events=$(sed -n 's/^high //p' "/sys/fs/cgroup${slice_cgroup}/memory.events" 2>/dev/null || echo 0)
        [[ "$high_events" =~ ^[0-9]+$ ]] || high_events=0
      fi
      echo "[rss] op=slice-pressure slice=claude-ptys current=${slice_current_mb} high=${slice_high_mb} ratio=${slice_ratio} highEvents=${high_events}" >> "$LOG_PATH"
      # Task 1297 — the exact standing-sampler line the task names: real
      # cgroup-accounted cohort total (not the claude.exe-only pgrep sum used
      # by the plain per-cycle line above) against the real ceiling, so
      # sizing can be tuned from evidence instead of guessed.
      echo "[realagent-mem-census] rss_total=${slice_current} ceiling=${slice_high} sessions_live=${claude_count}" >> "$LOG_PATH"
    fi
  fi

  # Task 1593 — per-service memory audit + approach-to-OOM warning (see the
  # emit_mem_lines definition above the loop). Guarded so a non-zero exit can
  # never kill the sampler under `set -e`.
  emit_mem_lines || true

  # Task 1191 — standing CDP-liveness check. The rss-sampler is the brand's
  # standing-check cadence; delegate the CDP probe + bounded Chromium respawn to
  # vnc.sh, which owns brand config and the start_chrome path. The browser
  # plugin is a pure CDP client, so a Chromium that dies while the VNC display
  # stays up leaves every browser-* tool returning cdp-unreachable until a
  # service bounce; this restores it without one. The op=cdp-liveness /
  # op=chromium-respawn lines flow to this service's journal (stdout), NOT to
  # LOG_PATH. Guarded so a non-zero exit (e.g. a host with no brand.json) never
  # kills the sampler loop under `set -e`.
  if [ -f "$VNC_SH" ]; then
    bash "$VNC_SH" cdp-liveness || true
  fi

  # Cap log at LOG_MAX_BYTES: drop oldest half when threshold is exceeded
  if [[ -f "$LOG_PATH" ]]; then
    actual_bytes=$(wc -c < "$LOG_PATH")
    if (( actual_bytes > LOG_MAX_BYTES )); then
      total_lines=$(wc -l < "$LOG_PATH")
      keep_lines=$(( total_lines / 2 ))
      tmp_file="${LOG_PATH}.tmp"
      tail -n "$keep_lines" "$LOG_PATH" > "$tmp_file" && mv "$tmp_file" "$LOG_PATH"
    fi
  fi

  sleep "$INTERVAL"
done