#!/usr/bin/env bash # cpu-triage.sh — sustained CPU triage for a Maxy Code device. # # WHY THIS EXISTS, AND WHY IT SAMPLES FOR MINUTES RATHER THAN SECONDS # # The booking reconcile loop runs on a 120 s cycle and a tick costs roughly # 800 ms of network wait plus a garbage-collection burst. Measured on laptop # 192.168.88.10 (2026-07-20), the SAME server process over the SAME method read # 0.2%, 0.2%, 0.2% in three consecutive 25 s windows and 10.8% in a 20 s window. # Every one of those readings was correct. Any sample shorter than the period # either lands on a tick or misses it, so spot checks on this platform produce # contradictory answers indefinitely. This tool therefore samples across at # least one full cycle and reports a DISTRIBUTION (mean, peak, and what share of # the window sat above threshold), never a single number. # # THREE INSTRUMENTS THIS DELIBERATELY DOES NOT USE, EACH HAVING PRODUCED A FALSE # CONCLUSION IN THE 2026-07-20 INVESTIGATION: # * `ps %cpu` / `top` %CPU — LIFETIME averages. A 40-minute-old process read # 8.5% against an 8-day-old peer at 0.1%; both were actually 0.2% live. This # nearly shipped as an "85x regression". # * `iostat -c` first line — a SINCE-BOOT average. Read as live it reported a # machine "42% busy" that was 87% idle at that moment. # * `ns_last_pid` deltas — counts THREAD creation as well as forks, and was # compared against a single 10 s sample used as a "baseline". Produced a # false 5.6x churn regression. Fork rate here comes from /proc/stat's # `processes` counter, which counts forks only. # Everything below is a DELTA over a stated window. No lifetime averages. # # Aggregate CPU is also avoided as a headline: two hot cores out of twenty # average down to ~3% and vanish, which is exactly how a real, operator-visible # condition was dismissed as "nothing changed". Per-core is the primary view. # # Unprivileged by design: the admin assistant runs without sudo, so this uses # only /proc and the user's own cgroup tree. `perf` and `bpftrace` need root and # are named in the escalation hint rather than invoked. # # EXIT CODES (gates, for deterministic callers) # 0 no core sustained above threshold — nothing to escalate # 1 one or more cores sustained above threshold — findings in the report # 2 usage error or unmet precondition (nothing measured) set -uo pipefail WINDOW=300 # total sampling window, seconds (>= 2 reconcile cycles) INTERVAL=10 # sample cadence, seconds THRESHOLD=15 # per-core busy % that counts as "hot" SUSTAIN=50 # % of samples above THRESHOLD to call a core "sustained" TOPN=12 # processes to report JSON=0 CONTROL="" # optional peer brand known NOT to have the change usage() { cat >&2 <&2; usage ;; esac done case "$WINDOW$INTERVAL$THRESHOLD$SUSTAIN$TOPN" in *[!0-9]*) usage ;; esac [ "$INTERVAL" -ge 1 ] || usage if [ "$WINDOW" -lt 120 ]; then echo "cpu-triage: refusing a window under 120s — shorter than the reconcile cycle," >&2 echo " which is what makes spot readings contradict each other." >&2 exit 2 fi [ -r /proc/stat ] || { echo "cpu-triage: /proc/stat unreadable — Linux only" >&2; exit 2; } SAMPLES=$(( WINDOW / INTERVAL )) [ "$SAMPLES" -ge 2 ] || { echo "cpu-triage: window/interval must yield >= 2 samples" >&2; exit 2; } CLK=$(getconf CLK_TCK 2>/dev/null || echo 100) TMP=$(mktemp -d "${TMPDIR:-/tmp}/cpu-triage.XXXXXX") || exit 2 trap 'rm -rf "$TMP"' EXIT INT TERM CG_BASE="/sys/fs/cgroup/user.slice/user-$(id -u).slice/user@$(id -u).service/app.slice" # --------------------------------------------------------------------------- # Sampling. One loop, one cadence, every measure derived from the same instants # so per-core, per-process and per-service figures are directly comparable. # --------------------------------------------------------------------------- snapshot() { local idx="$1" # NOTE: every redirect below MUST be `>>`. Each sample is a fresh awk process, # and awk's `>` truncates the file once per invocation — so `>` silently keeps # only the LAST sample, leaving nothing to diff. That bug shipped into the # first test run and produced an empty per-core report with mean 0%. awk -v idx="$idx" -v out="$TMP" ' FILENAME == "/proc/stat" { if ($1 ~ /^cpu[0-9]+$/) { total = 0 for (i = 2; i <= NF; i++) total += $i # busy excludes idle (field 5) and iowait (field 6) print idx, $1, total - $5 - $6, total >> (out "/cores.raw") } if ($1 == "processes") print idx, $2 >> (out "/forks.raw") next } # /proc//stat — comm may contain spaces and parens; the greedy .*\) # anchors on the LAST close paren, which is the only safe split. { if (match($0, /^[0-9]+ \(.*\) /)) { pid = $1 rest = substr($0, RSTART + RLENGTH) n = split(rest, f, " ") if (n >= 13) print idx, pid, f[12] + f[13] >> (out "/procs.raw") } } ' /proc/stat /proc/[0-9]*/stat 2>/dev/null # Per-service cgroup CPU. Counts CHILD processes, which per-process sampling # attributes nowhere — the original wrangler churn lived entirely in children. # # Read every cpu.stat in ONE awk rather than a shell loop forking per service. # This tool reports fork rate, so its own churn is measurement noise it would # otherwise attribute to the box: the loop form cost ~16 forks per sample. if [ -d "$CG_BASE" ]; then awk -v idx="$idx" ' /^usage_usec/ { n = split(FILENAME, p, "/") svc = p[n-1]; sub(/\.service$/, "", svc) print idx, svc, $2 } ' "$CG_BASE"/*.service/cpu.stat >> "$TMP/cgroup.raw" 2>/dev/null fi } HOST=$(hostname 2>/dev/null || echo unknown) NCPU=$(awk '/^cpu[0-9]+/{n++} END{print n+0}' /proc/stat) STARTED=$(date -Is 2>/dev/null || date) if [ "$JSON" -eq 0 ]; then echo "cpu-triage on $HOST — sampling ${WINDOW}s at ${INTERVAL}s cadence (${SAMPLES} samples, ${NCPU} cores)" echo "this spans at least one 120s reconcile cycle; spot readings on this platform are not reliable" echo fi i=0 while [ "$i" -lt "$SAMPLES" ]; do snapshot "$i" i=$(( i + 1 )) [ "$i" -lt "$SAMPLES" ] && sleep "$INTERVAL" done ENDED=$(date -Is 2>/dev/null || date) [ -s "$TMP/cores.raw" ] || { echo "cpu-triage: no samples captured" >&2; exit 2; } # --------------------------------------------------------------------------- # Analysis — distribution per core, not a single number. # --------------------------------------------------------------------------- awk -v thr="$THRESHOLD" ' { key = $2 if (prev_total[key] != "") { dt = $4 - prev_total[key] db = $3 - prev_busy[key] if (dt > 0) { pct = db / dt * 100 sum[key] += pct; n[key]++ if (pct > peak[key]) peak[key] = pct if (pct > thr) hot[key]++ } } prev_total[key] = $4; prev_busy[key] = $3 } END { for (k in sum) printf "%s %.2f %.2f %.1f %d\n", k, sum[k]/n[k], peak[k], (hot[k]+0)/n[k]*100, n[k] } ' "$TMP/cores.raw" | sort -k2 -rn > "$TMP/cores.out" awk -v clk="$CLK" -v iv="$INTERVAL" ' { key = $2 if (prev[key] != "") { d = $3 - prev[key] if (d >= 0) { pct = d / (clk * iv) * 100 sum[key] += pct; n[key]++ if (pct > peak[key]) peak[key] = pct } } prev[key] = $3 } END { for (k in sum) if (sum[k]/n[k] >= 0.05) printf "%s %.2f %.2f\n", k, sum[k]/n[k], peak[k] } ' "$TMP/procs.raw" | sort -k2 -rn > "$TMP/procs.out" if [ -s "${TMP}/cgroup.raw" ]; then awk -v iv="$INTERVAL" ' { key = $2 if (prev[key] != "") { d = $3 - prev[key] if (d >= 0) { pct = d / (iv * 1000000) * 100; sum[key] += pct; n[key]++ if (pct > peak[key]) peak[key] = pct } } prev[key] = $3 } END { for (k in sum) printf "%s %.2f %.2f\n", k, sum[k]/n[k], peak[k] } ' "$TMP/cgroup.raw" | sort -k2 -rn > "$TMP/cgroup.out" fi FORKS_MIN=$(awk -v iv="$INTERVAL" 'NR==1{f=$2;i=$1} END{ if (NR>1) printf "%.0f", ($2-f)/((($1-i)*iv))*60 }' "$TMP/forks.raw" 2>/dev/null) [ -n "$FORKS_MIN" ] || FORKS_MIN=0 HOT_COUNT=$(awk -v s="$SUSTAIN" '$4 >= s {n++} END{print n+0}' "$TMP/cores.out") MEAN_CORE=$(awk '{s+=$2; n++} END{ if(n) printf "%.2f", s/n; else print "0" }' "$TMP/cores.out") resolve() { tr '\0' ' ' < "/proc/$1/cmdline" 2>/dev/null | cut -c1-78 || true; } # Control lookup, computed once for BOTH report shapes. It previously lived # inside the text branch only, which made --control silently inert under # --json — dropping the single most valuable input to this measurement from # the machine-readable output the agent reads. CONTROL_MEAN="" if [ -n "$CONTROL" ] && [ -s "${TMP}/cgroup.out" ]; then CONTROL_MEAN=$(awk -v c="$CONTROL" '$1 == c {print $2}' "$TMP/cgroup.out") fi # --------------------------------------------------------------------------- # Report # --------------------------------------------------------------------------- if [ "$JSON" -eq 1 ]; then printf '{\n "host": %s,\n "started": %s,\n "ended": %s,\n' "\"$HOST\"" "\"$STARTED\"" "\"$ENDED\"" printf ' "windowSec": %s, "intervalSec": %s, "samples": %s, "cores": %s,\n' "$WINDOW" "$INTERVAL" "$SAMPLES" "$NCPU" printf ' "thresholdPct": %s, "sustainPct": %s,\n' "$THRESHOLD" "$SUSTAIN" printf ' "meanPerCorePct": %s, "sustainedHotCores": %s, "forksPerMin": %s,\n' "$MEAN_CORE" "$HOT_COUNT" "$FORKS_MIN" printf ' "perCore": [' awk '{ printf "%s{\"cpu\":\"%s\",\"meanPct\":%s,\"peakPct\":%s,\"aboveThresholdPct\":%s}", (NR>1?",":""), $1,$2,$3,$4 }' "$TMP/cores.out" printf '],\n "topProcesses": [' head -n "$TOPN" "$TMP/procs.out" | while read -r pid mean peak; do cl=$(resolve "$pid" | sed 's/\\/\\\\/g; s/"/\\"/g') printf '{"pid":%s,"meanPct":%s,"peakPct":%s,"cmd":"%s"},' "$pid" "$mean" "$peak" "$cl" done | sed 's/,$//' printf '],\n "perService": [' if [ -s "${TMP}/cgroup.out" ]; then awk '{ printf "%s{\"service\":\"%s\",\"meanPct\":%s,\"peakPct\":%s}", (NR>1?",":""), $1,$2,$3 }' "$TMP/cgroup.out" fi printf '],\n "control": ' if [ -z "$CONTROL" ]; then printf 'null' else cesc=$(printf '%s' "$CONTROL" | sed 's/\\/\\\\/g; s/"/\\"/g') if [ -n "$CONTROL_MEAN" ]; then printf '{"brand":"%s","meanPct":%s,"found":true}' "$cesc" "$CONTROL_MEAN" else printf '{"brand":"%s","found":false}' "$cesc" fi fi printf '\n}\n' else echo "PER-CORE over the window (mean / peak / share of window above ${THRESHOLD}%)" echo " a core is 'sustained' when it exceeds threshold in >= ${SUSTAIN}% of samples" awk -v thr="$THRESHOLD" -v s="$SUSTAIN" ' $2 >= 1 || $4 > 0 { flag = ($4 >= s) ? " <== SUSTAINED" : "" printf " %-7s mean=%6.2f%% peak=%6.2f%% above=%5.1f%% of window%s\n", $1, $2, $3, $4, flag }' "$TMP/cores.out" echo " ---" printf " mean per-core %s%% sustained hot cores: %s of %s forks %s/min\n\n" "$MEAN_CORE" "$HOT_COUNT" "$NCPU" "$FORKS_MIN" echo "TOP PROCESSES by measured delta (100% = one full core; NOT a lifetime average)" head -n "$TOPN" "$TMP/procs.out" | while read -r pid mean peak; do printf " %6.2f%% mean %6.2f%% peak pid=%-8s %s\n" "$mean" "$peak" "$pid" "$(resolve "$pid")" done echo if [ -s "${TMP}/cgroup.out" ]; then echo "PER-SERVICE cgroup CPU (child processes included — the platform's true footprint)" tot=$(awk '{s+=$2} END{printf "%.2f", s}' "$TMP/cgroup.out") awk '$2 >= 0.01 { printf " %-32s mean=%6.2f%% peak=%6.2f%%\n", $1, $2, $3 }' "$TMP/cgroup.out" printf " ---\n all services %s%% of one core (%.2f%% of the %s-core box)\n\n" \ "$tot" "$(awk -v t="$tot" -v c="$NCPU" 'BEGIN{print t/c}')" "$NCPU" if [ -n "$CONTROL" ]; then if [ -n "$CONTROL_MEAN" ]; then echo "CONTROL: '$CONTROL' is on the previous build and measured ${CONTROL_MEAN}% of one core." echo " Compare peers against this, not against their own past readings. A peer that" echo " did not get the change is worth more than any before/after inference." echo else echo "CONTROL: '$CONTROL' not found among running services — comparison skipped."; echo fi fi fi if command -v sar >/dev/null 2>&1; then y=$(date -d yesterday +%d 2>/dev/null) if [ -n "$y" ] && [ -r "/var/log/sysstat/sa$y" ]; then echo "HISTORY — same instrument, yesterday, for like-for-like regression checking" sar -P ALL -f "/var/log/sysstat/sa$y" 2>/dev/null | awk -v thr="$THRESHOLD" ' /Average/ && $2 ~ /^[0-9]+$/ { b = 100 - $8; s += b; n++; if (b > thr) h++ } END { if (n) printf " yesterday: mean per-core %.2f%% cores above %s%%: %d/%d\n", s/n, thr, h+0, n }' echo fi fi if [ "$HOT_COUNT" -gt 0 ]; then echo "VERDICT: ${HOT_COUNT} core(s) sustained above ${THRESHOLD}%. Findings above." echo " Before calling it a regression, check the desktop stack in the process list." echo " On 2026-07-20 gnome-shell + gnome-system-monitor + Xorg accounted for ~41% of" echo " a core, roughly three times the entire four-brand platform, and the System" echo " Monitor window doing the observing was itself a third of that." echo " To attribute short-lived process churn, escalate (needs root):" echo " bpftrace -e 'tracepoint:syscalls:sys_enter_execve { @[comm, str(args->filename)] = count(); }'" else echo "VERDICT: no core sustained above ${THRESHOLD}%." fi fi [ "$HOT_COUNT" -gt 0 ] && exit 1 exit 0