#!/usr/bin/env bash
# probe_live_eval.sh — the LIVE twin of /prompt-regression's static probe check.
#
# WHY THIS EXISTS. `plugins/fh-meta/skills/prompt-regression/SKILL.md` says this about itself,
# verbatim, in its own §Step 4: "What it therefore cannot catch — a rule that is still present but
# has stopped FIRING (salience loss, ordering, competition from another rule) — a trigger phrase
# present in the file but shadowed by a higher-priority route — any behavior change that leaves the
# source text identical." That gap is exactly what an eval-style harness closes (CLAUDE.md
# §Autonomous Initiative Layer cites the shape: settings-changed PR + a scored run against real
# tasks, not a source grep). This script IS that harness for FH's own golden probe set
# (.claude/regression/probes.md): it sends each selected probe's literal utterance to a floor-tier
# `claude -p` inside an ISOLATED, disposable clone (via scripts/sim_isolated_run.sh — never the live
# repo, per that script's own header) and greps the ACTUAL RESPONSE, not the source.
#
# WHAT IT IS NOT. It does not replace /prompt-regression (source-correctness stays cheap and
# instant) and it does not replace a blind sim-conductor persona run (which reads a whole artifact
# for open-ended judgment). This is narrow, mechanical, known-answer, and cheap enough to run
# nightly — the same trade CLAUDE.md's Measured-Loop memory entry describes for any recurring
# measurement: sealed pre-registration (probes_live.yaml is authored BEFORE a run, not fit to one),
# one variable per probe (ARM=primary utterance, CTRL=known-negative), a scorer that runs before
# results are read, and the falsification condition (a probe whose control does not discriminate
# is not scored PASS/FAIL — see §Scoring) executed exactly as it is written.
#
# SELECTION. Not every probes.md row is live-runnable — see .claude/regression/probes_live.yaml's
# header for the 4-step mechanical rule (class filter, utterance-shape filter, INERT-ANCHOR filter,
# hand-curated CLI-event filter) and scripts/probe_live_eval_lib.py for the implementation. Run
# `--dry-run` to see the full 33-row breakdown: which probes are selected, which are excluded and
# why, and which pass the mechanical rule but have no authored expect/control yet
# (NOT-YET-AUTHORED — an honest gap, not a silent drop). As of 2026-09-14 that bucket is EMPTY:
# G-LINT-01, its long-standing occupant, moved to ARM_CAPABILITY_EXCLUDE because the real blocker
# is that its route needs the Skill layer the arm does not have — an unauthored spec was never the
# reason. See probe_live_eval_lib.py for the three named exclude sets and what each one does and
# does not claim.
#
# COST. Each selected probe costs TWO live `claude -p` calls (primary + control) — THREE if it
# declares `control_input_b`, a second known-negative (only G-TRIG-07 does, as of 2026-09-14).
# At the current 13-probe selection that is 27 arm-calls per rep; the nightly plist passes
# `--reps 3`, so a full night is 81 live calls. Do not run the
# full selected set casually — use --subset N or --ids P1,P2 for a spot-check, and read
# sim_isolated_run.sh's own header before running unattended (isolation guarantees, what "observe"
# mode does and does not prevent, the three-valued rc/bytes verdict for a timeout vs an empty
# answer).
#
# 🟥 «미실행 ≠ 0» — a probe whose primary or control call produced 0 bytes (timeout, rate limit,
# crash) is scored FAILED-TO-RUN, excluded from the pass-rate denominator, and its count is
# reported separately. A FAILED-TO-RUN probe is not evidence the behavior is absent — it is
# evidence nothing was measured (CLAUDE.md §Instrument-Calibration: "not found ≠ 0").
#
# SCORING (see scripts/probe_live_eval_lib.py:score_probe for the exact rule). Each probe declares
# a `polarity` in probes_live.yaml:
#   present — expect_re must appear in PRIMARY and must NOT appear in CONTROL
#   absent  — expect_re must NOT appear in PRIMARY and MUST appear in CONTROL
# Either direction of "the control disagrees with what polarity predicts" scores that probe
# UNCALIBRATED, never PASS or FAIL — an instrument that cannot discriminate has not measured
# anything, per this repo's own instrument-calibration discipline. If ANY probe in a run is
# UNCALIBRATED, the WHOLE RUN's overall verdict is UNCALIBRATED (rc=2) — a pass rate computed
# alongside a proven-blind probe is not trustworthy just because the other rows look fine.
#
# EXIT CODES: 0 = PASS (pass_rate >= threshold, no UNCALIBRATED). 1 = FAIL (pass_rate < threshold).
# 2 = UNCALIBRATED, NO-PROBES-RAN, or a RUNNER PREFLIGHT FAILURE (sim_isolated_run.sh's own usage
# guards exit 2 before ever calling `claude` — e.g. no `claude` on PATH, or — measured 2026-09-05 —
# a launchd PATH with no `timeout(1)` on it). This script fails fast on the FIRST such rc=2 rather
# than burning the remaining clones against an environment already known broken: no verdict
# rendered either way — fix the instrument before trusting it. See sim_isolated_run.sh's own
# §timeout(1) RESOLUTION header for why that preflight can fail even when `claude` itself is fine.
#
# USAGE
#   bash scripts/probe_live_eval.sh --dry-run
#   bash scripts/probe_live_eval.sh --ids G-GREET-01,G-TRIG-02 --model sonnet
#   bash scripts/probe_live_eval.sh --subset 3
#   bash scripts/probe_live_eval.sh --subset 3 --report-out /tmp/spot.md   # spot-check: keep the nightly record untouched
#   bash scripts/probe_live_eval.sh                      # full selected set — nightly cron shape
#
# PORTABILITY: bash 3.2 (macOS) + bash 5.x (Linux CI). No associative arrays, no `${var,,}`,
# heredocs only inside functions with quoted delimiters (see [[feedback_unquoted_heredoc_backtick_executes]]
# — this script has none; the one heredoc-shaped block lives in probe_live_eval_lib.py, a real
# file, not a bash heredoc).

set -uo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PROBES_MD="$REPO_ROOT/.claude/regression/probes.md"
PROBES_LIVE="$REPO_ROOT/.claude/regression/probes_live.yaml"
LIB="$REPO_ROOT/scripts/probe_live_eval_lib.py"
# FH_SIM_RUNNER_BIN — same override name test_sim_isolated_run_lanes.sh already uses for the same
# purpose (point at an alternate runner build). Here it also lets test_probe_live_eval_lanes.sh
# swap in a stub runner (rc=2, no `claude` call, no network) to test the fail-fast behavior below
# without spawning a live session. No-op when unset — default behavior is unchanged.
SIM_RUNNER="${FH_SIM_RUNNER_BIN:-$REPO_ROOT/scripts/sim_isolated_run.sh}"

# ── file-header constant — the "문턱" the design brief calls for. Change here, not per-invocation. ──
THRESHOLD="0.8"

# ── EVIDENCE PRESERVATION (2026-09-14) ────────────────────────────────────────────────────────
# 🟥 WHY. Until today every run's response bodies were written into a `mktemp -d` and destroyed
# with it, and the launchd log carries only the runner's console tail — no answer text. So when
# 09-12 came back at pass_rate 0.18 and 09-13 came back 0/3 on EVERY probe, there was no way to
# tell a model-availability blip from a rate limit from a clone failure from a real regression.
# Both nights are now PERMANENTLY UNATTRIBUTABLE. That is the cost this block exists to stop
# paying: a nightly instrument that throws away its own evidence can report a number but can
# never explain one.
#
# WHAT IS KEPT, AND WHAT IS NOT. Only the FLAT artifacts each arm dir holds — the response `.txt`,
# its `.stderr.txt`, the `.prompt.txt`, the `.meta.tsv` (corpus_head_date / model / cutoff) and the
# runner console log. 🟥 NOT the clones: sim_isolated_run.sh puts each rep's full
# `git clone --local --no-hardlinks` under `<out>/w_<arm>_r<n>/repo`, and this repo's tracked tree
# is ~260MB, so preserving `--out` wholesale would deposit tens of GB per night. `find -maxdepth 1
# -type f` is what draws that line: the clone trees live at depth 2+ and are never copied.
#
# WHERE. tracks/_meta/live_eval_runs/<date>/ — gitignored via `.gitignore:45 tracks/**` (verified
# with `git check-ignore -v`), same zone as the report the run already writes.
EVIDENCE_ROOT_DEFAULT="tracks/_meta/live_eval_runs"
# Retention: a nightly full set is ~66 small text files (a few hundred KB). 14 days is chosen to
# span the weekly cadence with a margin, so a Monday session can still read the previous week's
# incident. Override with FH_LIVE_EVAL_KEEP_DAYS; 0 disables pruning entirely (never auto-deletes).
KEEP_DAYS="${FH_LIVE_EVAL_KEEP_DAYS:-14}"

MODEL="sonnet"
# ── reps — 프로브당 반복 횟수. 기본 1(종전 행동 그대로), 무인 런은 plist 에서 3 을 준다. ──
# 🟥 왜 1 이 기본이면서 야간은 3 인가 (2026-09-06 실측): 유효 런 3 개를 재채점하니 12 프로브 중
#   5 개가 flaky 였고, **같은 코퍼스·15분 간격**의 두 런에서 4 개가 뒤집혔다. 관측 pass_rate 는
#   0.50 / 0.67 / 0.67 — 단일 rep 의 노이즈 폭이 문턱 0.80 까지의 거리보다 넓다. 즉 reps=1 짜리
#   pass_rate 로는 문턱을 정할 수 없다. 채점은 과반이고, 분산은 리포트의 `reps(pass/ran)` 칸에
#   그대로 남긴다(3/3 과 2/3 은 다른 사실이다).
#   ⚠️ 비용: reps=3 이면 프로브당 `claude -p` 호출이 2 → 6 이다(12 프로브 = 24 → 72).
REPS=1
DRYRUN=0
SUBSET=""
IDS=""
OUTDIR=""
REPORT_OUT=""   # --report-out: where the markdown report lands (default: tracks/_meta/live_eval_<date>.md)
NO_EVIDENCE=0   # --no-evidence: skip the preserved-artifact copy (lanes / throwaway spot-checks)
while [ $# -gt 0 ]; do
  case "$1" in
    --subset)   SUBSET="${2:-}"; shift 2 ;;
    --ids)      IDS="${2:-}"; shift 2 ;;
    --model)    MODEL="${2:-sonnet}"; shift 2 ;;
    --reps)     REPS="${2:-1}"; shift 2 ;;
    --dry-run)  DRYRUN=1; shift ;;
    --out)      OUTDIR="${2:-}"; shift 2 ;;
    --report-out) REPORT_OUT="${2:-}"; shift 2 ;;   # lanes/spot-checks MUST pass this — never the live path
    --no-evidence) NO_EVIDENCE=1; shift ;;
    *) echo "unknown flag: $1" >&2; exit 2 ;;
  esac
done

case "$REPS" in ''|*[!0-9]*) echo "FAIL: --reps must be a positive integer (got '$REPS')" >&2; exit 2 ;; esac
[ "$REPS" -ge 1 ] || { echo "FAIL: --reps must be >= 1 (got '$REPS')" >&2; exit 2; }

command -v python3 >/dev/null 2>&1 || { echo "FAIL: python3 required" >&2; exit 2; }
[ -f "$PROBES_MD" ]   || { echo "FAIL: $PROBES_MD not found" >&2; exit 2; }
[ -f "$PROBES_LIVE" ] || { echo "FAIL: $PROBES_LIVE not found" >&2; exit 2; }
[ -f "$LIB" ]         || { echo "FAIL: $LIB not found" >&2; exit 2; }

WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/fh-live-eval-XXXXXX")"
SPEC_DIR="$WORKDIR/spec"
SELECT_JSON="$WORKDIR/select.json"

SELECT_ARGS=(select --probes-md "$PROBES_MD" --probes-live "$PROBES_LIVE"
             --json-out "$SELECT_JSON" --spec-dir "$SPEC_DIR")
[ -n "$SUBSET" ] && SELECT_ARGS+=(--subset "$SUBSET")
[ -n "$IDS" ]     && SELECT_ARGS+=(--ids "$IDS")

python3 "$LIB" "${SELECT_ARGS[@]}"
select_rc=$?
# select_rc nonzero = a DEAD-POINTER (probes_live.yaml names an id absent from probes.md). That is
# an authoring bug in this repo's own asset, not a runtime condition — fail loudly rather than
# silently running a smaller set than intended.
if [ "$select_rc" -ne 0 ]; then
  echo "" >&2
  echo "❌ selection reported a dead pointer — fix .claude/regression/probes_live.yaml before running." >&2
  rm -rf "$WORKDIR"
  exit "$select_rc"
fi

if [ "$DRYRUN" -eq 1 ]; then
  rm -rf "$WORKDIR"
  exit 0
fi

[ -f "$SPEC_DIR/selected_ids.txt" ] || { echo "FAIL: selection produced no spec dir" >&2; rm -rf "$WORKDIR"; exit 2; }
SELECTED_COUNT=$(wc -l < "$SPEC_DIR/selected_ids.txt" | tr -d ' ')
if [ "$SELECTED_COUNT" -eq 0 ]; then
  echo "❌ 0 probes selected after filtering — nothing to run (check --ids / --subset against the" >&2
  echo "   SELECTED list printed above)." >&2
  rm -rf "$WORKDIR"
  exit 2
fi

# `claude` presence is the REAL runner's preflight, so it is checked only on that path. Under
# FH_SIM_RUNNER_BIN the stub owns its own preflight — measured 2026-09-05 on CI (ubuntu, no
# `claude` installed): this line fired BEFORE the stub was ever called, so FF2/FF3/FF4/FF6 went
# red while FF1 passed by coincidence (both paths exit 2). Lane FF7 pins the seam; FF7b pins
# that the real path still refuses to run without `claude`.
if [ -z "${FH_SIM_RUNNER_BIN:-}" ]; then
  command -v claude >/dev/null 2>&1 || { echo "FAIL: claude CLI not on PATH — cannot run live" >&2; rm -rf "$WORKDIR"; exit 2; }
fi

RUN_DATE="$(date +%Y-%m-%d)"
OUTDIR="${OUTDIR:-$WORKDIR/run}"
mkdir -p "$OUTDIR"

# ── evidence dir + retention prune ────────────────────────────────────────────────────────────
EVIDENCE_DIR=""
if [ "$NO_EVIDENCE" -eq 0 ]; then
  # Overridable for the lanes ONLY — a lane that wrote into the live evidence path would be the
  # same class of defect as the 2026-09-05 incident where a lane overwrote that night's report
  # ([[feedback_sim_with_write_tools_is_a_fleet]]). Note the prune guard below deliberately
  # REFUSES to prune an overridden root, so this flag can never be used to point the delete at
  # something else.
  EVIDENCE_ROOT="${FH_LIVE_EVAL_EVIDENCE_ROOT:-$REPO_ROOT/$EVIDENCE_ROOT_DEFAULT}"
  mkdir -p "$EVIDENCE_ROOT"
  EVIDENCE_DIR="$EVIDENCE_ROOT/$RUN_DATE"
  # A second run on the same day gets its own dir rather than overwriting the first. The report
  # path already learned this lesson the hard way (2026-09-05 10:18, a lane overwrote the 02:30
  # record); evidence is the artifact you go back to *because* the record looked wrong, so losing
  # the earlier one is worse here, not better.
  [ -d "$EVIDENCE_DIR" ] && EVIDENCE_DIR="$EVIDENCE_ROOT/${RUN_DATE}_$(date +%H%M%S)"
  mkdir -p "$EVIDENCE_DIR"

  # Prune. Guarded three ways, because this is the one line in this script that deletes anything:
  #   (a) the root must be the literal path built above, under $REPO_ROOT/tracks/_meta
  #   (b) only DIRECTORIES exactly one level down whose name is date-shaped
  #   (c) only older than KEEP_DAYS; KEEP_DAYS=0 disables pruning outright
  # `-mtime +N` is portable across BSD (macOS) and GNU find; `-mindepth/-maxdepth 1` pins depth so
  # a nested path can never be reached.
  case "$EVIDENCE_ROOT" in
    "$REPO_ROOT/tracks/_meta/"*)
      if [ "$KEEP_DAYS" -gt 0 ] 2>/dev/null; then
        while IFS= read -r _old; do
          [ -n "$_old" ] || continue
          echo "  🧹 pruning evidence older than ${KEEP_DAYS}d: $_old"
          rm -rf "$_old"
        done < <(find "$EVIDENCE_ROOT" -mindepth 1 -maxdepth 1 -type d \
                      -name '20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' -mtime "+$KEEP_DAYS" 2>/dev/null)
      fi
      ;;
    *) echo "  ⚠️  evidence root outside tracks/_meta — refusing to prune: $EVIDENCE_ROOT" >&2 ;;
  esac
fi

# Copy ONLY the flat per-arm artifacts out of a probe dir. The full clone trees live at
# <probe_out>/w_<arm>_r<n>/repo/** — `-maxdepth 1 -type f` is what keeps them out (see the header
# block: preserving them wholesale would be tens of GB a night).
preserve_probe_evidence() {   # $1 = probe_out dir, $2 = probe id
  [ -n "$EVIDENCE_DIR" ] || return 0
  mkdir -p "$EVIDENCE_DIR/$2" || return 0
  find "$1" -maxdepth 1 -type f -exec cp {} "$EVIDENCE_DIR/$2/" \; 2>/dev/null || true
}

echo ""
echo "── live run: $SELECTED_COUNT probe(s), model=$MODEL, out=$OUTDIR ──────────────────────"

while IFS= read -r id; do
  [ -z "$id" ] && continue
  input_text="$(cat "$SPEC_DIR/$id.input.txt")"
  control_text="$(cat "$SPEC_DIR/$id.control.txt")"
  probe_out="$OUTDIR/$id"
  mkdir -p "$probe_out"

  echo ""
  echo "▶ $id — primary"
  bash "$SIM_RUNNER" --arm primary --reps "$REPS" --prompt "$input_text" \
       --mode observe --model "$MODEL" --out "$probe_out" \
       > "$probe_out/_runner_primary.log" 2>&1
  runner_rc=$?
  tail -n 6 "$probe_out/_runner_primary.log"
  # 🟥 fail-fast (2026-09-05) — rc=2 from the runner means its OWN usage/preflight guard tripped
  # before `claude` was ever invoked (missing --arm/--prompt, bogus --mode, no `claude` on PATH,
  # or — the incident this exists for — a launchd PATH with no `timeout(1)` resolvable). That
  # condition is identical for every remaining probe in this run, so continuing would just burn
  # the rest of the clones (up to 2*(N-1) more `claude -p` calls) to the same FAILED-TO-RUN wall.
  # Abort loudly instead of quietly producing a 12/12 FAILED-TO-RUN report with no clue why.
  if [ "$runner_rc" -eq 2 ]; then
    echo "" >&2
    echo "❌ $id primary runner call exited 2 (preflight failure, before \`claude\` ran)." >&2
    echo "   Aborting the whole run rather than burning the remaining clones." >&2
    echo "   Runner log:   $probe_out/_runner_primary.log" >&2
    echo "   Partial run artifacts kept at: $OUTDIR" >&2
    exit 2
  fi

  echo "▶ $id — control"
  bash "$SIM_RUNNER" --arm control --reps "$REPS" --prompt "$control_text" \
       --mode observe --model "$MODEL" --out "$probe_out" \
       > "$probe_out/_runner_control.log" 2>&1
  runner_rc=$?
  tail -n 6 "$probe_out/_runner_control.log"
  if [ "$runner_rc" -eq 2 ]; then
    echo "" >&2
    echo "❌ $id control runner call exited 2 (preflight failure, before \`claude\` ran)." >&2
    echo "   Aborting the whole run rather than burning the remaining clones." >&2
    echo "   Runner log:   $probe_out/_runner_control.log" >&2
    echo "   Partial run artifacts kept at: $OUTDIR" >&2
    exit 2
  fi

  # OPTIONAL second known-negative — a THIRD live call per rep, spent only where one control has
  # been measured to under-discriminate. Keyed off the spec file's existence, which the selector
  # writes only when `control_input_b` is declared (so "file absent" can only mean "not declared",
  # never "declared but empty").
  if [ -f "$SPEC_DIR/$id.control_b.txt" ]; then
    control_b_text="$(cat "$SPEC_DIR/$id.control_b.txt")"
    echo "▶ $id — control_b"
    bash "$SIM_RUNNER" --arm control_b --reps "$REPS" --prompt "$control_b_text" \
         --mode observe --model "$MODEL" --out "$probe_out" \
         > "$probe_out/_runner_control_b.log" 2>&1
    runner_rc=$?
    tail -n 6 "$probe_out/_runner_control_b.log"
    if [ "$runner_rc" -eq 2 ]; then
      echo "" >&2
      echo "❌ $id control_b runner call exited 2 (preflight failure, before \`claude\` ran)." >&2
      echo "   Aborting the whole run rather than burning the remaining clones." >&2
      echo "   Runner log:   $probe_out/_runner_control_b.log" >&2
      echo "   Partial run artifacts kept at: $OUTDIR" >&2
      exit 2
    fi
  fi

  preserve_probe_evidence "$probe_out" "$id"
done < "$SPEC_DIR/selected_ids.txt"

echo ""
# 🟥 The live report path is the nightly RECORD. A lane or spot-check that reaches this line with the
# default overwrote a real night's distribution once (2026-09-05 10:18: test_probe_live_eval_lanes.sh FF4
# ran the real script with a stub runner and replaced the 02:30 report). Lanes pass --report-out.
REPORT_PATH="${REPORT_OUT:-$REPO_ROOT/tracks/_meta/live_eval_${RUN_DATE}.md}"
python3 "$LIB" score \
  --probes-live "$PROBES_LIVE" \
  --select-json "$SELECT_JSON" \
  --ids-file "$SPEC_DIR/selected_ids.txt" \
  --run-root "$OUTDIR" \
  --threshold "$THRESHOLD" \
  --model "$MODEL" \
  --report-out "$REPORT_PATH" \
  --run-date "$RUN_DATE" \
  --reps "$REPS"
score_rc=$?

if [ -n "$EVIDENCE_DIR" ]; then
  # The report is part of the evidence: a preserved body set with no verdict beside it forces the
  # next reader to re-derive what the run concluded.
  cp "$REPORT_PATH" "$EVIDENCE_DIR/_report.md" 2>/dev/null || true
  _kept=$(find "$EVIDENCE_DIR" -type f 2>/dev/null | wc -l | tr -d ' ')
  _size=$(du -sh "$EVIDENCE_DIR" 2>/dev/null | cut -f1)
  echo ""
  echo "📦 response bodies preserved: $EVIDENCE_DIR  ($_kept files, ${_size:-?}; retention ${KEEP_DAYS}d)"
fi

echo ""
echo "run artifacts kept at: $OUTDIR"
echo "(temp selection workspace $WORKDIR is NOT auto-deleted when --out was passed explicitly; ok to remove by hand)"

exit "$score_rc"