/** * `stats` — what the fleet accomplished and at what cost, over a window * (#282, legibility 5/5). * * Pure by construction: every fact arrives as injected run rows, a window and * a gh-call count, so the aggregation is testable without a daemon or a store * (the same seam split `doctor` uses). The CLI command in ./commands/stats.ts * supplies the rows through read-only store queries, then renders the same * numbers the --json shape carries. * * The unit of outcome is the **issue journey**, never the run row: an issue * that took six attempts is one merged outcome with six runs and one lead * time. Counting rows would make a stranded issue look like productive * throughput, which is the exact misreading this surface exists to prevent. * * Spend semantics follow the store's own convention (README Limitations, and * the doctor `spend-telemetry` finding): a run whose `spendUsd` reads 0.00 * means harness telemetry was absent, not that the run was free. Such runs are * counted separately as **unmetered** and never averaged into a per-issue cost. * * ## Attribution (Phase 4) * * Two additions, both about *where the money went* rather than what it bought: * * - **Per model.** Worker runs grouped by `resolvedModel` — the model the * harness actually ran and therefore the one that was billed. Never grouped * by the requested model: a requested model is not evidence of a resolved * one, and a row that observed none groups under {@link UNRESOLVED_MODEL} * rather than being guessed into a bucket. * - **Daemon-owned sessions.** To-spec grooming and review adjudication run as * the daemon's own sessions, not as worker runs, so their cost was logged and * then lost. They arrive here as `sessionSpendSince` rows and are summed *per * role*, deliberately outside the issue-journey aggregation: a role row in * `runs` would corrupt the merged/settled counts, which is why the store * keeps them in their own table. * * The orchestrator's own tick turns are absent from both, and that absence is * reported rather than hidden ({@link StatsReport.unmeteredRoles}): the tick * extension observes no usage events at all, so there is no number — and a * `$0.00` for it would read as free. */ import { utcDay } from "./store.ts"; import { SESSION_SPEND_ROLES } from "./types.ts"; import type { RunRecord, SessionSpendRole, SessionSpendRow } from "./types.ts"; /** The bounded window a report answers over. Day keys are UTC (`YYYY-MM-DD`). */ export interface StatsWindow { sinceDay: string; untilDay: string; sinceEpochMs: number; untilEpochMs: number; } /** One aggregate, per repo and for the whole selection. */ export interface RepoStats { /** The repo name; `"(all)"` on the totals row. */ repo: string; /** Distinct issues whose PR merged and settled inside the window. */ merged: number; /** `merged` plus issues whose terminal state inside the window did not merge. */ settled: number; /** `merged / settled`; `null` when nothing settled (never a fake 0). */ mergeRate: number | null; /** Mean attempts per merged issue, over the whole continuation chain. */ runsPerMerged: number | null; /** First queue-label claim → merge settlement, over merged issues, in ms. */ leadTimeMedianMs: number | null; leadTimeP90Ms: number | null; /** Metered (spendUsd > 0) spend across merged chains, rounded to cents. */ spendUsd: number; /** `spendUsd / merged issues with ≥1 metered run`; null when none. */ spendPerMerged: number | null; /** Runs examined with `spendUsd === 0` — telemetry absent, not free. */ unmeteredRuns: number; /** Merged issues whose whole chain metered nothing — cost unknown, not $0. */ unmeteredMerged: number; /** Terminal non-merged runs in the window, bucketed by failure class. */ failureClasses: Record; } /** * Worker runs attributed to one billed model (Phase 4). * * Counted over every run in the outcomes this report counts — the failures * included, because money spent losing is precisely what a per-model * comparison exists to expose: a model that burns a budget and never merges * must not read as cheap. * * So this is deliberately a *wider* figure than {@link RepoStats.spendUsd}, * which sums merged chains only. The two answer different questions ("what did * delivered work cost" versus "what did this model cost us"), and they are not * meant to add up to each other. */ export interface ModelStats { /** `resolvedModel`, or {@link UNRESOLVED_MODEL} when the run observed none. */ model: string; /** Runs in the window on this model, across every issue. */ runs: number; /** Runs whose state is `merged` — the attempt that actually delivered. */ merges: number; /** Metered (spendUsd > 0) spend on this model, rounded to cents. */ spendUsd: number; /** `spendUsd / merges`; `null` with no merge to divide by — never a zero, * and never a division by zero. */ spendPerMerge: number | null; /** Runs on this model whose spend read $0.00: telemetry absent, not free. */ unmeteredRuns: number; } /** One daemon-owned session role's accounting (Phase 4). */ export interface SessionRoleStats { role: SessionSpendRole; sessions: number; turns: number; /** Summed cost of the sessions that reported one, rounded to cents. */ spendUsd: number; /** * Sessions whose provider reported no cost at all. Distinct from a reported * `0` by contract (`SessionSpendRow.spendUsd` is optional for exactly this * reason), so this count is the honest "cost unknown" — never folded into * `spendUsd` as a zero. */ unmeteredSessions: number; } /** The stable report shape `--json` prints; the human renderer reads it too. */ export interface StatsReport { project: string; window: StatsWindow; /** Tracked GitHub API calls over the window's UTC days (#198). */ ghCalls: number; /** * True when nothing settled in the window: a fresh store or an idle fleet. * A consumer must render this as "no measurements", never as zero outcomes. */ empty: boolean; /** The whole selection (`--project NAME`, or the sole configured project). */ total: RepoStats; /** Per-repo aggregates of the same shape, busiest (by merges) first. */ repos: RepoStats[]; /** Worker runs by billed model, busiest (by runs) first (Phase 4). */ models: ModelStats[]; /** Daemon-owned session spend by role, in `SESSION_SPEND_ROLES` order. */ sessions: SessionRoleStats[]; /** * Metered spend on to-spec grooming sessions — the plan's `groomSpendUsd`, * named at the top level because "what did keeping the queue full cost" is a * question asked on its own. Same number as the `groom` row in * {@link StatsReport.sessions}; `groomUnmeteredSessions` is the part of it * whose cost is unknown rather than zero. */ groomSpendUsd: number; groomUnmeteredSessions: number; /** * Roles whose spend cannot be observed at all, so no row exists to sum and * no total here includes them. Present in the shape so a consumer adding * `total.spendUsd` to `sessions` knows what is still missing instead of * reading the sum as the fleet's whole bill. Today: `orchestrator` — the * tick extension observes no usage events. */ unmeteredRoles: string[]; } /** Terminal non-merged run states — everything the failure histogram counts. */ const FAILED_STATES: ReadonlySet = new Set([ "failed", "blocked", "killed", "orphaned", "stopped", ]); /** Runs with no classified failure fall into this explicit bucket. */ export const UNCLASSIFIED = "unclassified"; /** * Runs that never observed a resolved model group here (Phase 4). * * Parenthesised so it cannot collide with a real model id, and explicit rather * than omitted: a run predating the column, or one whose harness never reported * the model it ran, still cost money, and dropping it would make the per-model * spend quietly fail to add up to the total. */ export const UNRESOLVED_MODEL = "(unresolved)"; /** The timestamp a run is placed on the window timeline. Mirrors the store's * `COALESCE(endedAt, startedAt)` convention everywhere else in this package. */ const WINDOW_KEY = (run: RunRecord): number => run.endedAt ?? run.startedAt; /** One issue's settled outcome, reduced from its full attempt chain. */ interface IssueOutcome { repo: string; merged: boolean; /** Attempts in the whole chain (continuations collapse into one journey). */ attempts: number; /** First claim → merge settlement, in ms. Present iff `merged`. */ leadTimeMs?: number; /** Sum of metered (spendUsd > 0) spend across the chain, in USD. */ meteredSpend: number; /** Runs in the chain whose spend actually metered. */ meteredRuns: number; } function median(sorted: readonly number[]): number { if (sorted.length === 0) return Number.NaN; const lo = sorted[Math.floor((sorted.length - 1) / 2)]!; const hi = sorted[Math.ceil((sorted.length - 1) / 2)]!; return (lo + hi) / 2; } /** Nearest-rank p90: at least 90% of the samples are at or below the result. */ function p90(sorted: readonly number[]): number { if (sorted.length === 0) return Number.NaN; return sorted[Math.min(sorted.length - 1, Math.ceil(0.9 * sorted.length) - 1)]!; } function cents(usd: number): number { return Math.round(usd * 100) / 100; } function emptyRepo(repo: string): RepoStats { return { repo, merged: 0, settled: 0, mergeRate: null, runsPerMerged: null, leadTimeMedianMs: null, leadTimeP90Ms: null, spendUsd: 0, spendPerMerged: null, unmeteredRuns: 0, unmeteredMerged: 0, failureClasses: {}, }; } /** * Aggregate one repo's settled issues and failure runs into {@link RepoStats}. * Lead times and per-issue cost are reported only where a merged issue exists; * `null` is "not measured", never a zero that could read as a measurement. */ function summarize(args: { repo: string; outcomes: readonly IssueOutcome[]; failureRuns: readonly { repo: string; cls: string }[]; unmeteredRuns: number; }): RepoStats { const out = emptyRepo(args.repo); const merged = args.outcomes.filter((o) => o.merged); const leadTimes = merged .map((o) => o.leadTimeMs!) .sort((a, b) => a - b); const meteredMerged = merged.filter((o) => o.meteredRuns > 0); const meteredSpend = merged.reduce((sum, o) => sum + o.meteredSpend, 0); out.merged = merged.length; out.settled = args.outcomes.length; out.mergeRate = args.outcomes.length === 0 ? null : merged.length / args.outcomes.length; if (merged.length > 0) { out.runsPerMerged = merged.reduce((sum, o) => sum + o.attempts, 0) / merged.length; out.leadTimeMedianMs = Math.round(median(leadTimes)); out.leadTimeP90Ms = Math.round(p90(leadTimes)); } out.spendUsd = cents(meteredSpend); out.spendPerMerged = meteredMerged.length === 0 ? null : cents(meteredSpend / meteredMerged.length); out.unmeteredMerged = merged.filter((o) => o.meteredRuns === 0).length; out.unmeteredRuns = args.unmeteredRuns; for (const run of args.failureRuns) { out.failureClasses[run.cls] = (out.failureClasses[run.cls] ?? 0) + 1; } return out; } /** * Compute the report from store rows. `runs` is whatever the store's * stats-window query returned for the project: rows settling at or after * `window.sinceEpochMs`, plus the full chain of every issue whose merge * settled there. Rows within an issue must be ordered by `startedAt`, so the * first row of a chain is its first queue-label claim. */ /** `7d` / `30d` / any positive whole-day duration. */ const DURATION_FORM = /^(\d+)d$/; /** A bare UTC date: the window starts at 00:00:00Z that day. */ const DATE_FORM = /^\d{4}-\d{2}-\d{2}$/; const DAY_MS = 86_400_000; /** * `--since` (or `?since=`) to a full window, or `undefined` when it is not one. * * Here rather than in the CLI because the dashboard needs the identical parse * (#297): two parsers would eventually disagree about what `30d` means, and the * page's numbers are supposed to equal `stats --json` for the same window. The * CLI turns `undefined` into its exit-2 message and the endpoint turns it into a * 400 — neither silently defaults a typo'd window, because measuring the wrong * week and saying nothing is the failure this shape prevents. */ export function parseStatsWindow(raw: string, now: number): StatsWindow | undefined { const until = { untilEpochMs: now, untilDay: utcDay(now) }; const duration = DURATION_FORM.exec(raw); if (duration !== null) { const days = Number(duration[1]); if (days >= 1) { const sinceEpochMs = now - days * DAY_MS; return { sinceEpochMs, sinceDay: utcDay(sinceEpochMs), ...until }; } return undefined; } if (DATE_FORM.test(raw)) { const sinceEpochMs = Date.parse(`${raw}T00:00:00Z`); if (Number.isFinite(sinceEpochMs)) return { sinceEpochMs, sinceDay: raw, ...until }; } return undefined; } /** * `sessions` is `store.sessionSpendSince(project, window.sinceEpochMs)`. It is * optional because a caller with nothing to say about daemon-owned sessions — * and every pre-Phase-4 caller — must still get a report; an empty list means * "no grooming or adjudication in this window", which the renderer states as * that rather than as a $0.00 bill. */ export function computeStats(args: { project: string; window: StatsWindow; ghCalls: number; runs: readonly RunRecord[]; sessions?: readonly SessionSpendRow[]; }): StatsReport { const chains = new Map(); for (const run of args.runs) { const chain = chains.get(run.issue); if (chain === undefined) chains.set(run.issue, [run]); else chain.push(run); } const outcomes: IssueOutcome[] = []; const byRepo = new Map(); const failureRuns: { repo: string; cls: string }[] = []; let unmeteredRuns = 0; const unmeteredByRepo = new Map(); const byModel = new Map(); for (const chain of chains.values()) { // One issue merges once; if a weird history ever carried two merged rows, // the newest settlement wins. let merged: RunRecord | undefined; for (const run of chain) { if ( run.state === "merged" && (merged === undefined || WINDOW_KEY(run) > WINDOW_KEY(merged)) ) { merged = run; } } const terminatedInWindow = chain.some( (run) => FAILED_STATES.has(run.state) && WINDOW_KEY(run) >= args.window.sinceEpochMs, ); // A chain appears in the result for two reasons: its merge settled in the // window, or an individual run ended there. Without a merged row and // without a terminal row in the window it is still in flight — news, not // an outcome. if (merged === undefined && !terminatedInWindow) continue; for (const run of chain) { if (run.spendUsd === 0) { unmeteredRuns += 1; unmeteredByRepo.set(run.repo, (unmeteredByRepo.get(run.repo) ?? 0) + 1); } // Attributed over exactly the runs this report already counts, so the // per-model breakdown adds up to the totals printed beside it. Grouped on // the *resolved* model: that is the one that was billed. const model = modelBucket(byModel, run.resolvedModel ?? UNRESOLVED_MODEL); model.runs += 1; if (run.state === "merged") model.merges += 1; if (run.spendUsd > 0) model.spend += run.spendUsd; else model.unmetered += 1; if (merged === undefined && FAILED_STATES.has(run.state) && WINDOW_KEY(run) >= args.window.sinceEpochMs) { failureRuns.push({ repo: run.repo, cls: run.failureClass ?? UNCLASSIFIED }); } } const claimAt = chain[0]!.startedAt; const mergedAt = merged === undefined ? undefined : WINDOW_KEY(merged); const repo = merged?.repo ?? chain.find((r) => FAILED_STATES.has(r.state) && WINDOW_KEY(r) >= args.window.sinceEpochMs)!.repo; const outcome: IssueOutcome = { repo, merged: merged !== undefined, attempts: chain.length, leadTimeMs: mergedAt === undefined ? undefined : Math.max(0, mergedAt - claimAt), meteredSpend: chain.reduce((sum, run) => sum + (run.spendUsd > 0 ? run.spendUsd : 0), 0), meteredRuns: chain.filter((run) => run.spendUsd > 0).length, }; outcomes.push(outcome); const bucket = byRepo.get(repo); if (bucket === undefined) byRepo.set(repo, [outcome]); else bucket.push(outcome); } const repos = [...byRepo.entries()] .map(([repo, list]) => summarize({ repo, outcomes: list, failureRuns: failureRuns.filter((r) => r.repo === repo), unmeteredRuns: unmeteredByRepo.get(repo) ?? 0, }), ) .sort((a, b) => b.merged - a.merged || b.settled - a.settled || a.repo.localeCompare(b.repo)); const models = [...byModel.entries()] .map(([model, b]) => ({ model, runs: b.runs, merges: b.merges, spendUsd: cents(b.spend), // No merge to divide by is "not measured", never a zero and never a // division by zero — the same rule the per-issue cost follows. spendPerMerge: b.merges === 0 ? null : cents(b.spend / b.merges), unmeteredRuns: b.unmetered, })) .sort((a, b) => b.runs - a.runs || b.spendUsd - a.spendUsd || a.model.localeCompare(b.model)); const sessions = summarizeSessions(args.sessions ?? []); const groom = sessions.find((s) => s.role === "groom"); return { project: args.project, window: args.window, ghCalls: args.ghCalls, empty: outcomes.length === 0, total: summarize({ repo: "(all)", outcomes, failureRuns, unmeteredRuns }), repos, models, sessions, groomSpendUsd: groom?.spendUsd ?? 0, groomUnmeteredSessions: groom?.unmeteredSessions ?? 0, unmeteredRoles: [...UNMETERED_ROLES], }; } /** One model's accumulating totals while the chains are walked. */ interface ModelBucket { runs: number; merges: number; spend: number; unmetered: number; } function modelBucket(map: Map, model: string): ModelBucket { let found = map.get(model); if (found === undefined) { found = { runs: 0, merges: 0, spend: 0, unmetered: 0 }; map.set(model, found); } return found; } /** * Roles the fleet runs whose spend nothing can observe, so no row exists to * sum. The orchestrator session is the case: its tick extension reads content * text out of `message_start` and no usage event at all, so there is no number * to record. Named in the report rather than left out, because a total that * silently omits a whole role is the misreading this surface exists to prevent. */ const UNMETERED_ROLES = ["orchestrator"] as const; /** * Daemon-owned session rows reduced per role, in `SESSION_SPEND_ROLES` order so * the shape is stable whether or not a role ran in the window. * * A session whose `spendUsd` is absent is counted in `unmeteredSessions` and * contributes nothing to `spendUsd`: the store's contract distinguishes "the * provider reported no cost" from "the provider reported zero", and collapsing * them here would throw away the only place that distinction exists. */ function summarizeSessions(rows: readonly SessionSpendRow[]): SessionRoleStats[] { return SESSION_SPEND_ROLES.map((role) => { const mine = rows.filter((r) => r.role === role); const metered = mine.filter((r) => r.spendUsd !== undefined); return { role, sessions: mine.length, turns: mine.reduce((sum, r) => sum + r.turns, 0), spendUsd: cents(metered.reduce((sum, r) => sum + (r.spendUsd ?? 0), 0)), unmeteredSessions: mine.length - metered.length, }; }); } /** `N` and its unit, for a count line that must not print "0" as a measure. */ function countLine(n: number, noun: string, plural: string): string { return `${n} ${n === 1 ? noun : plural}`; } function humanDuration(ms: number): string { const s = Math.max(0, Math.round(ms / 1000)); if (s < 60) return `${s}s`; const m = Math.floor(s / 60); if (m < 60) return `${m}m ${String(s % 60).padStart(2, "0")}s`; const h = Math.floor(m / 60); if (h < 24) return `${h}h ${String(m % 60).padStart(2, "0")}m`; return `${Math.floor(h / 24)}d ${String(h % 24).padStart(2, "0")}h`; } function renderRepo(repo: RepoStats, indent: string): string { const lines: string[] = []; const leadTimeMedian = repo.leadTimeMedianMs; const leadTimeP90 = repo.leadTimeP90Ms; const lead = leadTimeMedian === null || leadTimeP90 === null ? "—" : `median ${humanDuration(leadTimeMedian)} · p90 ${humanDuration(leadTimeP90)}`; const cost = repo.spendPerMerged === null ? `$${repo.spendUsd.toFixed(2)} metered on merged work (no metered merge to divide by)` : `$${repo.spendUsd.toFixed(2)} metered ($${repo.spendPerMerged.toFixed(2)}/merged)`; const unmetered: string[] = []; if (repo.unmeteredRuns > 0) unmetered.push(`${countLine(repo.unmeteredRuns, "run", "runs")} unmetered`); if (repo.unmeteredMerged > 0) unmetered.push(`${countLine(repo.unmeteredMerged, "merged issue", "merged issues")} never metered`); const rates = repo.mergeRate === null ? "nothing settled" : `${(repo.mergeRate * 100).toFixed(1)}% of ${repo.settled} settled`; const attempts = repo.runsPerMerged === null ? "—" : `${repo.runsPerMerged.toFixed(2)} per merged issue`; lines.push(`${indent}${repo.repo} merged ${repo.merged} (${rates})`); lines.push(`${indent} lead ${lead} (first claim → merge)`); lines.push(`${indent} attempts ${attempts}`); lines.push(`${indent} cost ${cost}${unmetered.length === 0 ? "" : ` · ${unmetered.join(" · ")}`}`); const classes = Object.entries(repo.failureClasses).sort((a, b) => b[1] - a[1]); if (classes.length > 0) { lines.push(`${indent} failures ${classes.map(([cls, n]) => `${cls}: ${n}`).join(" · ")}`); } return lines.join("\n"); } /** * The attribution block (Phase 4): where the money went, as opposed to what it * bought. Rendered from the same report `--json` carries, and rendered even in * the empty case — a dry queue is exactly when daemon-owned grooming runs, so * hiding this behind "nothing settled" would hide the spend that had no * outcome, which is the one an operator most wants to see. */ function renderAttribution(report: StatsReport, indent: string): string[] { const lines: string[] = []; if (report.models.length > 0) { lines.push(`${indent}per model:`); for (const m of report.models) { // Nothing metered at all is stated as unknown, not as "$0.00 metered": // the dollar figure is true but reads as free, which is the exact // misreading this whole surface exists to prevent. const cost = m.spendUsd === 0 && m.unmeteredRuns === m.runs ? `cost unknown — no run on this model metered` : m.spendPerMerge === null ? `$${m.spendUsd.toFixed(2)} metered (no merge on this model to divide by)` : `$${m.spendUsd.toFixed(2)} metered ($${m.spendPerMerge.toFixed(2)}/merge)`; const unmetered = m.unmeteredRuns === 0 || m.unmeteredRuns === m.runs ? "" : ` · ${countLine(m.unmeteredRuns, "run", "runs")} unmetered`; lines.push( `${indent} ${m.model} ${countLine(m.runs, "run", "runs")}, ` + `${countLine(m.merges, "merge", "merges")} ${cost}${unmetered}`, ); } } const ran = report.sessions.filter((s) => s.sessions > 0); // Silence rather than a row of zeros: "no grooming sessions in this window" // and "grooming cost nothing" are different claims, and only the first is true. if (ran.length > 0) { lines.push(`${indent}daemon sessions:`); for (const s of ran) { // Same rule as the per-model line: an unmetered session's cost is // unknown, so it is a count, never a zero folded into the dollar figure. if (s.spendUsd === 0 && s.unmeteredSessions === s.sessions) { lines.push( `${indent} ${s.role} ${countLine(s.sessions, "session", "sessions")}, ` + `${countLine(s.turns, "turn", "turns")} cost unknown — no session metered`, ); continue; } const unknown = s.unmeteredSessions === 0 ? "" : ` · ${countLine(s.unmeteredSessions, "session", "sessions")} unmetered (cost unknown, not $0.00)`; lines.push( `${indent} ${s.role} ${countLine(s.sessions, "session", "sessions")}, ` + `${countLine(s.turns, "turn", "turns")} $${s.spendUsd.toFixed(2)} metered${unknown}`, ); } } if (report.unmeteredRoles.length > 0) { lines.push( `${indent} ${report.unmeteredRoles.join(", ")} unmetered — no usage events are observable for ` + `${report.unmeteredRoles.length === 1 ? "this session" : "these sessions"}, so the cost is unknown and no figure above includes it`, ); } return lines; } /** Render the human form of the same {@link StatsReport} `--json` prints. */ export function renderStatsHuman(report: StatsReport): string { const w = report.window; const header = `${report.project} — outcomes ${w.sinceDay} → ${w.untilDay}`; if (report.empty) { return [ header, " nothing settled in the window — no runs recorded, so there are no measurements to report yet (a fresh store or an idle fleet).", ...renderAttribution(report, " "), "", ].join("\n"); } const lines = [header, renderRepo(report.total, " "), ` gh api ${report.ghCalls} tracked ${report.ghCalls === 1 ? "call" : "calls"} in the window`]; if (report.repos.length > 0) { lines.push(" per repo:"); for (const repo of report.repos) lines.push(renderRepo(repo, " ")); } lines.push(...renderAttribution(report, " ")); return `${lines.join("\n")}\n`; }