/** * pi-tool-stats * * Tracks how often you actually use your skills, prompt templates, extensions, * and built-in tools so you can prune the dead weight and keep pi light. * See README.md for install + usage. * * What it records (one JSON line per event -> ~/.pi/agent/tool-stats.jsonl): * - tool : every LLM tool call, with model attribution when available * - tool-result : successful/failed tool outcomes, correlated by toolCallId * - skill : agent reads a known skill file, or you run /skill: * - template : you invoke a prompt template slash command * - ext-cmd : you invoke an extension-registered slash command * * Report: * /tool-stats [days] overview + prune candidates * /tool-stats model-tools [days] detailed model/tool effectiveness * Shows per-resource counts + last-used date, and a PRUNE CANDIDATES * section listing everything that exists but had 0 hits in the window. * * Storage is a plain append-only JSONL log: durable, greppable, survives * restarts. Delete the file to reset, or `tail -f` it to watch live. */ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import type { ExtensionAPI, ExtensionCommandContext, ExtensionContext, InputEvent, ToolCallEvent, ToolResultEvent, } from "@earendil-works/pi-coding-agent"; const LOG_FILE = path.join(os.homedir(), ".pi", "agent", "tool-stats.jsonl"); const GENERIC_ENTRY_DIRS = new Set([ "src", "dist", "lib", "build", "out", "source", ]); type Kind = "tool" | "tool-result" | "skill" | "template" | "ext-cmd"; export interface UsageRecord { ts: number; kind: Kind; name: string; ext?: string; // owning extension (for tool / ext-cmd), when known session?: string; // Model-attribution fields (added v0.2; absent in legacy records) provider?: string; model?: string; modelName?: string; thinking?: string; toolCallId?: string; // tool-result fields success?: boolean; durationMs?: number; } export interface CmdInfo { source: "extension" | "prompt" | "skill"; ext?: string; } export interface SkillDef { name: string; filePath: string; visible: boolean; promptChars: number; aliases: string[]; } export interface Agg { count: number; last: number; } // --------------------------------------------------------------------------- // helpers (pure functions exported for testing) // --------------------------------------------------------------------------- export function append(rec: UsageRecord): void { try { fs.appendFileSync(LOG_FILE, JSON.stringify(rec) + "\n"); } catch { // never let metrics break the agent } } export function packageNameNear(startDir: string): string | undefined { let dir = path.resolve(startDir); while (true) { try { const pkg = JSON.parse( fs.readFileSync(path.join(dir, "package.json"), "utf8"), ) as { name?: unknown }; if (typeof pkg.name === "string" && pkg.name.trim()) return pkg.name.trim(); } catch { /* no readable package.json here */ } const parent = path.dirname(dir); if (parent === dir) return undefined; dir = parent; } } export function localEntryExtensionName(p: string): string | undefined { let dir = path.dirname(p); const pkgName = packageNameNear(dir); if (pkgName) return pkgName; while (true) { const base = path.basename(dir); if (base && base !== "." && !GENERIC_ENTRY_DIRS.has(base)) return base; const parent = path.dirname(dir); if (parent === dir) return undefined; dir = parent; } } /** Friendly extension/package name from a sourceInfo.path. */ export function extNameFromPath(p?: string): string | undefined { if (!p) return undefined; // node_modules package: take @scope/name or name const nm = p.split(/[\\/]node_modules[\\/]/).pop(); if (nm && nm !== p) { const parts = nm.split(/[\\/]/); if (parts[0]?.startsWith("@") && parts[1]) return `${parts[0]}/${parts[1]}`; if (parts[0]) return parts[0]; } // local extension file or dir: ~/.pi/agent/extensions/(.ts|/index.ts) const m = p.match( /extensions[\\/]([^\\/]+?)(?:\.[tj]s)?(?:[\\/]index\.[tj]s)?$/, ); if (m) return m[1]; const base = path .basename(p) .replace(/\.[tj]s$/, "") .replace(/\.md$/, ""); if ( base === "index" || base === "SKILL" || GENERIC_ENTRY_DIRS.has(path.basename(path.dirname(p))) ) { return localEntryExtensionName(p) ?? path.basename(path.dirname(p)); } return base; } export function expandHome(p: string): string { if (p === "~") return os.homedir(); if (p.startsWith(`~${path.sep}`) || p.startsWith("~/")) return path.join(os.homedir(), p.slice(2)); return p; } export function normalizedPath(p: string, cwd?: string): string { const expanded = expandHome(p); return path.normalize( path.isAbsolute(expanded) ? expanded : path.resolve(cwd ?? process.cwd(), expanded), ); } export function filePathKeys(p?: string, cwd?: string): string[] { if (!p) return []; const keys = new Set(); try { const norm = normalizedPath(p, cwd); keys.add(norm); try { keys.add(fs.realpathSync(norm)); } catch { /* file may not exist in tests or stale command entries */ } } catch { /* ignore malformed paths */ } return [...keys]; } export function commandSkillName(commandName: string): string { return commandName.replace(/^skill:/i, ""); } /** Skill name if a read path points at a known skill file, else conservative fallback. */ export function skillNameFromReadPath( p?: string, cwd?: string, knownSkillFiles?: Map, ): string | undefined { for (const key of filePathKeys(p, cwd)) { const known = knownSkillFiles?.get(key); if (known) return known; } // Fallback for older/partial pi APIs: only infer from conventional SKILL.md paths. // Do not treat arbitrary *.md reads as skills; that creates false positives. if (!p) return undefined; const norm = p.replace(/\\/g, "/"); if (!/\/SKILL\.md$/i.test(norm)) return undefined; const parts = norm.split("/"); return parts[parts.length - 2]; } /** Leading slash-command token of an input line, lowercased, no leading slash. */ export function leadingCommand(text: string): string | undefined { const t = text.trim(); if (!t.startsWith("/")) return undefined; const tok = t.slice(1).split(/\s+/)[0]; return tok ? tok.toLowerCase() : undefined; } export function splitFrontmatter(raw: string): { frontmatter: string; body: string; } { const normalized = raw.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); if (!normalized.startsWith("---\n")) return { frontmatter: "", body: normalized }; const end = normalized.indexOf("\n---", 4); if (end === -1) return { frontmatter: "", body: normalized }; return { frontmatter: normalized.slice(4, end), body: normalized.slice(end + 4).trim(), }; } export function yamlStringField( frontmatter: string, key: string, ): string | undefined { const lines = frontmatter.split("\n"); for (let i = 0; i < lines.length; i++) { const m = lines[i]?.match(new RegExp(`^${key}:\\s*(.*)$`)); if (!m) continue; const value = (m[1] ?? "").trim(); if (value === ">" || value === "|") { const parts: string[] = []; for (let j = i + 1; j < lines.length; j++) { const line = lines[j] ?? ""; if (/^[a-zA-Z0-9_-]+:\s*/.test(line)) break; if (line.trim()) parts.push(line.trim()); } return value === ">" ? parts.join(" ") : parts.join("\n"); } return value.replace(/^['"]|['"]$/g, ""); } return undefined; } export function yamlBoolField(frontmatter: string, key: string): boolean { return new RegExp(`^${key}:\\s*true\\s*$`, "im").test(frontmatter); } export function skillAliases( name: string, filePath: string, extra: string[] = [], ): string[] { const aliases = new Set(); for (const value of [name, ...extra]) { if (value) aliases.add(value); } if (filePath) { const fileBase = path.basename(filePath, path.extname(filePath)); if (fileBase && fileBase !== "SKILL") aliases.add(fileBase); if (/SKILL\.md$/i.test(filePath)) { const dirBase = path.basename(path.dirname(filePath)); if (dirBase) aliases.add(dirBase); } } return [...aliases]; } export function estimateSkillPromptChars( name: string, description: string, filePath: string, ): number { // Matches pi's formatSkillsForPrompt() per docs/core implementation. return [ " ", ` ${name}`, ` ${description}`, ` ${filePath}`, " ", ].join("\n").length; } export function skillDefFromFile( filePath: string, fallbackName: string, ): SkillDef | undefined { try { const raw = fs.readFileSync(filePath, "utf8"); const { frontmatter } = splitFrontmatter(raw); const description = yamlStringField(frontmatter, "description"); if (!description || !description.trim()) return undefined; // pi does not load skills without descriptions const frontmatterName = yamlStringField(frontmatter, "name"); const name = frontmatterName || fallbackName; const visible = !yamlBoolField(frontmatter, "disable-model-invocation"); return { name, filePath, visible, promptChars: visible ? estimateSkillPromptChars(name, description, filePath) : 0, aliases: skillAliases(name, filePath, [ fallbackName, frontmatterName ?? "", ]), }; } catch { return undefined; } } export function fallbackSkillDef(name: string, filePath: string): SkillDef { return { name, filePath, visible: true, promptChars: 0, aliases: skillAliases(name, filePath), }; } export function withCanonicalName( def: SkillDef, canonicalName: string, ): SkillDef { return { ...def, name: canonicalName, promptChars: def.promptChars, aliases: skillAliases(canonicalName, def.filePath, [ def.name, ...def.aliases, ]), }; } export function addSkillDef( skillDefs: Map, def: SkillDef, ): void { const existing = skillDefs.get(def.name); if (!existing) { skillDefs.set(def.name, def); return; } skillDefs.set(def.name, { ...existing, filePath: existing.filePath || def.filePath, visible: existing.visible || def.visible, promptChars: Math.max(existing.promptChars, def.promptChars), aliases: [...new Set([...existing.aliases, ...def.aliases])], }); } export function discoverSkillDefs( root: string, includeRootFiles: boolean, ): SkillDef[] { const out: SkillDef[] = []; function walk(dir: string, isRoot: boolean): void { let entries: fs.Dirent[]; try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } const skillFile = path.join(dir, "SKILL.md"); if (fs.existsSync(skillFile)) { const def = skillDefFromFile(skillFile, path.basename(dir)); if (def) out.push(def); return; } if (isRoot && includeRootFiles) { for (const entry of entries) { if (entry.isFile() && entry.name.endsWith(".md")) { const def = skillDefFromFile( path.join(dir, entry.name), path.basename(entry.name, ".md"), ); if (def) out.push(def); } } } for (const entry of entries) { if ( !entry.isDirectory() || entry.name.startsWith(".") || entry.name === "node_modules" ) continue; walk(path.join(dir, entry.name), false); } } walk(root, true); return out; } export function fmtApproxTokens(chars: number): string { return chars > 0 ? `~${Math.max(1, Math.ceil(chars / 4))}` : "?"; } export function fmtRelatedUsage(count: number): string { return count > 0 ? `ext:${count}` : "-"; } // --------------------------------------------------------------------------- // reporting // --------------------------------------------------------------------------- export function readRecords(sinceMs?: number): UsageRecord[] { let raw: string; try { raw = fs.readFileSync(LOG_FILE, "utf8"); } catch { return []; } const out: UsageRecord[] = []; for (const line of raw.split("\n")) { if (!line.trim()) continue; try { const r = JSON.parse(line) as UsageRecord; if (sinceMs && r.ts < sinceMs) continue; out.push(r); } catch { /* skip bad line */ } } return out; } export function aggregate( records: UsageRecord[], keyOf: (r: UsageRecord) => string | undefined, ): Map { const m = new Map(); for (const r of records) { const k = keyOf(r); if (!k) continue; const cur = m.get(k) ?? { count: 0, last: 0 }; cur.count += 1; if (r.ts > cur.last) cur.last = r.ts; m.set(k, cur); } return m; } export function fmtDate(ts: number): string { if (!ts) return "never"; return new Date(ts).toISOString().slice(0, 10); } export function pad(s: string, n: number): string { return s.length >= n ? s : s + " ".repeat(n - s.length); } export function section( title: string, agg: Map, defined: Set, ): string[] { const lines: string[] = [`--- ${title} ---`]; const names = new Set([...agg.keys(), ...defined]); if (names.size === 0) { lines.push(" (none)"); return lines; } const rows = [...names].map((n) => ({ n, a: agg.get(n) ?? { count: 0, last: 0 }, })); rows.sort((a, b) => b.a.count - a.a.count || a.n.localeCompare(b.n)); const w = Math.max(8, ...rows.map((r) => r.n.length)); lines.push(` ${pad("name", w)} used last`); for (const { n, a } of rows) { const flag = a.count === 0 ? " <- prune?" : ""; lines.push( ` ${pad(n, w)} ${pad(String(a.count), 4)} ${fmtDate(a.last)}${flag}`, ); } return lines; } export function rpad(s: string, n: number): string { return s.length >= n ? s : s + " ".repeat(n - s.length); } export function lpad(s: string, n: number): string { return s.length >= n ? s : " ".repeat(n - s.length) + s; } export function modelKeyFrom(r: UsageRecord): string | undefined { const p = r.provider; const m = r.model; if (!p || !m) return undefined; return `${p}/${m}`; } export interface ModelAgg { key: string; displayName: string; calls: number; ok: number; fail: number; unk: number; toolCallCounts: Map; toolOkCounts: Map; toolFailCounts: Map; toolUnknownCounts: Map; } export interface ToolAgg { name: string; calls: number; ok: number; fail: number; unk: number; modelCalls: Map; modelOks: Map; modelFails: Map; modelUnknowns: Map; } export interface ModelToolStats { models: ModelAgg[]; tools: ToolAgg[]; } export type ToolStatsView = "overview" | "model-tools" | "help"; export interface ParsedToolStatsArgs { view: ToolStatsView; days: number; } function inc(m: Map, key: string, by = 1): void { m.set(key, (m.get(key) ?? 0) + by); } function toolCallRecordKey(r: UsageRecord): string | undefined { if (!r.toolCallId) return undefined; return `${r.session ?? ""}:${r.toolCallId}`; } function emptyModelAgg(key: string): ModelAgg { return { key, displayName: key, calls: 0, ok: 0, fail: 0, unk: 0, toolCallCounts: new Map(), toolOkCounts: new Map(), toolFailCounts: new Map(), toolUnknownCounts: new Map(), }; } function emptyToolAgg(name: string): ToolAgg { return { name, calls: 0, ok: 0, fail: 0, unk: 0, modelCalls: new Map(), modelOks: new Map(), modelFails: new Map(), modelUnknowns: new Map(), }; } /** * Compute per-model and per-tool aggregates from tool attempts. * * Windowing is based on the tool-call timestamp. Matching results are read from * the full record set by toolCallId so a result is not detached from its attempt * by independent timestamp filtering. */ export function computeModelToolStats( records: UsageRecord[], sinceMs = 0, ): ModelToolStats { const resultByCallId = new Map(); for (const r of records) { if (r.kind !== "tool-result") continue; const k = toolCallRecordKey(r); if (!k) continue; resultByCallId.set(k, r.success); } const modelMap = new Map(); const toolMap = new Map(); for (const r of records) { if (r.kind !== "tool") continue; if (sinceMs && r.ts < sinceMs) continue; const mk = modelKeyFrom(r); if (!mk) continue; let model = modelMap.get(mk); if (!model) { model = emptyModelAgg(mk); modelMap.set(mk, model); } let tool = toolMap.get(r.name); if (!tool) { tool = emptyToolAgg(r.name); toolMap.set(r.name, tool); } const resultKey = toolCallRecordKey(r); const result = resultKey ? resultByCallId.get(resultKey) : undefined; model.calls++; inc(model.toolCallCounts, r.name); tool.calls++; inc(tool.modelCalls, mk); if (result === true) { model.ok++; inc(model.toolOkCounts, r.name); tool.ok++; inc(tool.modelOks, mk); } else if (result === false) { model.fail++; inc(model.toolFailCounts, r.name); tool.fail++; inc(tool.modelFails, mk); } else { model.unk++; inc(model.toolUnknownCounts, r.name); tool.unk++; inc(tool.modelUnknowns, mk); } } const models = [...modelMap.values()].sort( (a, b) => b.calls - a.calls || a.key.localeCompare(b.key), ); const tools = [...toolMap.values()].sort( (a, b) => b.calls - a.calls || a.name.localeCompare(b.name), ); return { models, tools }; } export function fmtOkPct(ok: number, fail: number): string { const total = ok + fail; if (total === 0) return " -"; const pct = Math.round((ok / total) * 100); return `${pct.toString().padStart(3)}%`; } export function topToolList(counts: Map, n: number): string { return [...counts.entries()] .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) .slice(0, n) .map(([t]) => t) .join(", "); } export function weakSpot(fails: Map): string { let best: [string, number] | undefined; for (const [tool, count] of fails) { if (!best || count > best[1] || (count === best[1] && tool < best[0])) best = [tool, count]; } return best ? `${best[0]} ${best[1]} fail` : "-"; } export function renderModelToolSnapshot( stats: ModelToolStats, includeDetailHint = false, ): string[] { const lines: string[] = ["--- MODEL / TOOL SNAPSHOT ---"]; if (stats.models.length === 0) { lines.push(" (no model-attributed tool calls in window)"); return lines; } const colM = Math.max(15, ...stats.models.map((m) => m.displayName.length)); const colT = Math.max( 9, ...stats.models.map((m) => topToolList(m.toolCallCounts, 3).length), ); lines.push( ` ${rpad("model", colM)} ${lpad("calls", 5)} ${lpad("ok", 2)} ${lpad("fail", 4)} ${lpad("unk", 3)} ok% ${rpad("top tools", colT)} weak spot`, ); for (const m of stats.models) { lines.push( ` ${rpad(m.displayName, colM)} ${lpad(String(m.calls), 5)} ${lpad(String(m.ok), 2)} ${lpad(String(m.fail), 4)} ${lpad(String(m.unk), 3)} ${fmtOkPct(m.ok, m.fail)} ${rpad(topToolList(m.toolCallCounts, 3), colT)} ${weakSpot(m.toolFailCounts)}`, ); } if (includeDetailHint) { lines.push(" details: /tool-stats model-tools [days]"); } return lines; } function bestModelForTool(t: ToolAgg): string { let bestName = "-"; let bestScore = -1; let bestCalls = -1; for (const [mk, calls] of t.modelCalls) { const ok = t.modelOks.get(mk) ?? 0; const fail = t.modelFails.get(mk) ?? 0; const totalResults = ok + fail; if (totalResults === 0) continue; const score = ok / totalResults; if (score > bestScore || (score === bestScore && calls > bestCalls)) { bestScore = score; bestCalls = calls; bestName = mk; } } return bestName; } function worstModelForTool(t: ToolAgg): string { let worstName = "-"; let worstFails = 0; for (const [mk, fail] of t.modelFails) { if (fail > worstFails || (fail === worstFails && mk < worstName)) { worstFails = fail; worstName = mk; } } return worstFails > 0 ? `${worstName} ${worstFails} fail` : "-"; } export function renderModelToolDetailReport( stats: ModelToolStats, days: number, totalEvents: number, windowEvents: number, ): string[] { let totalAttempts = 0; let totalOk = 0; let totalFail = 0; let totalUnk = 0; for (const m of stats.models) { totalAttempts += m.calls; totalOk += m.ok; totalFail += m.fail; totalUnk += m.unk; } const lines: string[] = []; lines.push(`Model Tool Effectiveness (last ${days}d)`); lines.push( `Attempts: ${totalAttempts} Results: ${totalOk + totalFail} OK: ${totalOk} Fail: ${totalFail} Unknown: ${totalUnk} (log: ${totalEvents} events total, ${windowEvents} in window)`, ); lines.push( "Note: ok% is tool-execution success (not proof of task success).", ); lines.push(""); lines.push("--- BY MODEL ---"); if (stats.models.length === 0) { lines.push(" (no model-attributed tool calls in window)"); } else { const colM = Math.max(15, ...stats.models.map((m) => m.displayName.length)); const colT = Math.max( 9, ...stats.models.map((m) => topToolList(m.toolCallCounts, 3).length), ); lines.push( ` ${rpad("model", colM)} ${lpad("calls", 5)} ${lpad("ok", 2)} ${lpad("fail", 4)} ${lpad("unk", 3)} ok% tools ${rpad("top tools", colT)}`, ); for (const m of stats.models) { lines.push( ` ${rpad(m.displayName, colM)} ${lpad(String(m.calls), 5)} ${lpad(String(m.ok), 2)} ${lpad(String(m.fail), 4)} ${lpad(String(m.unk), 3)} ${fmtOkPct(m.ok, m.fail)} ${lpad(String(m.toolCallCounts.size), 5)} ${rpad(topToolList(m.toolCallCounts, 3), colT)}`, ); } } lines.push(""); lines.push("--- BY TOOL ---"); if (stats.tools.length === 0) { lines.push(" (no model-attributed tool events in window)"); } else { const colT = Math.max(8, ...stats.tools.map((t) => t.name.length)); const colBest = Math.max( 10, ...stats.tools.map((t) => bestModelForTool(t).length), ); const colWorst = Math.max( 11, ...stats.tools.map((t) => worstModelForTool(t).length), ); lines.push( ` ${rpad("tool", colT)} ${lpad("calls", 5)} ${lpad("ok", 2)} ${lpad("fail", 4)} ${lpad("unk", 3)} ok% ${rpad("best model", colBest)} ${rpad("worst model", colWorst)}`, ); for (const t of stats.tools) { lines.push( ` ${rpad(t.name, colT)} ${lpad(String(t.calls), 5)} ${lpad(String(t.ok), 2)} ${lpad(String(t.fail), 4)} ${lpad(String(t.unk), 3)} ${fmtOkPct(t.ok, t.fail)} ${rpad(bestModelForTool(t), colBest)} ${rpad(worstModelForTool(t), colWorst)}`, ); } } lines.push(""); lines.push("--- FAILURE HOTSPOTS ---"); type Hotspot = { model: string; tool: string; fail: number; calls: number; ok: number; }; const hotspots: Hotspot[] = []; for (const m of stats.models) { for (const [tool, fail] of m.toolFailCounts) { if (fail === 0) continue; hotspots.push({ model: m.displayName, tool, fail, calls: m.toolCallCounts.get(tool) ?? 0, ok: m.toolOkCounts.get(tool) ?? 0, }); } } hotspots.sort( (a, b) => b.fail - a.fail || b.calls - a.calls || a.model.localeCompare(b.model), ); if (hotspots.length === 0) { lines.push(" (no failures recorded)"); } else { const colM = Math.max(15, ...hotspots.map((h) => h.model.length)); const colT = Math.max(8, ...hotspots.map((h) => h.tool.length)); lines.push( ` ${rpad("model", colM)} ${rpad("tool", colT)} fail ${lpad("calls", 5)} ok%`, ); for (const h of hotspots) { lines.push( ` ${rpad(h.model, colM)} ${rpad(h.tool, colT)} ${lpad(String(h.fail), 4)} ${lpad(String(h.calls), 5)} ${fmtOkPct(h.ok, h.fail)}`, ); } } lines.push(""); const maxMatrixColumns = 7; const matrixTools = new Set(stats.tools.slice(0, 5).map((t) => t.name)); for (const m of stats.models) { for (const t of m.toolFailCounts.keys()) matrixTools.add(t); } const matrixToolList = [...matrixTools].slice(0, maxMatrixColumns); lines.push("--- MODEL × TOP TOOLS ---"); if (stats.models.length === 0 || matrixToolList.length === 0) { lines.push(" (no data)"); } else { const colM = Math.max(8, ...stats.models.map((m) => m.displayName.length)); const colT = Math.max(5, ...matrixToolList.map((t) => t.length)); const cw = colT + 3; lines.push( ` ${rpad("model", colM)}` + matrixToolList.map((t) => ` ${rpad(t, cw)}`).join(""), ); lines.push( ` ${" ".repeat(colM)}` + matrixToolList.map(() => ` ${lpad("ok/calls", cw)}`).join(""), ); for (const m of stats.models) { let row = ` ${rpad(m.displayName, colM)}`; for (const t of matrixToolList) { const calls = m.toolCallCounts.get(t) ?? 0; const ok = m.toolOkCounts.get(t) ?? 0; row += ` ${lpad(calls > 0 ? `${ok}/${calls}` : "-", cw)}`; } lines.push(row); } } return lines; } export function parseWindowDays( value: string | undefined, fallback = 30, ): number { if (!value || !value.trim()) return fallback; const n = parseInt(value, 10); return Number.isFinite(n) && n >= 0 ? n : fallback; } export function parseToolStatsArgs(args: string): ParsedToolStatsArgs { const tokens = args.trim().split(/\s+/).filter(Boolean); if (tokens.length === 0) return { view: "overview", days: 30 }; const first = tokens[0]?.toLowerCase() ?? ""; if (first === "help" || first === "--help" || first === "-h") { return { view: "help", days: 30 }; } if (["model-tools", "models", "tools", "mt"].includes(first)) { return { view: "model-tools", days: parseWindowDays(tokens[1]) }; } return { view: "overview", days: parseWindowDays(first) }; } export function toolStatsHelpLines(): string[] { return [ "Tool Stats commands:", "", " /tool-stats [days] overview + prune candidates", " /tool-stats model-tools [days] detailed model/tool effectiveness", " /tool-stats models [days] alias for model-tools", " /tool-stats tools [days] alias for model-tools", " /tool-stats mt [days] alias for model-tools", "", "days defaults to 30; use 0 for all time.", ]; } // --------------------------------------------------------------------------- // extension // --------------------------------------------------------------------------- export default function usageMetrics(pi: ExtensionAPI) { // command name -> classification, refreshed on session_start / resources_discover let cmdMap = new Map(); // tool name -> owning extension (only extension-sourced tools) let toolExt = new Map(); // normalized skill file path -> canonical skill command/name let skillFileToName = new Map(); // Pending tool-call attribution for tool_result correlation. // Bounded to prevent memory growth from blocked/cancelled calls. const MAX_PENDING = 500; const pendingCalls = new Map< string, { provider: string; model: string; modelName: string; thinking: string; ts: number; toolName: string; } >(); function refreshMaps(): void { try { cmdMap = new Map(); skillFileToName = new Map(); for (const c of pi.getCommands()) { cmdMap.set(c.name.toLowerCase(), { source: c.source, ext: c.source === "extension" ? extNameFromPath(c.sourceInfo?.path) : undefined, }); if (c.source === "skill") { const skillName = commandSkillName(c.name); for (const key of filePathKeys(c.sourceInfo?.path)) skillFileToName.set(key, skillName); } } } catch { /* ignore */ } try { toolExt = new Map(); for (const t of pi.getAllTools()) { const src = t.sourceInfo?.source; if (src && src !== "builtin" && src !== "sdk") { const name = extNameFromPath(t.sourceInfo?.path) ?? src; toolExt.set(t.name, name); } } } catch { /* ignore */ } } function sessionId(ctx: ExtensionContext): string | undefined { try { const f = ctx.sessionManager?.getSessionFile?.(); return f ? path.basename(f) : undefined; } catch { return undefined; } } pi.on("session_start", async () => { refreshMaps(); }); pi.on("resources_discover", async () => { refreshMaps(); }); // Tool calls (built-in + extension custom tools), plus skill-read detection. pi.on("tool_call", async (event: ToolCallEvent, ctx: ExtensionContext) => { try { const name = event.toolName; if (!name) return undefined; const session = sessionId(ctx); // skill read: agent loading a skill file. Prefer pi.getCommands() sourceInfo // so package skills, root .md skills, and name!=directory skills share one key. if (name === "read") { const p = (event.input as { path?: string })?.path; const skill = skillNameFromReadPath(p, ctx.cwd, skillFileToName); if (skill) append({ ts: Date.now(), kind: "skill", name: skill, session }); } const toolCallId = event.toolCallId; const provider = ctx.model?.provider ?? "unknown"; const modelId = ctx.model?.id ?? "unknown"; const modelDisplayName = ctx.model?.name ?? "unknown"; const thinking = pi.getThinkingLevel(); const ts = Date.now(); // Store for tool_result correlation, evicting oldest stale entries first. if (toolCallId) { while (pendingCalls.size >= MAX_PENDING) { const oldest = pendingCalls.keys().next().value as string | undefined; if (!oldest) break; pendingCalls.delete(oldest); } pendingCalls.set(toolCallId, { provider, model: modelId, modelName: modelDisplayName, thinking, ts, toolName: name, }); } append({ ts, kind: "tool", name, ext: toolExt.get(name), provider, model: modelId, modelName: modelDisplayName, thinking, toolCallId, session, }); } catch { /* never block */ } return undefined; // observe only, never block }); // Slash-command usage: skills, prompt templates, extension commands. pi.on("input", async (event: InputEvent, ctx: ExtensionContext) => { try { const tok = leadingCommand(event.text ?? ""); if (tok) { const session = sessionId(ctx); if (tok.startsWith("skill:")) { append({ ts: Date.now(), kind: "skill", name: commandSkillName(tok), session, }); } else { const info = cmdMap.get(tok); if (info?.source === "prompt") { append({ ts: Date.now(), kind: "template", name: tok, session }); } else if (info?.source === "skill") { append({ ts: Date.now(), kind: "skill", name: commandSkillName(tok), session, }); } else if (info?.source === "extension") { append({ ts: Date.now(), kind: "ext-cmd", name: tok, ext: info.ext, session, }); } } } } catch { /* ignore */ } return undefined; // pass through unchanged }); // Tool-result outcomes: correlate with pending tool calls to record // model-attributed success/failure. pi.on( "tool_result", async (event: ToolResultEvent, ctx: ExtensionContext) => { try { const pending = pendingCalls.get(event.toolCallId); if (!pending) return undefined; append({ ts: Date.now(), kind: "tool-result", name: event.toolName, provider: pending.provider, model: pending.model, modelName: pending.modelName, thinking: pending.thinking, toolCallId: event.toolCallId, success: !event.isError, durationMs: Date.now() - pending.ts, session: sessionId(ctx), }); pendingCalls.delete(event.toolCallId); } catch { /* ignore */ } return undefined; }, ); // ------------------------------------------------------------------------- // /tool-stats // ------------------------------------------------------------------------- pi.registerCommand("tool-stats", { description: "Show tool stats; use 'model-tools' for model/tool details", handler: async (args: string, ctx: ExtensionCommandContext) => { refreshMaps(); const parsed = parseToolStatsArgs(args); if (parsed.view === "help") { await ctx.ui.select("Tool Stats Help", toolStatsHelpLines()); return; } const days = parsed.days; const sinceMs = days > 0 ? Date.now() - days * 86_400_000 : 0; const windowRecs = readRecords(sinceMs); const allRecs = readRecords(); if (parsed.view === "model-tools") { const mtStats = computeModelToolStats(allRecs, sinceMs); await ctx.ui.select( `Model Tool Effectiveness (last ${days}d)`, renderModelToolDetailReport( mtStats, days, allRecs.length, windowRecs.length, ), ); return; } // Defined resource sets (what currently exists -> prune candidates). const skillDefs = new Map(); // Canonical source: pi's skill slash commands. This covers package skills, // root .md skills, non-SKILL.md skills, and name!=directory cases. for (const c of pi.getCommands()) { if (c.source !== "skill") continue; const skillName = commandSkillName(c.name); const filePath = c.sourceInfo?.path ?? ""; const parsed = filePath ? skillDefFromFile(filePath, skillName) : undefined; addSkillDef( skillDefs, withCanonicalName( parsed ?? fallbackSkillDef(skillName, filePath), skillName, ), ); } // Supplement for configurations with skill commands disabled or unusual paths. const skillRoots: Array<{ path: string; rootFiles: boolean }> = [ { path: path.join(os.homedir(), ".pi", "agent", "skills"), rootFiles: true, }, { path: path.join(os.homedir(), ".agents", "skills"), rootFiles: false, }, { path: path.join(ctx.cwd, ".pi", "skills"), rootFiles: true }, { path: path.join(ctx.cwd, ".agents", "skills"), rootFiles: false }, ]; for (const root of skillRoots) { for (const def of discoverSkillDefs(root.path, root.rootFiles)) addSkillDef(skillDefs, def); } const definedSkills = new Set(skillDefs.keys()); const definedTemplates = new Set(); const definedExts = new Set(); for (const c of pi.getCommands()) { if (c.source === "prompt") definedTemplates.add(c.name.toLowerCase()); else if (c.source === "extension") { const e = extNameFromPath(c.sourceInfo?.path); if (e) definedExts.add(e); } } const definedTools = new Set(); for (const t of pi.getAllTools()) { if (t.sourceInfo?.source === "builtin") definedTools.add(t.name); else { const e = extNameFromPath(t.sourceInfo?.path); if (e) definedExts.add(e); } } const skillAliasToName = new Map(); for (const def of skillDefs.values()) skillAliasToName.set(def.name, def.name); for (const def of skillDefs.values()) { for (const alias of def.aliases) if (!skillAliasToName.has(alias)) skillAliasToName.set(alias, def.name); } // Aggregations (over the window). const skillAgg = aggregate(windowRecs, (r) => r.kind === "skill" ? (skillAliasToName.get(r.name) ?? r.name) : undefined, ); const tmplAgg = aggregate(windowRecs, (r) => r.kind === "template" ? r.name : undefined, ); const builtinAgg = aggregate(windowRecs, (r) => r.kind === "tool" && !r.ext ? r.name : undefined, ); // Extension usage = its custom tool calls + its slash-command invocations. const extAgg = aggregate(windowRecs, (r) => (r.kind === "tool" || r.kind === "ext-cmd") && r.ext ? r.ext : undefined, ); const lines: string[] = []; lines.push( `Usage over last ${days}d (log: ${allRecs.length} events total, ${windowRecs.length} in window)`, ); lines.push(""); // ── Model / tool snapshot ────────────────────────────────────── const mtStats = computeModelToolStats(allRecs, sinceMs); if (mtStats.models.length > 0) { lines.push(...renderModelToolSnapshot(mtStats, true)); lines.push(""); } lines.push(...section("SKILLS", skillAgg, definedSkills)); lines.push(""); lines.push(...section("PROMPT TEMPLATES", tmplAgg, definedTemplates)); lines.push(""); lines.push(...section("EXTENSIONS", extAgg, definedExts)); lines.push(""); lines.push(...section("BUILT-IN TOOLS", builtinAgg, definedTools)); // Prune candidates summary. const prune = (agg: Map, defined: Set) => [...defined].filter((n) => (agg.get(n)?.count ?? 0) === 0).sort(); const ps = prune(skillAgg, definedSkills); const pt = prune(tmplAgg, definedTemplates); const pe = prune(extAgg, definedExts); lines.push(""); lines.push(`=== PRUNE CANDIDATES (0 hits in ${days}d) ===`); lines.push(` skills : ${ps.length ? ps.join(", ") : "(none)"}`); lines.push(` templates : ${pt.length ? pt.join(", ") : "(none)"}`); lines.push(` extensions: ${pe.length ? pe.join(", ") : "(none)"}`); const promptVisibleUnused = [...skillDefs.values()] .filter((s) => s.visible && (skillAgg.get(s.name)?.count ?? 0) === 0) .sort( (a, b) => b.promptChars - a.promptChars || a.name.localeCompare(b.name), ); lines.push(""); lines.push( `=== PROMPT-VISIBLE UNUSED SKILLS (0 skill loads in ${days}d) ===`, ); lines.push( " These are in and cost prompt tokens, but no read or /skill use was observed.", ); if (promptVisibleUnused.length === 0) { lines.push(" (none)"); } else { const w = Math.max(8, ...promptVisibleUnused.map((s) => s.name.length)); lines.push( ` ${pad("name", w)} approx prompt tokens related usage file`, ); for (const s of promptVisibleUnused) { const related = extNameFromPath(s.filePath); const relatedHits = related ? (extAgg.get(related)?.count ?? 0) : 0; lines.push( ` ${pad(s.name, w)} ${pad(fmtApproxTokens(s.promptChars), 20)} ${pad(fmtRelatedUsage(relatedHits), 13)} ${s.filePath}`, ); } } const explicitOnlyUnused = [...skillDefs.values()] .filter((s) => !s.visible && (skillAgg.get(s.name)?.count ?? 0) === 0) .map((s) => s.name) .sort(); if (explicitOnlyUnused.length > 0) { lines.push(""); lines.push("=== EXPLICIT-ONLY UNUSED SKILLS (hidden from model) ==="); lines.push(` ${explicitOnlyUnused.join(", ")}`); } // Display in a scrollable selector (does not pollute the LLM context). await ctx.ui.select(`Tool Stats (last ${days}d)`, lines); }, }); }