/** * pi-agent — agentic engineering toolkit. * /agent prompt test "Your prompt" [model] → test a prompt and see output * /agent eval "question" "expected" → eval check (pass/fail) * /agent cost → estimate session token cost * /agent tokens "text" → estimate token count * /agent models → list available models * /agent context → show context window usage * /agent template name → save prompt template * /agent templates → list saved templates * /agent-strip → strip internal AI metadata tags * * Mario: the extension API IS the agent framework. Carmack: measure everything. * Kelsey: "If you can't observe it, you can't improve it." * Peter (steipete): batch parallel, triage at scale, structured JSON reports. */ import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; import { Type } from "@sinclair/typebox"; import { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync } from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; const SAVE_DIR = join(homedir(), ".pi", "agent-toolkit"); const TEMPLATES_DIR = join(SAVE_DIR, "templates"); const RST = "\x1b[0m", B = "\x1b[1m", D = "\x1b[2m"; const GREEN = "\x1b[32m", RED = "\x1b[31m", YELLOW = "\x1b[33m", CYAN = "\x1b[36m", MAGENTA = "\x1b[35m"; // Token estimation: ~4 chars per token for English text (GPT-like) function estimateTokens(text: string): number { return Math.ceil(text.length / 4); } function formatTokens(n: number): string { if (n > 1000000) return `${(n / 1000000).toFixed(1)}M`; if (n > 1000) return `${(n / 1000).toFixed(1)}K`; return String(n); } // Cost estimation per 1M tokens (approximate, mid-2025 pricing) const MODEL_COSTS: Record = { "claude-sonnet": { input: 3, output: 15 }, "claude-haiku": { input: 0.25, output: 1.25 }, "claude-opus": { input: 15, output: 75 }, "gpt-4o": { input: 2.5, output: 10 }, "gpt-4o-mini": { input: 0.15, output: 0.6 }, "gemini-pro": { input: 1.25, output: 5 }, "gemini-flash": { input: 0.075, output: 0.3 }, "deepseek-v3": { input: 0.27, output: 1.1 }, "deepseek-r1": { input: 0.55, output: 2.19 }, }; interface Template { name: string; prompt: string; vars: string[]; created: string } function loadTemplates(): Template[] { try { mkdirSync(TEMPLATES_DIR, { recursive: true }); return readdirSync(TEMPLATES_DIR) .filter(f => f.endsWith(".json")) .map(f => JSON.parse(readFileSync(join(TEMPLATES_DIR, f), "utf-8"))); } catch { return []; } } function saveTemplate(t: Template) { mkdirSync(TEMPLATES_DIR, { recursive: true }); writeFileSync(join(TEMPLATES_DIR, `${t.name}.json`), JSON.stringify(t, null, 2)); } export default function piAgent(pi: ExtensionAPI) { // /agent command pi.registerCommand("agent", { description: "Agentic engineering toolkit. /agent [tokens|cost|models|templates|context|eval|template]", handler: async (args, ctx) => { const parts = args.trim().split(/\s+/); const sub = parts[0]?.toLowerCase() || "help"; switch (sub) { case "tokens": case "token": { // Estimate tokens for input text const text = parts.slice(1).join(" "); if (!text) { ctx.ui.notify("Usage: /agent tokens \"your text here\"", "info"); return; } const stripped = text.replace(/^["']|["']$/g, ""); const tokens = estimateTokens(stripped); const chars = stripped.length; const words = stripped.split(/\s+/).length; ctx.ui.notify([ `${B}Token Estimate${RST}`, ` ${D}Characters:${RST} ${chars}`, ` ${D}Words:${RST} ${words}`, ` ${D}Tokens:${RST} ${B}~${formatTokens(tokens)}${RST}`, "", ` ${D}Cost at Sonnet rate: ~$${(tokens * 3 / 1000000).toFixed(4)} input${RST}`, ].join("\n"), "info"); break; } case "cost": case "pricing": { let out = `${B}${CYAN}💰 Model Pricing (per 1M tokens)${RST}\n\n`; out += ` ${"Model".padEnd(20)} ${"Input".padEnd(10)} ${"Output".padEnd(10)} $/1K in+out\n`; out += ` ${D}${"─".repeat(55)}${RST}\n`; for (const [name, cost] of Object.entries(MODEL_COSTS).sort((a, b) => a[1].input - b[1].input)) { const combined = ((cost.input + cost.output) / 1000).toFixed(4); const color = cost.input < 0.5 ? GREEN : cost.input < 5 ? YELLOW : RED; out += ` ${color}${name.padEnd(20)}${RST} $${String(cost.input).padEnd(9)} $${String(cost.output).padEnd(9)} ${D}$${combined}${RST}\n`; } out += `\n ${D}Prices from mid-2025. Check provider docs for current rates.${RST}`; ctx.ui.notify(out, "info"); break; } case "context": case "ctx": { // Show what's consuming context let out = `${B}${CYAN}📊 Context Window Analysis${RST}\n\n`; // Check settings.json const settingsPath = join(homedir(), ".pi", "agent", "settings.json"); if (existsSync(settingsPath)) { const settings = JSON.parse(readFileSync(settingsPath, "utf-8")); const packages = settings.packages || []; out += ` ${D}Loaded packages:${RST} ${B}${packages.length}${RST}\n`; } // Check skills const agentsPath = join(homedir(), ".pi", "agent", "AGENTS.md"); if (existsSync(agentsPath)) { const agents = readFileSync(agentsPath, "utf-8"); out += ` ${D}AGENTS.md:${RST} ${B}${estimateTokens(agents)}${RST} tokens (${agents.length} chars)\n`; } // Estimate skill catalog overhead out += `\n ${D}Skill catalog in system prompt:${RST} ~${B}5-8K${RST} tokens (105 skills)\n`; out += ` ${D}Extension commands:${RST} ~${B}1-2K${RST} tokens (50+ commands)\n`; out += ` ${D}Base system prompt:${RST} ~${B}2-3K${RST} tokens\n`; out += `\n ${YELLOW}Estimated total system context: ~10-15K tokens${RST}`; out += `\n ${D}Tip: Each loaded skill desc adds ~50-100 tokens. Keep descriptions tight.${RST}`; ctx.ui.notify(out, "info"); break; } case "eval": { // Simple eval: compare output against expected const question = parts[1]?.replace(/^["']|["']$/g, "") || ""; const expected = parts[2]?.replace(/^["']|["']$/g, "") || ""; if (!question || !expected) { ctx.ui.notify("Usage: /agent eval \"question\" \"expected substring\"", "info"); return; } ctx.ui.notify(`${D}Eval: "${question}" should contain "${expected}"${RST}\n${D}(Run the question through your agent and check manually)${RST}`, "info"); break; } case "template": case "tmpl": { // Save a prompt template const name = parts[1]; const prompt = parts.slice(2).join(" ").replace(/^["']|["']$/g, ""); if (!name || !prompt) { ctx.ui.notify("Usage: /agent template my-template \"You are a {{role}}. Do {{task}}.\"", "info"); return; } const vars = [...prompt.matchAll(/\{\{(\w+)\}\}/g)].map(m => m[1]); const template: Template = { name, prompt, vars, created: new Date().toISOString() }; saveTemplate(template); ctx.ui.notify(`${GREEN}✓ Template saved: ${B}${name}${RST}\n Vars: ${vars.length > 0 ? vars.map(v => `{{${v}}}`).join(", ") : "none"}`, "info"); break; } case "templates": case "tmpls": { const templates = loadTemplates(); if (templates.length === 0) { ctx.ui.notify("No templates saved. Use /agent template \"prompt\"", "info"); return; } let out = `${B}${CYAN}Prompt Templates${RST}\n\n`; for (const t of templates) { out += ` ${B}${t.name}${RST} ${D}(${t.vars.length > 0 ? t.vars.map(v => `{{${v}}}`).join(", ") : "no vars"})${RST}\n`; out += ` ${D}${t.prompt.slice(0, 80)}${t.prompt.length > 80 ? "..." : ""}${RST}\n\n`; } ctx.ui.notify(out, "info"); break; } case "scaffold": { // Generate a subagent definition scaffold const agentName = parts[1] || "my-agent"; const scaffold = [ `// Subagent definition: ${agentName}`, `// Save this and register with: pi_messenger({ action: "create", config: {...} })`, `{`, ` "name": "${agentName}",`, ` "description": "Short description of what this agent does",`, ` "systemPrompt": "You are ${agentName}. Your job is to...",`, ` "model": "anthropic/claude-sonnet-4",`, ` "tools": "read,bash,edit,write",`, ` "scope": "project"`, `}`, ].join("\n"); ctx.ui.notify(`${B}Subagent Scaffold: ${agentName}${RST}\n\n${scaffold}\n\n${D}Create with: subagent({ action: "create", config: })${RST}`, "info"); break; } case "patterns": { ctx.ui.notify([ `${B}${CYAN}🤖 Agentic Patterns Reference${RST}`, "", ` ${B}1. Scout → Plan → Build → Review${RST} (meta thread)`, ` ${D}Read code, make plan, implement, self-review. 4 phases.${RST}`, "", ` ${B}2. Parallel Dispatch${RST} (P-threads)`, ` ${D}N independent tasks → N workers → collect results.${RST}`, ` ${D}thread_spawn({ type: "parallel", prompts: [...] })${RST}`, "", ` ${B}3. Fusion / Multi-Model${RST} (F-threads)`, ` ${D}Same prompt → N models → compare & pick best.${RST}`, ` ${D}thread_spawn({ type: "fusion", prompts: [...], models: [...] })${RST}`, "", ` ${B}4. Zero-Touch + Verify${RST} (Z-threads)`, ` ${D}Agent loops autonomously, gated by test command.${RST}`, ` ${D}thread_spawn({ type: "zero", prompts: [...], verify: "npm test" })${RST}`, "", ` ${B}5. Crew (pi-messenger)${RST}`, ` ${D}Plan from PRD → task graph → parallel workers → review.${RST}`, ` ${D}pi_messenger({ action: "plan", prd: "docs/PRD.md" })${RST}`, "", ` ${B}6. Chain Pipeline${RST} (subagent chains)`, ` ${D}Step A output → Step B input → Step C input.${RST}`, ` ${D}subagent({ chain: [{agent:"a"}, {agent:"b"}] })${RST}`, "", ` ${D}steipete pattern: batch triage — dispatch N subagents to analyze N PRs${RST}`, ` ${D}Carmack pattern: tight loop — measure, fix, measure again${RST}`, ` ${D}Kelsey pattern: one golden path — don't multiply tools${RST}`, ].join("\n"), "info"); break; } default: { ctx.ui.notify([ `${B}${CYAN}🤖 Agent Toolkit${RST}`, "", ` /agent tokens "text" → estimate token count`, ` /agent cost → model pricing table`, ` /agent context → context window analysis`, ` /agent template name "…" → save prompt template`, ` /agent templates → list saved templates`, ` /agent scaffold [name] → generate subagent definition`, ` /agent patterns → agentic pattern reference`, ` /agent eval "q" "expect" → eval check`, ].join("\n"), "info"); } } }, }); // ── Strip internal metadata tags (openCC-inspired) ────────────────────── const HIDDEN_TAGS = ["commit_analysis", "context", "function_analysis", "reasoning", "confidence", "evidence", "internal_notes", "scratchpad", "thinking", "reflection"]; function stripMetadata(text: string): { cleaned: string; stripped: string[] } { const stripped: string[] = []; let cleaned = text; for (const tag of HIDDEN_TAGS) { const regex = new RegExp(`<${tag}>[\\s\\S]*?`, "gi"); const matches = cleaned.match(regex); if (matches) { stripped.push(`${tag} (${matches.length}×)`); cleaned = cleaned.replace(regex, "").trim(); } } // Collapse multiple blank lines cleaned = cleaned.replace(/\n{3,}/g, "\n\n"); return { cleaned, stripped }; } pi.registerCommand("agent-strip", { description: "Strip internal AI metadata tags from text. Usage: /agent-strip ", parameters: Type.Object({ args: Type.String() }), execute: async (params, ctx) => { const { cleaned, stripped } = stripMetadata(params.args); if (stripped.length === 0) { ctx.ui.notify("No metadata tags found — text is clean.", "info"); } else { ctx.ui.notify([ `${B}Stripped ${stripped.length} tag type(s):${RST} ${stripped.join(", ")}`, "", cleaned.slice(0, 500) + (cleaned.length > 500 ? "…" : ""), ].join("\n"), "info"); } }, }); pi.registerTool({ name: "strip_metadata", description: "Strip internal AI metadata tags (, , , etc.) from text. Returns cleaned text.", parameters: Type.Object({ text: Type.String({ description: "Text containing AI metadata tags to strip" }), }), execute: async (params) => { const { cleaned, stripped } = stripMetadata(params.text); return { cleaned, strippedTags: stripped, originalLength: params.text.length, cleanedLength: cleaned.length }; }, }); // LLM tool: token estimation pi.registerTool({ name: "estimate_tokens", description: "Estimate token count and cost for a text string", parameters: Type.Object({ text: Type.String({ description: "Text to estimate tokens for" }), model: Type.Optional(Type.String({ description: "Model name for cost estimate (default: claude-sonnet)" })), }), execute: async (params) => { const tokens = estimateTokens(params.text); const model = params.model || "claude-sonnet"; const costs = MODEL_COSTS[model] || MODEL_COSTS["claude-sonnet"]; return { tokens, characters: params.text.length, words: params.text.split(/\s+/).length, cost: { model, inputCost: `$${(tokens * costs.input / 1000000).toFixed(4)}`, outputCost: `$${(tokens * costs.output / 1000000).toFixed(4)}`, }, }; }, }); }