/** * pi-verify v1.0.0 - Auto-verification for Pi * * Two-agent system: * 1. Builder runs in terminal * 2. Verifier watches session.jsonl * 3. Auto-injects fixes when issues found * * Usage: * pi run --verify → Start with verifier * /verify status → Check verification status */ import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; import { existsSync, readFileSync, writeFileSync, watchFile, unwatchFile } from "node:fs"; import { join, dirname } from "node:path"; const SESSION_FILE = join(process.cwd(), ".pi", "session.jsonl"); const SOCKET_FILE = join(process.cwd(), ".pi", "verify.sock"); const MAX_LOOPS = 3; interface VerificationResult { confidence: "PERFECT" | "VERIFIED" | "PARTIAL" | "FAILED"; issues: string[]; suggestions: string[]; } interface Turn { id: string; role: "user" | "assistant"; timestamp: string; files?: { path: string; action: string }[]; command?: string; output?: string; success?: boolean; } // Confidence grades const GRADES = { PERFECT: { color: "green", weight: 0 }, VERIFIED: { color: "cyan", weight: 1 }, PARTIAL: { color: "yellow", weight: 2 }, FAILED: { color: "red", weight: 3 } }; function verifyTurn(turn: Turn): VerificationResult { const issues: string[] = []; const suggestions: string[] = []; // Check 1: Files created exist if (turn.files) { for (const file of turn.files) { if (file.action === "create" || file.action === "modify") { if (!existsSync(file.path)) { issues.push(`File ${file.path} was not created`); } } } } // Check 2: Command success if (turn.success === false) { issues.push(`Command failed: ${turn.command}`); } // Check 3: Output sanity if (turn.output) { if (turn.output.includes("ERROR") || turn.output.includes("FATAL")) { issues.push("Error output detected"); } if (turn.output.includes("undefined is not a function")) { issues.push("Runtime error: undefined function"); } } // Generate suggestions if (issues.length > 0) { suggestions.push("Review the issues above and fix before continuing"); } // Determine confidence let confidence: VerificationResult["confidence"] = "PERFECT"; if (issues.length > 0) { confidence = "FAILED"; } else if (suggestions.length > 0) { confidence = "PARTIAL"; } return { confidence, issues, suggestions }; } function readLastTurn(): Turn | null { if (!existsSync(SESSION_FILE)) return null; const content = readFileSync(SESSION_FILE, "utf-8"); const lines = content.trim().split("\n").filter(Boolean); if (lines.length === 0) return null; try { return JSON.parse(lines[lines.length - 1]); } catch { return null; } } export default function init(pi: ExtensionAPI) { let verificationEnabled = false; let loopCount = 0; // /verify command pi.registerCommand("verify", { description: "Toggle verification mode or check status", handler: async (args: string) => { if (args === "on" || args === "enable") { verificationEnabled = true; loopCount = 0; return "✅ Verification enabled. Each turn will be auto-checked."; } if (args === "off" || args === "disable") { verificationEnabled = false; return "⏭️ Verification disabled."; } if (args === "status") { const lastTurn = readLastTurn(); if (!lastTurn) { return "📋 No session data yet."; } const result = verifyTurn(lastTurn); const grade = GRADES[result.confidence]; let status = `🔍 Verification Status\n\n`; status += `Confidence: ${result.confidence}\n`; status += `Loops: ${loopCount}/${MAX_LOOPS}\n`; if (result.issues.length > 0) { status += `\n❌ Issues:\n`; for (const issue of result.issues) { status += ` - ${issue}\n`; } } if (result.suggestions.length > 0) { status += `\n💡 Suggestions:\n`; for (const sug of result.suggestions) { status += ` - ${sug}\n`; } } return status; } return `🔍 Verification\n\nUsage: /verify [on|off|status]`; }, }); // Auto-verify after each turn (if enabled) pi.on("turn_complete", async (event, ctx) => { if (!verificationEnabled) return; const lastTurn = readLastTurn(); if (!lastTurn) return; const result = verifyTurn(lastTurn); if (result.confidence === "FAILED" || result.confidence === "PARTIAL") { loopCount++; if (loopCount >= MAX_LOOPS) { verificationEnabled = false; await ctx.ui.notify("⚠️ Max verification loops reached. Escalating to human.", "warning"); return; } // Inject feedback const feedback = result.issues.join("; "); ctx.ui.notify(`🔄 Verification feedback (${loopCount}/${MAX_LOOPS}): ${feedback}`, "info"); // Send to builder await pi.sendUserMessage(`Please fix the following issues: ${feedback}`, { deliverAs: "followUp" }); } }); // Session end cleanup pi.on("session_end", async () => { if (verificationEnabled) { verificationEnabled = false; loopCount = 0; } }); console.log("[pi-verify] Loaded - use /verify to toggle"); }