/** * Adversarial verifier role: prompts, verdict parsing, majority aggregation. * Bias-to-refute: ties and malformed verdicts count as refuted. */ export interface VerifierPromptInput { objective: string; planMarkdown: string; gaps?: string[]; evidenceIndex?: string; } export interface VerifierVerdict { refuted: boolean; gaps: string[]; confidence?: string; } /** * Build an adversarial skeptic prompt. Default to refute if uncertain. */ export function buildVerifierPrompt(input: VerifierPromptInput): string { const prior = input.gaps && input.gaps.length > 0 ? input.gaps.map((g, i) => `${i + 1}. ${g}`).join("\n") : "(none — first verification round)"; const evidence = input.evidenceIndex?.trim() ? input.evidenceIndex : "(missing — treat incomplete evidence index as grounds to refute)"; return [ "You are an **adversarial verifier** for the pi-goal-expander harness.", "You are NOT the agent that produced the work. Your job is to **refute** that", "the objective has been met. **Default to refuted: true if uncertain** — a", "false-positive (passing broken work) ends the loop wrongly and is far worse", "than one more iteration.", "", "## Inputs", "", `OBJECTIVE: ${input.objective}`, "", "PLAN:", "```markdown", input.planMarkdown, "```", "", "PRIOR_GAPS:", prior, "", "EVIDENCE_INDEX:", evidence, "", "## Rules", "", "1. OBJECTIVE + acceptance criteria + verification plan are the immutable contract.", "2. AUDIT the implementer's evidence — do NOT author parallel proof.", "3. No test theater: tests must drive the real shipped path.", "4. Missing or incomplete evidence index → refute.", "5. Weakened acceptance criteria vs baseline → refute.", "6. On re-verification, primarily check prior gaps are fixed; do not raise the bar.", "", "## Output contract", "", "End with a JSON block:", "```json", '{ "refuted": true|false, "gaps": ["..."], "confidence": "low|medium|high" }', "```", "Also include a clear line: `Refuted` or `Not Refuted`.", ].join("\n"); } /** * Parse a verifier response for Refuted / Not Refuted + optional JSON gaps. * Malformed / ambiguous → treated as refuted by callers (bias-to-refute). */ export function parseVerifierVerdict(text: string): VerifierVerdict { const raw = text ?? ""; // Prefer fenced or bare JSON object with refuted field const jsonMatch = raw.match(/```(?:json)?\s*(\{[\s\S]*?"refuted"[\s\S]*?\})\s*```/i) ?? raw.match(/(\{[^{}]*"refuted"\s*:\s*(?:true|false)[^{}]*\})/i); if (jsonMatch?.[1]) { try { const obj = JSON.parse(jsonMatch[1]) as { refuted?: boolean; gaps?: unknown; confidence?: string; }; const gaps = Array.isArray(obj.gaps) ? obj.gaps.map((g) => String(g)).filter(Boolean) : []; if (typeof obj.refuted === "boolean") { return { refuted: obj.refuted, gaps, confidence: typeof obj.confidence === "string" ? obj.confidence : undefined, }; } } catch { /* fall through to token scan */ } } // Token scan — prefer explicit phrases const notRefuted = /\bnot\s+refuted\b/i.test(raw); const refuted = /\brefuted\b/i.test(raw); if (notRefuted && !/\brefuted\s*:\s*true\b/i.test(raw)) { return { refuted: false, gaps: [] }; } if (refuted) { // Pull simple gap bullets if present const gaps = extractGapBullets(raw); return { refuted: true, gaps }; } // Ambiguous → bias-to-refute return { refuted: true, gaps: ["Malformed or missing verdict (treated as refuted)"], }; } function extractGapBullets(text: string): string[] { const lines = text.split(/\r?\n/); const gaps: string[] = []; let inGaps = false; for (const line of lines) { if (/^\s*#{0,3}\s*gaps?\b/i.test(line) || /"gaps"\s*:/i.test(line)) { inGaps = true; continue; } if (inGaps && /^\s*[-*]\s+/.test(line)) { gaps.push(line.replace(/^\s*[-*]\s+/, "").trim()); continue; } if (inGaps && line.trim() === "") { if (gaps.length > 0) break; } } return gaps; } /** * Aggregate N skeptic verdicts with bias-to-refute. * - Majority refuted → not achieved * - Tie → not achieved (bias-to-refute) * - Majority not-refuted → achieved */ export function aggregateVerdicts( verdicts: Array<{ refuted: boolean; gaps: string[] }>, ): { achieved: boolean; gaps: string[] } { if (verdicts.length === 0) { return { achieved: false, gaps: ["No verifier verdicts (treated as not achieved)"] }; } let refuteVotes = 0; let passVotes = 0; const gapSet: string[] = []; const seen = new Set(); for (const v of verdicts) { if (v.refuted) { refuteVotes += 1; for (const g of v.gaps) { const key = g.trim(); if (key && !seen.has(key)) { seen.add(key); gapSet.push(key); } } } else { passVotes += 1; } } // Bias-to-refute on ties: achieved only when pass strictly outnumbers refute const achieved = passVotes > refuteVotes; if (!achieved && gapSet.length === 0) { gapSet.push("Majority (or tie) of verifiers refuted achievement"); } return { achieved, gaps: gapSet }; } /** Stable fingerprint for stall detection (order-independent). */ export function fingerprintGaps(gaps: string[]): string { return gaps .map((g) => g.trim().toLowerCase()) .filter(Boolean) .sort() .join("|"); }