import * as fs from "node:fs";
import * as path from "node:path";
import type { Static } from "@earendil-works/pi-ai";
import { Type } from "@earendil-works/pi-ai";
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
import type { Model } from "@earendil-works/pi-ai";
import {
createAgentSession,
createExtensionRuntime,
defineTool,
SessionManager,
SettingsManager,
type ExtensionContext,
type ResourceLoader,
} from "@earendil-works/pi-coding-agent";
import type { GoalRecord, GoalTask, GoalTaskList } from "./goal-record.ts";
import { loadGoalSettings, type GoalSettings } from "./goal-settings.ts";
export interface AuditorProgress {
/** Current tool being executed by the auditor, if any */
currentTool?: string;
/** Arguments passed to the current tool (truncated for display) */
currentToolArgs?: string;
/** When the current tool started (ms since epoch) */
currentToolStartedAt?: number;
/** Recent text output lines from the auditor's assistant messages */
recentOutput: string[];
/** Phase of the audit */
phase: "running" | "tool_executing" | "producing_report" | "thinking" | "done";
/** Elapsed ms since audit started */
elapsedMs: number;
/** Current step label shown to the user (e.g. "Inspecting files...") */
label?: string;
/** Completion percentage from 0 to 100 */
percentage?: number;
}
export type AuditorProgressCallback = (progress: AuditorProgress) => void;
export interface GoalAuditorResult {
approved: boolean;
disapproved: boolean;
output: string;
model?: string;
thinkingLevel?: ThinkingLevel;
error?: string;
}
const THINKING_LEVELS = new Set(["off", "minimal", "low", "medium", "high", "xhigh"]);
function asNonEmptyString(value: unknown): string | undefined {
return typeof value === "string" && value.trim() ? value.trim() : undefined;
}
function asThinkingLevel(value: unknown): ThinkingLevel | undefined {
const text = asNonEmptyString(value);
return text && THINKING_LEVELS.has(text) ? text as ThinkingLevel : undefined;
}
export function parseAuditorDecision(output: string): { approved: boolean; disapproved: boolean } {
const approved = //.test(output);
const disapproved = //.test(output);
return { approved: approved && !disapproved, disapproved };
}
export interface AuditorVerificationEvidence {
/** The agent's verification summary describing what was checked. */
summary: string;
/** The goal's verification contract (what the agent was required to verify), if any. */
contract?: string;
}
function renderAuditorTaskTree(tasks: GoalTask[], indent: number): string[] {
const prefix = " ".repeat(indent);
const lines: string[] = [];
for (const task of tasks) {
const marker = task.status === "complete" ? "[x]" : task.status === "skipped" ? "[~]" : "[ ]";
lines.push(`${prefix}${marker} ${task.id}: ${task.title}`);
if (task.subtasks && task.subtasks.length > 0) {
lines.push(...renderAuditorTaskTree(task.subtasks, indent + 1));
}
}
return lines;
}
function countAuditorTasks(tasks: GoalTask[]): { total: number; complete: number; skipped: number; pending: number } {
let total = 0;
let complete = 0;
let skipped = 0;
for (const t of tasks) {
total++;
if (t.status === "complete") complete++;
else if (t.status === "skipped") skipped++;
if (t.subtasks && t.subtasks.length > 0) {
const child = countAuditorTasks(t.subtasks);
total += child.total;
complete += child.complete;
skipped += child.skipped;
}
}
return { total, complete, skipped, pending: total - complete - skipped };
}
function taskSummaryBlock(taskList?: GoalTaskList | null): string {
if (!taskList || taskList.tasks.length === 0) return "";
const { total, complete, skipped, pending } = countAuditorTasks(taskList.tasks);
const lines: string[] = [`Tasks: ${complete}/${total} complete${skipped > 0 ? `, ${skipped} skipped` : ""}`];
lines.push(...renderAuditorTaskTree(taskList.tasks, 0));
const gate = taskList.blockCompletion && pending > 0 ? " | TASK GATE: pending tasks block completion" : "";
lines[0] = lines[0]! + gate;
return lines.join("\n");
}
export function buildGoalAuditorPrompt(args: {
goal: GoalRecord;
completionSummary?: string | null;
detailedSummary: string;
verificationSummary?: string | null;
settings?: GoalSettings;
}): string {
return [
"You are the independent completion auditor for pi-goal.",
"The executor claims the goal is complete. Your job is to decide whether the user's objective is actually satisfied.",
"Be skeptical and semantic. Do not approve from paperwork, intent, file count, word count, build success, or a plausible summary alone.",
"Use read/grep/find/ls/bash as needed to inspect real artifacts. Do not mutate files or run destructive commands.",
"If the work is only an alpha scaffold, generated template, shallow draft, proxy milestone, or lacks the user-facing value requested, disapprove.",
"If any explicit requirement is missing, weakly verified, contradicted, or not inspectable with the available evidence, disapprove.",
"Return a concise audit report. The final line MUST be exactly one of:",
"",
"",
"",
"Goal objective:",
"",
args.goal.objective,
"",
"",
"Executor completion claim:",
"",
args.completionSummary?.trim() || "(none provided)",
"",
"",
"Current goal metadata:",
"",
args.detailedSummary,
...(!args.settings?.disableTasks && taskSummaryBlock(args.goal.taskList) ? ["", taskSummaryBlock(args.goal.taskList)] : []),
"",
...(args.verificationSummary?.trim() ? [
"",
"Executor verification summary:",
"",
args.verificationSummary.trim(),
"",
] : []),
...(!args.settings?.disableContracts && args.goal.verificationContract?.trim() ? [
"",
"Goal verification contract (what the executor was required to verify):",
"",
args.goal.verificationContract.trim(),
"",
] : []),
"",
"Audit checklist:",
...[
"1. Extract the real success criteria from the objective, including quality/reader outcomes.",
"2. Inspect artifacts or command output that can prove or disprove those criteria.",
...(args.verificationSummary?.trim()
? ["3. Check the against real artifacts. If the executor claims to have run tests or searched for references, verify those claims with actual file/shell evidence. The summary is a claim, not proof — cross-check it."]
: []),
...(!args.settings?.disableContracts && args.goal.verificationContract?.trim()
? ["4. Verify that the executor has satisfied every item in the . If any item is missing or weakly addressed, disapprove."]
: []),
"5. Explain missing or weak evidence, especially scaffold-vs-final quality gaps.",
"6. End with exactly only if the objective is truly complete; otherwise end with exactly .",
],
"",
"Progress reporting:",
"You have the report_auditor_progress tool available to report your progress to the user.",
"Please use it at natural phase boundaries:",
" - When starting: report_auditor_progress(label='Starting audit...', percentage=0)",
" - When beginning file inspection: report_auditor_progress(label='Inspecting files...', percentage=25)",
" - When verifying success criteria: report_auditor_progress(label='Verifying success criteria...', percentage=50)",
" - When evaluating evidence: report_auditor_progress(label='Evaluating evidence...', percentage=75)",
" - When producing final report: report_auditor_progress(label='Producing report...', percentage=90)",
"This is purely for user visibility and does not affect the audit outcome.",
].join("\n");
}
/** Tool name for auditor progress reporting */
export const REPORT_AUDITOR_PROGRESS_TOOL_NAME = "report_auditor_progress";
/** Parameters for the report_auditor_progress tool */
export const reportAuditorProgressParams = Type.Object({
label: Type.String({ description: "Current step label describing what the auditor is doing (e.g. 'Inspecting files...', 'Verifying success criteria...', 'Producing report...')" }),
percentage: Type.Number({ description: "Completion percentage from 0 to 100", minimum: 0, maximum: 100 }),
});
function makeAuditorResourceLoader(): ResourceLoader {
return {
getExtensions: () => ({ extensions: [], errors: [], runtime: createExtensionRuntime() }),
getSkills: () => ({ skills: [], diagnostics: [] }),
getPrompts: () => ({ prompts: [], diagnostics: [] }),
getThemes: () => ({ themes: [], diagnostics: [] }),
getAgentsFiles: () => ({ agentsFiles: [] }),
getSystemPrompt: () => [
"You are a read-only completion auditor running in an isolated pi agent session.",
"Inspect the repository and decide whether the claimed goal completion is genuinely satisfied.",
"Never modify files. Never approve unless the actual user objective is complete.",
"",
"You have the report_auditor_progress tool available. Use it to report your audit progress",
"to the user at natural phase boundaries (starting, inspecting files, verifying criteria,",
"producing report). This helps the user understand what the auditor is doing and how far",
"along it is.",
].join("\n"),
getAppendSystemPrompt: () => [],
extendResources: () => {},
reload: async () => {},
};
}
function resolveAuditorModel(ctx: ExtensionContext, config: GoalSettings): { model: Model | undefined; error?: string } {
if (!config.model && !config.provider) return { model: ctx.model };
if (config.provider && config.model) {
const model = ctx.modelRegistry.find(config.provider, config.model);
return model ? { model } : { model: undefined, error: `Configured auditor model not found: ${config.provider}/${config.model}` };
}
if (config.provider) {
const matches = ctx.modelRegistry.getAvailable().filter((model) => model.provider === config.provider);
return matches[0] ? { model: matches[0] } : { model: undefined, error: `No available auditor model for provider: ${config.provider}` };
}
if (!config.model) return { model: ctx.model };
const slash = config.model.indexOf("/");
if (slash > 0) {
const provider = config.model.slice(0, slash);
const modelId = config.model.slice(slash + 1);
const model = ctx.modelRegistry.find(provider, modelId);
return model ? { model } : { model: undefined, error: `Configured auditor model not found: ${config.model}` };
}
const matches = ctx.modelRegistry.getAvailable().filter((model) => model.id === config.model || model.name === config.model);
if (matches.length === 1) return { model: matches[0] };
return { model: undefined, error: `Configured auditor model is ambiguous or unavailable: ${config.model}` };
}
function modelLabel(model: Model | undefined): string | undefined {
return model ? `${model.provider}/${model.id}` : undefined;
}
export async function runGoalCompletionAuditor(args: {
ctx: ExtensionContext;
goal: GoalRecord;
completionSummary?: string | null;
detailedSummary: string;
verificationSummary?: string | null;
settings?: GoalSettings;
signal?: AbortSignal;
onProgress?: AuditorProgressCallback;
/**
* Optional factory for creating the auditor agent session.
* Exposed for testing so a mock/controllable session can be injected.
* Defaults to the real createAgentSession from @earendil-works/pi-coding-agent.
*/
createSession?: typeof createAgentSession;
}): Promise {
const config = loadGoalSettings(args.ctx.cwd);
const resolved = resolveAuditorModel(args.ctx, config);
const model = resolved.model;
const thinkingLevel = config.thinkingLevel;
const outputParts: string[] = [];
if (resolved.error) {
return { approved: false, disapproved: true, output: "", model: modelLabel(model), thinkingLevel, error: resolved.error };
}
try {
const createSession = args.createSession ?? createAgentSession;
const startedAt = Date.now();
const progress: AuditorProgress = {
recentOutput: [],
phase: "running",
elapsedMs: 0,
};
function emitProgress(): void {
progress.elapsedMs = Date.now() - startedAt;
args.onProgress?.({ ...progress });
}
// Build the report_auditor_progress tool, capturing the progress state
const reportProgressTool = defineTool({
name: REPORT_AUDITOR_PROGRESS_TOOL_NAME,
label: "Report Auditor Progress",
description: "Report current progress of the audit to the user. Call this at natural phase boundaries (starting, inspecting files, verifying criteria, producing report) to keep the user informed.",
promptSnippet: "Report current audit progress (step label and completion percentage) to the user.",
promptGuidelines: [
"Use report_auditor_progress at natural phase boundaries during the audit:",
" - When starting the audit: label='Starting audit...' percentage=0",
" - When beginning file inspection: label='Inspecting files...' percentage=25",
" - When verifying success criteria: label='Verifying success criteria...' percentage=50",
" - When evaluating evidence: label='Evaluating evidence...' percentage=75",
" - When producing final report: label='Producing report...' percentage=90",
"This is purely for user visibility — it does not affect the audit outcome.",
"Do not call this tool more than once every few seconds to avoid flooding.",
],
parameters: reportAuditorProgressParams,
executionMode: "sequential",
async execute(_toolCallId, params) {
const { label, percentage } = params as Static;
progress.label = label;
progress.percentage = percentage;
progress.phase = "running";
emitProgress();
return {
content: [{ type: "text", text: `Progress reported: ${label} (${percentage}%)` }],
details: {},
};
},
});
const { session } = await createSession({
cwd: args.ctx.cwd,
model,
thinkingLevel,
modelRegistry: args.ctx.modelRegistry,
resourceLoader: makeAuditorResourceLoader(),
sessionManager: SessionManager.inMemory(args.ctx.cwd),
settingsManager: SettingsManager.inMemory({ compaction: { enabled: false } }),
tools: ["read", "grep", "find", "ls", "bash", REPORT_AUDITOR_PROGRESS_TOOL_NAME],
customTools: [reportProgressTool],
});
const unsubscribe = session.subscribe((event) => {
if (event.type === "tool_execution_start") {
progress.currentTool = event.toolName;
progress.currentToolArgs = typeof event.args === "object" && event.args !== null
? JSON.stringify(event.args).slice(0, 120)
: String(event.args ?? "").slice(0, 120);
progress.currentToolStartedAt = Date.now();
progress.phase = "tool_executing";
emitProgress();
return;
}
if (event.type === "tool_execution_end") {
progress.currentTool = undefined;
progress.currentToolArgs = undefined;
progress.currentToolStartedAt = undefined;
progress.phase = "running";
emitProgress();
return;
}
if (event.type === "message_update") {
// Check for thinking events from the assistant stream
const streamEvent = (event as any).assistantMessageEvent;
if (streamEvent?.type === "thinking_start") {
progress.phase = "thinking";
if (!progress.label) progress.label = "Analyzing goal...";
emitProgress();
return;
}
if (streamEvent?.type === "thinking_end") {
progress.phase = "running";
emitProgress();
return;
}
// For text content, show producing_report phase
progress.phase = "producing_report";
const message = event.message as any;
if (message?.role === "assistant") {
for (const part of message.content ?? []) {
if (part.type === "text" && typeof part.text === "string" && part.text.trim()) {
// Keep the last 5 non-empty text lines for live display
const lines = part.text.split("\n").filter((l: string) => l.trim());
progress.recentOutput = [...lines.slice(-5)];
}
}
}
emitProgress();
return;
}
if (event.type !== "message_end") return;
const message = event.message as any;
if (message.role !== "assistant") return;
for (const part of message.content ?? []) {
if (part.type === "text" && typeof part.text === "string") outputParts.push(part.text);
}
// Show the accumulated output in progress
const fullText = outputParts.join("\n\n");
const lines = fullText.split("\n").filter((l: string) => l.trim());
progress.recentOutput = lines.slice(-8);
emitProgress();
});
// Wire the external AbortSignal to abort the running session when fired
// This is the mechanism that makes Esc-to-skip actually stop the auditor.
const abortSession = () => { session.abort(); };
args.signal?.addEventListener("abort", abortSession, { once: true });
// Emit initial progress
progress.label = "Starting audit...";
progress.percentage = 0;
emitProgress();
try {
if (args.signal?.aborted) return { approved: false, disapproved: true, output: "", model: modelLabel(model), thinkingLevel, error: "Auditor aborted." };
await session.prompt(buildGoalAuditorPrompt(args));
} finally {
args.signal?.removeEventListener("abort", abortSession);
progress.phase = "done";
progress.label = "Audit complete.";
progress.percentage = 100;
emitProgress();
unsubscribe();
}
// session.abort() does NOT throw — the agent loop returns normally with
// whatever output was captured before the abort. Check the signal after
// prompt completes and treat any abort as auditor-aborted regardless of
// whether an exception propagated.
if (args.signal?.aborted) {
return {
approved: false,
disapproved: true,
output: outputParts.join("\n\n").trim(),
model: modelLabel(model),
thinkingLevel,
error: "Auditor aborted.",
};
}
const output = outputParts.join("\n\n").trim();
const decision = parseAuditorDecision(output);
return { ...decision, output, model: modelLabel(model), thinkingLevel };
} catch (error) {
const isAborted = args.signal?.aborted || (error instanceof Error && error.name === "AbortError");
return {
approved: false,
disapproved: true,
output: outputParts.join("\n\n").trim(),
model: modelLabel(model),
thinkingLevel,
error: isAborted ? "Auditor aborted." : (error instanceof Error ? error.message : String(error)),
};
}
}