/** * Headless / QA harness UX — avoid Phase 0 stalls and multi-hour plan debate loops. */ import { spawnSync } from "node:child_process"; import { constants } from "node:fs"; import { access, mkdir, readdir, readFile, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { parse as parseYaml, stringify as stringifyYaml } from "yaml"; import { isHarnessNonInteractive } from "./ask-user/policy.js"; import { canAutoApprovePlan, isHarnessPlanAutoApproveEnabled, } from "./harness-auto-approve.js"; import { isHarnessGitQaCommitComplete, SMOKE_FILE_REL, smokeFileHasIsoLine, } from "./harness-git-qa.mjs"; import { appendPlanApprovalIfNew, type HarnessRunContext, hasPlanUserApproval, indexOfLastPlanCommand, type PlanPacketLike, readExecutorHandoffFromRun, readPlanPacketFromPath, saveRunContextToDisk, } from "./harness-run-context.js"; import { executeCreatePlan } from "./plan-approval/create-plan.js"; import { validatePlanApprovalReadiness } from "./plan-approval-readiness.js"; import { loadPlanDebateEligibilitySnapshot } from "./plan-debate-eligibility-snapshot.js"; import { getPlanFocusCoverage } from "./plan-debate-focus.js"; import { validatePlanDebateGate } from "./plan-debate-gate.js"; import { planDebateIdForRun } from "./plan-debate-id.js"; import { checkDebateWallClock, type DebateWallClockResult, } from "./plan-debate-wall-clock.js"; import { resolvePlanHumanGateStatus } from "./plan-human-gates.js"; import { loadMessengerState, type MessengerState } from "./plan-messenger.js"; import { computeTaskInputHash, isTaskClarificationReady, readTaskClarificationDoc, TASK_CLARIFICATION_ARTIFACT, } from "./plan-task-clarification.js"; const QA_SMOKE_TASK_RE = /\b(qa smoke|e2e-last-run|evals\/smoke\/|iso-?8601.*timestamp|append one .* timestamp line)\b/i; const QA_GIT_TASK_RE = /\b(harness git|harness-git-branch|harness-git-commit|auto-feature-branch|git workflow|git branch.*commit)\b/i; export function isHarnessGitQaTask(taskSummary: string): boolean { return QA_GIT_TASK_RE.test(taskSummary.trim()); } export function isHarnessQaSmokeTask(taskSummary: string): boolean { return ( QA_SMOKE_TASK_RE.test(taskSummary.trim()) || isHarnessGitQaTask(taskSummary) ); } export function shouldSeedHeadlessTaskClarification( taskSummary: string, ): boolean { if (!isHarnessNonInteractive() || !isHarnessPlanAutoApproveEnabled()) { return false; } if (process.env.HARNESS_PLAN_NONINTERACTIVE === "1") return true; if (process.env.HARNESS_QA_SMOKE === "1") return true; return isHarnessQaSmokeTask(taskSummary); } const PLANNING_CONTEXT_ARTIFACT = "artifacts/planning-context.yaml"; export function shouldSeedHeadlessQaPlanningArtifacts( taskSummary: string, ): boolean { if (!isHarnessNonInteractive() || !isHarnessPlanAutoApproveEnabled()) { return false; } if (process.env.HARNESS_QA_SMOKE !== "1") return false; return isHarnessQaSmokeTask(taskSummary); } /** Minimal planning-context for QA smoke so headless auto-approve is not blocked. */ export async function seedHeadlessQaPlanningArtifactsIfNeeded(args: { runDir: string; taskSummary: string; }): Promise { if (!shouldSeedHeadlessQaPlanningArtifacts(args.taskSummary)) return false; const target = join(args.runDir, PLANNING_CONTEXT_ARTIFACT); if (await fileExists(target)) return false; await mkdir(join(args.runDir, "artifacts"), { recursive: true }); const doc = { schema_version: "1.0.0", status: "ok", task_ref: TASK_CLARIFICATION_ARTIFACT, summary: "Headless QA smoke: single marker file under .pi/harness/evals/smoke; no code changes.", coverage: { architecture: { status: "ok", tools_used: ["read"], key_paths: [".pi/harness/evals/smoke/E2E-LAST-RUN.txt"], summary: "Smoke marker only; no architectural code surface.", }, structure: { status: "ok", tools_used: ["read"], key_paths: [".pi/harness/evals/smoke"], summary: "Target directory and marker file exist for append-only smoke.", }, semantic: { status: "skipped", reason: "QA smoke task is explicit single-file marker update.", }, }, findings: [], key_paths: [".pi/harness/evals/smoke/E2E-LAST-RUN.txt"], evidence_refs: [TASK_CLARIFICATION_ARTIFACT], open_questions: [], source: "headless_qa_auto", recorded_at: new Date().toISOString(), }; await writeFile(target, stringifyYaml(doc), "utf-8"); return true; } export async function seedHeadlessTaskClarificationIfNeeded(args: { runDir: string; taskSummary: string; riskLevel?: string; quick?: boolean; }): Promise { if (!shouldSeedHeadlessTaskClarification(args.taskSummary)) return false; const existing = await readTaskClarificationDoc(args.runDir); if (existing && String(existing.status ?? "").toLowerCase() === "ready") { return false; } const clarified = args.taskSummary.trim(); if (clarified.length < 8) return false; const doc = { schema_version: "1.0.0", status: "ready", clarified_task: clarified, unresolved_questions: [], risk_level: String(args.riskLevel ?? "low").toLowerCase(), quick: args.quick === true, task_input_hash: computeTaskInputHash({ sourceTask: clarified, riskLevel: args.riskLevel, quick: args.quick, }), user_engagement: { source: "headless_auto", recorded_at: new Date().toISOString(), }, needs_clarification: false, }; await mkdir(join(args.runDir, "artifacts"), { recursive: true }); await writeFile( join(args.runDir, TASK_CLARIFICATION_ARTIFACT), stringifyYaml(doc), "utf-8", ); return true; } async function fileExists(path: string): Promise { try { await access(path, constants.R_OK); return true; } catch { return false; } } function entriesSincePlanCommand(entries: unknown[]): number { const since = Math.max(0, indexOfLastPlanCommand(entries)); return Math.max(0, entries.length - since); } function headlessStuckEntryThreshold(): number { const raw = process.env.HARNESS_HEADLESS_STUCK_ENTRIES?.trim(); if (raw) { const parsed = Number.parseInt(raw, 10); if (Number.isFinite(parsed) && parsed > 0) return parsed; } return process.env.HARNESS_QA_SMOKE === "1" ? 48 : 120; } export function messengerDebateReadyForHeadless( messenger: MessengerState | null, ): boolean { if (!messenger?.rounds) return false; const indices = Object.keys(messenger.rounds) .map((k) => Number.parseInt(k, 10)) .filter((n) => Number.isFinite(n)) .sort((a, b) => b - a); if (indices.length === 0) return false; const last = messenger.rounds[String(indices[0]!)]; if (!last?.integrator_posted) return false; return (last.unresolved_claim_ids?.length ?? 0) === 0; } export function headlessPlanDebateForceReason(args: { entries: unknown[]; wall: DebateWallClockResult; messengerReady?: boolean; }): string | null { if (args.messengerReady) { return "messenger integrator round complete with no unresolved claims"; } if (args.wall.exceeded) { return `debate wall-clock exceeded (${Math.round(args.wall.elapsed_ms / 1000)}s)`; } if (entriesSincePlanCommand(args.entries) >= headlessStuckEntryThreshold()) { return `session entries since plan command >= ${headlessStuckEntryThreshold()}`; } return null; } async function patchLastReviewRoundGateReady(runDir: string): Promise { const artifactsDir = join(runDir, "artifacts"); let files: string[] = []; try { files = (await readdir(artifactsDir)).filter((f) => /^review-round(?:-r\d+|-consolidated|-parallel-probes)\.yaml$/i.test(f), ); } catch { return false; } if (files.length === 0) return false; files.sort(); const target = join(artifactsDir, files[files.length - 1]!); const raw = await readFile(target, "utf-8"); const doc = parseYaml(raw) as Record; if (doc.review_gate_ready === true) return false; doc.review_gate_ready = true; await writeFile(target, stringifyYaml(doc), "utf-8"); return true; } export async function writeHeadlessPlanDebateConsensusBypass(args: { projectRoot: string; runId: string; rationale: string; }): Promise { const debateId = planDebateIdForRun(args.runId); const debatesDir = join(args.projectRoot, ".pi", "harness", "debates"); await mkdir(debatesDir, { recursive: true }); const consensusPath = join(debatesDir, `${debateId}.consensus.json`); if (await fileExists(consensusPath)) return false; const runDir = join(args.projectRoot, ".pi", "harness", "runs", args.runId); const coverage = await getPlanFocusCoverage(runDir); const consensus = { schema_version: "1.0.0", contract_version: "1.0.0", run_id: args.runId, debate_id: debateId, debate_phase: "plan", round_count: Math.max(1, coverage.last_round_index), budget_used: 0, severity_scores: { correctness: 0.1, security: 0.1, architecture: 0.1, test_integrity: 0.1, }, severity_thresholds: { correctness_block_at: 0.85, security_block_at: 0.85, architecture_block_at: 0.85, test_integrity_block_at: 0.85, }, confidence_weights: { claim_quality: 0.4, reproducibility: 0.35, agreement: 0.25, }, evidence_refs: [], strict_gate_prerequisites: { plan_gate_passed: false, execution_completed: false, evaluator_passed: coverage.last_review_gate_ready, adversarial_debate_completed: true, severity_policy_ok: true, benchmark_delta_checks_passed: false, rollback_artifacts_generated: false, }, policy_decision: "conditional_pass", rationale: args.rationale, headless_bypass: true, }; await writeFile( consensusPath, `${JSON.stringify(consensus, null, 2)}\n`, "utf-8", ); return true; } export interface HeadlessPlanProgressResult { seeded_clarification: boolean; seeded_planning_context: boolean; patched_review_gate: boolean; wrote_consensus_bypass: boolean; force_reason: string | null; } export async function maybeForceHeadlessPlanProgress(args: { projectRoot: string; runId: string; taskSummary: string; entries: unknown[]; riskLevel?: string; quick?: boolean; }): Promise { const result: HeadlessPlanProgressResult = { seeded_clarification: false, seeded_planning_context: false, patched_review_gate: false, wrote_consensus_bypass: false, force_reason: null, }; if (!isHarnessNonInteractive() || !isHarnessPlanAutoApproveEnabled()) { return result; } if (hasPlanUserApproval(args.entries, { sincePlanCommand: true })) { return result; } const runDir = join(args.projectRoot, ".pi", "harness", "runs", args.runId); result.seeded_clarification = await seedHeadlessTaskClarificationIfNeeded({ runDir, taskSummary: args.taskSummary, riskLevel: args.riskLevel, quick: args.quick, }); result.seeded_planning_context = await seedHeadlessQaPlanningArtifactsIfNeeded({ runDir, taskSummary: args.taskSummary, }); const gateStatus = await resolvePlanHumanGateStatus( args.projectRoot, args.runId, args.entries, { quick: args.quick, taskSummary: args.taskSummary, }, ); if (!gateStatus.debateRequired) return result; const messenger = await loadMessengerState(runDir); const wall = checkDebateWallClock({ opened_at: messenger?.opened_at, debate_profile: messenger?.debate_profile, }); const messengerReady = messengerDebateReadyForHeadless(messenger); const forceReason = headlessPlanDebateForceReason({ entries: args.entries, wall, messengerReady, }); if (!forceReason) return result; result.force_reason = forceReason; const planPacketPath = join(runDir, "plan-packet.yaml"); if (!(await fileExists(planPacketPath))) return result; result.patched_review_gate = await patchLastReviewRoundGateReady(runDir); const eligibility = await loadPlanDebateEligibilitySnapshot(runDir); const debateGate = await validatePlanDebateGate( args.projectRoot, args.runId, eligibility ?? undefined, ); if (debateGate.ok) return result; result.wrote_consensus_bypass = await writeHeadlessPlanDebateConsensusBypass({ projectRoot: args.projectRoot, runId: args.runId, rationale: `Headless plan progress: ${forceReason}.`, }); return result; } export interface HeadlessAutoPlanFinalizeDeps { appendEntry: (customType: string, data: unknown) => void; getEntries: () => unknown[]; getSubagentEntries: () => unknown[]; onPlanCommitted: ( runCtx: HarnessRunContext, packet: PlanPacketLike, planPath: string, ) => void; } export async function tryHeadlessAutoPlanFinalize(args: { projectRoot: string; runCtx: HarnessRunContext; taskSummary: string; entries: unknown[]; riskLevel?: string; quick?: boolean; deps: HeadlessAutoPlanFinalizeDeps; }): Promise<{ finalized: boolean; progress: HeadlessPlanProgressResult; reason?: string; }> { const progress = await maybeForceHeadlessPlanProgress({ projectRoot: args.projectRoot, runId: args.runCtx.run_id, taskSummary: args.taskSummary, entries: args.entries, riskLevel: args.riskLevel, quick: args.quick, }); if (args.runCtx.plan_ready) { return { finalized: true, progress }; } if (hasPlanUserApproval(args.entries, { sincePlanCommand: true })) { return { finalized: false, progress, reason: "approval pending create_plan", }; } const runDir = join( args.projectRoot, ".pi", "harness", "runs", args.runCtx.run_id, ); await seedHeadlessQaPlanningArtifactsIfNeeded({ runDir, taskSummary: args.taskSummary, }); const planPath = join(runDir, "plan-packet.yaml"); if (!(await fileExists(planPath))) { return { finalized: false, progress, reason: "plan-packet.yaml missing" }; } const packet = await readPlanPacketFromPath(planPath); if (!packet) { return { finalized: false, progress, reason: "plan packet unreadable" }; } const readiness = await validatePlanApprovalReadiness( args.projectRoot, args.runCtx.run_id, { risk_level: String(args.riskLevel ?? "med").toLowerCase(), quick: args.quick, }, ); const eligibility = await loadPlanDebateEligibilitySnapshot(runDir); const debateGate = await validatePlanDebateGate( args.projectRoot, args.runCtx.run_id, eligibility ?? undefined, ); const policy = await canAutoApprovePlan({ projectRoot: args.projectRoot, runId: args.runCtx.run_id, riskLevel: String(args.riskLevel ?? "med").toLowerCase(), readiness, debateGate, }); if (!policy.allowed) { return { finalized: false, progress, reason: policy.reasons.join("; ") || "auto-approve blocked", }; } appendPlanApprovalIfNew( args.deps.appendEntry, args.entries, { plan_id: String(packet.plan_id ?? args.runCtx.plan_id ?? ""), approved_at: new Date().toISOString(), source: "noninteractive", }, args.runCtx, ); const entriesAfterApproval = args.deps.getEntries(); const created = await executeCreatePlan(packet, { projectRoot: args.projectRoot, getParentEntries: () => entriesAfterApproval, getSubagentEntries: args.deps.getSubagentEntries, getParentRunContext: () => args.runCtx, onCommitted: args.deps.onPlanCommitted, }); if (!created.ok) { return { finalized: false, progress, reason: created.error }; } args.runCtx.plan_ready = true; args.runCtx.plan_id = created.planId; args.runCtx.plan_packet_path = created.planPath; return { finalized: true, progress }; } export async function headlessTaskClarificationReady( runDir: string, ): Promise { const readiness = await isTaskClarificationReady(runDir); return readiness.ok; } export { isHarnessGitQaCommitComplete, smokeFileHasIsoLine, } from "./harness-git-qa.mjs"; export function shouldEndHeadlessPlanPrintSession(args: { command: string; planReady: boolean; }): boolean { return ( args.command === "harness-plan" && args.planReady && isHarnessNonInteractive() ); } export async function shouldEndHeadlessHarnessPrintSession(args: { command: string | null | undefined; runCtx: HarnessRunContext | null; projectRoot: string; }): Promise { const command = String(args.command ?? ""); const runCtx = args.runCtx; if (!runCtx || !isHarnessNonInteractive()) return false; if ( shouldEndHeadlessPlanPrintSession({ command, planReady: runCtx.plan_ready === true, }) ) { return true; } const lastStep = String(runCtx.last_completed_step ?? "").toLowerCase(); const lastOutcome = String(runCtx.last_outcome ?? "").toLowerCase(); if (command === "harness-run" && runCtx.run_id) { const handoff = await readExecutorHandoffFromRun( runCtx.run_id, args.projectRoot, ); const execStatus = String(handoff?.execution_status ?? "").toLowerCase(); if ( execStatus === "completed" || execStatus === "scope_drift" || execStatus === "blocked" ) { return true; } if (lastStep === "execute" && lastOutcome) { return true; } } if ( command === "harness-steer" && (lastStep === "steer" || lastOutcome === "completed") ) { return true; } if (process.env.HARNESS_QA_SMOKE !== "1") return false; const gitCommitDone = await isHarnessGitQaCommitComplete(args.projectRoot); const hasSmoke = await smokeFileHasIsoLine(args.projectRoot); const gitTask = isHarnessGitQaTask(runCtx.task_summary ?? ""); if (command === "harness-run") { if (gitTask && gitCommitDone) return true; if (!gitTask && hasSmoke && lastOutcome === "completed") return true; } if ( (command === "harness-review" || command === "harness-eval" || command === "harness-critic") && (lastStep === "review" || lastStep === "adversary") ) { return true; } if (command === "harness-auto") { if (gitTask && gitCommitDone) return true; if (!gitTask && hasSmoke) { if (lastStep === "review" || lastStep === "adversary") return true; if (runCtx.plan_ready === true && lastOutcome === "pass") return true; } } return false; } export function endHeadlessHarnessPrintSession(ctx: { abort?: () => void; }): void { ctx.abort?.(); // pi -p often keeps the Node process alive after abort(); exit so headless QA/CI does not hang. if (isHarnessNonInteractive() && process.env.HARNESS_HEADLESS_EXIT !== "0") { setTimeout(() => process.exit(0), 50); } } /** QA smoke: after headless auto plan, append ISO directly and skip full executor/review loop. */ export async function maybeHeadlessQaAutoExecuteSmoke(args: { projectRoot: string; runCtx: HarnessRunContext; command: string; }): Promise { if (args.command !== "harness-auto") return false; if (process.env.HARNESS_QA_SMOKE !== "1" || !isHarnessNonInteractive()) { return false; } if (!args.runCtx.plan_ready) return false; if (!isHarnessQaSmokeTask(args.runCtx.task_summary ?? "")) return false; if (isHarnessGitQaTask(args.runCtx.task_summary ?? "")) return false; if (await smokeFileHasIsoLine(args.projectRoot)) return true; const smokePath = join(args.projectRoot, SMOKE_FILE_REL); await mkdir(dirname(smokePath), { recursive: true }); await writeFile(smokePath, `${new Date().toISOString()}\n`, "utf-8"); const updated: HarnessRunContext = { ...args.runCtx, phase: "evaluate", last_completed_step: "review", last_outcome: "pass", updated_at: new Date().toISOString(), }; await saveRunContextToDisk(updated); Object.assign(args.runCtx, updated); return true; } /** QA git task: append smoke marker and commit via harness-git-commit on feature branch. */ async function ensureGitWorkflowArtifactForRun(args: { projectRoot: string; runId: string; upPkg?: string; }): Promise { const upPkg = args.upPkg ?? args.projectRoot; const { ensureHarnessGitBranch, writeGitWorkflowArtifact } = await import( "./harness-git-branch.mjs" ); const runDir = join(args.projectRoot, ".pi", "harness", "runs", args.runId); const branchResult = await ensureHarnessGitBranch({ projectRoot: args.projectRoot, runId: args.runId, upPkg, }); await writeGitWorkflowArtifact({ runDir, result: branchResult }); } export async function maybeHeadlessGitQaFinalizeOnRun(args: { projectRoot: string; runCtx: HarnessRunContext; command: string; upPkg?: string; }): Promise { if (args.command !== "harness-run" && args.command !== "harness-auto") { return false; } if (process.env.HARNESS_QA_SMOKE !== "1" || !isHarnessNonInteractive()) { return false; } if (!isHarnessGitQaTask(args.runCtx.task_summary ?? "")) return false; await ensureGitWorkflowArtifactForRun({ projectRoot: args.projectRoot, runId: args.runCtx.run_id, upPkg: args.upPkg, }); if (await isHarnessGitQaCommitComplete(args.projectRoot)) return true; const smokePath = join(args.projectRoot, SMOKE_FILE_REL); await mkdir(dirname(smokePath), { recursive: true }); await writeFile(smokePath, `${new Date().toISOString()}\n`, "utf-8"); const upPkg = args.upPkg ?? args.projectRoot; const commitScript = join(upPkg, ".pi", "scripts", "harness-git-commit.mjs"); const commit = spawnSync( process.execPath, [ commitScript, "--root", args.projectRoot, "--only-path", SMOKE_FILE_REL, "--type", "chore", "--scope", "harness", "--subject", `qa smoke marker for ${args.runCtx.run_id}`, ], { encoding: "utf8", cwd: args.projectRoot }, ); if (commit.status !== 0) { return false; } if (!(await isHarnessGitQaCommitComplete(args.projectRoot))) { return false; } const updated: HarnessRunContext = { ...args.runCtx, phase: "evaluate", last_completed_step: "execute", last_outcome: "completed", next_recommended_command: "/harness-review", updated_at: new Date().toISOString(), }; await saveRunContextToDisk(updated); Object.assign(args.runCtx, updated); return true; }