/**
 * pi-eval — agent evaluation harness. Inspired by opencc's eval system.
 * /eval run "prompt"                → run prompt, capture tool calls, judge quality
 * /eval judge                       → judge the current session's tool usage
 * /eval methodology                 → check read-before-edit and other patterns
 * /eval report                      → generate eval report for session
 * /eval suite file.json             → run eval suite from file
 * /eval history                     → show past eval results
 *
 * From opencc: 6-criteria weighted scoring (success, tool_usage, efficiency,
 * output_quality, error_handling, methodology). Plus Carmack's "measure everything."
 * Plus Kelsey's "if you can't observe it, you can't improve it."
 */
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
import { Type } from "@sinclair/typebox";
import { readFileSync, writeFileSync, mkdirSync, readdirSync } from "node:fs";
import { join } from "node:path";
import { homedir } from "node:os";

const SAVE_DIR = join(homedir(), ".pi", "eval");
const RESULTS_DIR = join(SAVE_DIR, "results");
const RST = "\x1b[0m", B = "\x1b[1m", D = "\x1b[2m";
const GREEN = "\x1b[32m", RED = "\x1b[31m", YELLOW = "\x1b[33m", CYAN = "\x1b[36m", MAGENTA = "\x1b[35m";

interface EvalScores {
  success: number;       // Did it complete without errors?
  tool_usage: number;    // Tool success rate
  efficiency: number;    // Reasonable tool count?
  output_quality: number; // Did it produce output?
  error_handling: number; // Did it recover from errors?
  methodology: number;   // Read-before-edit, proper tool selection?
}

interface EvalResult {
  id: string;
  prompt: string;
  timestamp: string;
  duration: number;
  toolCalls: { name: string; success: boolean }[];
  errors: string[];
  outputLength: number;
  scores: EvalScores;
  overall: number;
  grade: string;
  feedback: string[];
}

// ── Scoring ─────────────────────────────────────────────────────────────────

function getGrade(overall: number): string {
  if (overall >= 0.9) return `${GREEN}A${RST}`;
  if (overall >= 0.8) return `${GREEN}B${RST}`;
  if (overall >= 0.7) return `${YELLOW}C${RST}`;
  if (overall >= 0.6) return `${YELLOW}D${RST}`;
  return `${RED}F${RST}`;
}

function getGradeRaw(overall: number): string {
  if (overall >= 0.9) return "A";
  if (overall >= 0.8) return "B";
  if (overall >= 0.7) return "C";
  if (overall >= 0.6) return "D";
  return "F";
}

function scoreBar(value: number): string {
  const filled = Math.round(value * 10);
  return `${GREEN}${"█".repeat(filled)}${D}${"░".repeat(10 - filled)}${RST}`;
}

function judgeToolUsage(tools: { name: string; success: boolean }[]): { score: number; feedback: string[] } {
  const feedback: string[] = [];
  if (tools.length === 0) return { score: 1.0, feedback: ["No tools used"] };

  const successRate = tools.filter(t => t.success).length / tools.length;
  const errorCount = tools.filter(t => !t.success).length;

  if (errorCount > 0) feedback.push(`${errorCount} tool errors out of ${tools.length} calls`);
  if (successRate === 1) feedback.push("All tool calls succeeded");

  return { score: successRate, feedback };
}

function judgeEfficiency(toolCount: number): { score: number; feedback: string[] } {
  const feedback: string[] = [];
  // 0-5 tools = 1.0, 6-10 = 0.8, 11-20 = 0.5, 20+ = 0.3
  let score = 1.0;
  if (toolCount > 20) { score = 0.3; feedback.push(`High tool usage: ${toolCount} calls — may be thrashing`); }
  else if (toolCount > 10) { score = 0.5; feedback.push(`${toolCount} tool calls — moderate complexity`); }
  else if (toolCount > 5) { score = 0.8; }
  else { feedback.push(`Efficient: ${toolCount} tool calls`); }
  return { score, feedback };
}

function judgeMethodology(tools: { name: string; success: boolean }[]): { score: number; feedback: string[]; patterns: string[] } {
  const feedback: string[] = [];
  const patterns: string[] = [];
  let score = 1.0;
  const names = tools.map(t => t.name.toLowerCase());
  const WRITE_OPS = ["edit", "write", "multiedit"];
  const READ_OPS = ["read", "bash", "grep", "glob", "find", "rg", "lsp"];

  // Check: read before edit (openCC-inspired)
  const hasEdit = names.some(n => WRITE_OPS.includes(n));
  const hasRead = names.some(n => READ_OPS.includes(n));
  const firstEditIdx = names.findIndex(n => WRITE_OPS.includes(n));
  const firstReadIdx = names.findIndex(n => READ_OPS.includes(n));

  if (hasEdit && hasRead && firstReadIdx < firstEditIdx) {
    score += 0.2; // bonus for reading first
    patterns.push("read-before-edit ✓");
    feedback.push(`${GREEN}✓${RST} Read before edit — good discipline`);
  } else if (hasEdit && !hasRead) {
    feedback.push(`${RED}⚠${RST} Modified files without reading first`);
    patterns.push("no-read-before-edit ✗");
    score -= 0.3;
  } else if (hasEdit && hasRead && firstEditIdx < firstReadIdx) {
    feedback.push(`${YELLOW}⚠${RST} Edited before reading — risky`);
    patterns.push("edit-before-read ⚠");
    score -= 0.15;
  }

  // Check: blind writes — edit/write without a preceding read within last 5 calls
  let blindWrites = 0;
  for (let i = 0; i < names.length; i++) {
    if (!WRITE_OPS.includes(names[i])) continue;
    const window = names.slice(Math.max(0, i - 5), i);
    if (!window.some(n => READ_OPS.includes(n))) blindWrites++;
  }
  if (blindWrites > 0) {
    const penalty = Math.min(0.9, blindWrites * 0.3);
    score -= penalty;
    patterns.push(`blind-writes:${blindWrites}`);
    feedback.push(`${RED}⚠${RST} ${blindWrites} blind write(s) — no read within preceding 5 tool calls`);
  } else if (hasEdit) {
    patterns.push("all-writes-informed ✓");
  }

  // Check: bash for search instead of dedicated tools
  const bashSearches = tools.filter(t => t.name.toLowerCase() === "bash").length;
  const totalSearchTools = tools.filter(t =>
    ["grep", "glob", "find", "rg"].some(s => t.name.toLowerCase().includes(s))
  ).length;

  if (bashSearches > 3 && totalSearchTools === 0) {
    feedback.push(`${YELLOW}⚠${RST} Heavy bash usage for search — consider dedicated tools`);
    patterns.push("bash-heavy-search");
    score -= 0.1;
  }

  // Check: repeated identical calls (thrashing)
  const repeats = names.filter((c, i) => i > 0 && c === names[i - 1]).length;
  if (repeats > 3) {
    feedback.push(`${RED}⚠${RST} ${repeats} repeated consecutive tool calls — possible loop`);
    patterns.push(`thrashing:${repeats}`);
    score -= 0.2;
  }

  if (feedback.length === 0) {
    feedback.push(`${GREEN}✓${RST} Good methodology — read before edit, proper tool selection`);
    patterns.push("clean");
  }

  return { score: Math.max(0, Math.min(1, score)), feedback, patterns };
}

function calculateOverall(scores: EvalScores): number {
  return (
    scores.success * 0.30 +
    scores.tool_usage * 0.20 +
    scores.efficiency * 0.15 +
    scores.output_quality * 0.15 +
    scores.error_handling * 0.10 +
    scores.methodology * 0.10
  );
}

// ── Persistence ─────────────────────────────────────────────────────────────

function saveResult(result: EvalResult) {
  mkdirSync(RESULTS_DIR, { recursive: true });
  writeFileSync(join(RESULTS_DIR, `${result.id}.json`), JSON.stringify(result, null, 2));
}

function loadResults(): EvalResult[] {
  try {
    mkdirSync(RESULTS_DIR, { recursive: true });
    return readdirSync(RESULTS_DIR)
      .filter(f => f.endsWith(".json"))
      .map(f => JSON.parse(readFileSync(join(RESULTS_DIR, f), "utf-8")))
      .sort((a, b) => b.timestamp.localeCompare(a.timestamp));
  } catch { return []; }
}

// ── Report Formatting ───────────────────────────────────────────────────────

function formatReport(result: EvalResult): string {
  const lines: string[] = [];
  lines.push(`${B}${CYAN}════════════════════════════════════════${RST}`);
  lines.push(`${B}${CYAN}📊 EVAL REPORT${RST}`);
  lines.push(`${B}${CYAN}════════════════════════════════════════${RST}`);
  lines.push(`  ${D}ID:${RST}       ${result.id}`);
  lines.push(`  ${D}Prompt:${RST}   ${result.prompt.slice(0, 60)}${result.prompt.length > 60 ? "..." : ""}`);
  lines.push(`  ${D}Time:${RST}     ${result.timestamp}`);
  lines.push(`  ${D}Duration:${RST} ${result.duration}ms`);
  lines.push("");

  lines.push(`  ${B}SCORES:${RST}`);
  const overall = result.overall;
  lines.push(`  ${B}Overall: ${getGrade(overall)} ${(overall * 100).toFixed(1)}%${RST}`);
  lines.push("");

  for (const [key, value] of Object.entries(result.scores)) {
    const label = key.replace(/_/g, " ").padEnd(18);
    lines.push(`  ${label} ${scoreBar(value)} ${(value * 100).toFixed(0)}%`);
  }

  lines.push("");
  lines.push(`  ${B}FEEDBACK:${RST}`);
  for (const f of result.feedback) lines.push(`  • ${f}`);

  if (result.toolCalls.length > 0) {
    lines.push("");
    lines.push(`  ${B}TOOL USAGE:${RST}`);
    const stats: Record<string, { count: number; errors: number }> = {};
    for (const t of result.toolCalls) {
      if (!stats[t.name]) stats[t.name] = { count: 0, errors: 0 };
      stats[t.name].count++;
      if (!t.success) stats[t.name].errors++;
    }
    for (const [name, s] of Object.entries(stats).sort((a, b) => b[1].count - a[1].count)) {
      const errStr = s.errors > 0 ? ` ${RED}(${s.errors} errors)${RST}` : "";
      lines.push(`    ${CYAN}${name}${RST}: ${s.count}x${errStr}`);
    }
  }

  lines.push(`${B}${CYAN}════════════════════════════════════════${RST}`);
  return lines.join("\n");
}

export default function piEval(pi: ExtensionAPI) {
  pi.registerCommand("eval", {
    description: "Agent evaluation harness. /eval [judge|methodology|report|history|patterns]",
    handler: async (args, ctx) => {
      const parts = args.trim().split(/\s+/);
      const sub = parts[0]?.toLowerCase() || "help";

      switch (sub) {
        case "judge": {
          // Judge a hypothetical session — user provides tool call list
          ctx.ui.notify([
            `${B}${CYAN}Eval Judge${RST}`,
            "",
            `Use the LLM tool ${CYAN}eval_judge${RST} to programmatically judge sessions.`,
            `Or run /eval patterns to see methodology checks.`,
            "",
            `${D}The judge scores on 6 criteria:${RST}`,
            `  1. ${B}Success${RST} (30%) — did it complete?`,
            `  2. ${B}Tool Usage${RST} (20%) — tool success rate`,
            `  3. ${B}Efficiency${RST} (15%) — reasonable tool count`,
            `  4. ${B}Output Quality${RST} (15%) — produced output?`,
            `  5. ${B}Error Handling${RST} (10%) — recovered from errors?`,
            `  6. ${B}Methodology${RST} (10%) — read-before-edit, proper tools`,
          ].join("\n"), "info");
          break;
        }

        case "patterns": case "methodology": {
          ctx.ui.notify([
            `${B}${CYAN}Methodology Patterns (from opencc eval)${RST}`,
            "",
            `${GREEN}✓ Good Patterns:${RST}`,
            `  • Read files before editing them`,
            `  • Use dedicated search tools (grep/glob) over bash`,
            `  • Minimal tool calls for the task`,
            `  • No repeated identical calls (thrashing)`,
            `  • Error recovery — succeed despite tool failures`,
            "",
            `${RED}✗ Anti-Patterns:${RST}`,
            `  • Edit without reading first`,
            `  • Bash grep instead of dedicated Grep tool`,
            `  • 20+ tool calls for simple tasks (thrashing)`,
            `  • Repeated identical calls in sequence`,
            `  • No output generated`,
            "",
            `${D}Scoring weights: success 30%, tools 20%, efficiency 15%,`,
            `output 15%, error_handling 10%, methodology 10%${RST}`,
          ].join("\n"), "info");
          break;
        }

        case "history": {
          const results = loadResults();
          if (results.length === 0) { ctx.ui.notify("No eval results yet. Use the eval_judge tool.", "info"); return; }
          let out = `${B}${CYAN}Eval History${RST} (${results.length} results)\n\n`;
          for (const r of results.slice(0, 15)) {
            const grade = getGradeRaw(r.overall);
            const color = r.overall >= 0.8 ? GREEN : r.overall >= 0.6 ? YELLOW : RED;
            out += `  ${color}${grade}${RST} ${(r.overall * 100).toFixed(0).padStart(3)}% | ${r.prompt.slice(0, 50).padEnd(50)} | ${D}${r.timestamp.slice(0, 10)}${RST}\n`;
          }
          ctx.ui.notify(out, "info");
          break;
        }

        case "report": {
          const results = loadResults();
          if (results.length === 0) { ctx.ui.notify("No eval results yet.", "info"); return; }
          const latest = results[0];
          ctx.ui.notify(formatReport(latest), "info");
          break;
        }

        default: {
          ctx.ui.notify([
            `${B}${CYAN}🧪 Eval Harness${RST}  ${D}(inspired by opencc)${RST}`,
            "",
            `  /eval judge       — scoring criteria explanation`,
            `  /eval patterns    — methodology pattern reference`,
            `  /eval history     — past eval results`,
            `  /eval report      — latest eval report`,
            "",
            `  ${D}Use the eval_judge LLM tool to programmatically score sessions${RST}`,
          ].join("\n"), "info");
        }
      }
    },
  });

  // LLM tool for programmatic eval
  pi.registerTool({ name: "eval_judge",
    description: "Judge an agent session's quality. Provide tool calls made and evaluate methodology, efficiency, and success.",
    parameters: Type.Object({
      prompt: Type.String({ description: "The original prompt/task" }),
      toolCalls: Type.Array(Type.Object({
        name: Type.String({ description: "Tool name (read, edit, write, bash, etc)" }),
        success: Type.Boolean({ description: "Did the tool call succeed?" }),
      }), { description: "List of tool calls made during the session" }),
      errors: Type.Optional(Type.Array(Type.String(), { description: "Error messages encountered" })),
      outputLength: Type.Optional(Type.Number({ description: "Length of final output in characters" })),
      completed: Type.Optional(Type.Boolean({ description: "Did the session complete successfully?" })),
    }),
    execute: async (params) => {
      const tools = params.toolCalls;
      const errors = params.errors || [];
      const outputLen = params.outputLength || 0;
      const completed = params.completed ?? true;

      const toolJudge = judgeToolUsage(tools);
      const effJudge = judgeEfficiency(tools.length);
      const methJudge = judgeMethodology(tools);

      const scores: EvalScores = {
        success: completed ? 1.0 : 0.0,
        tool_usage: toolJudge.score,
        efficiency: effJudge.score,
        output_quality: outputLen >= 5 ? 1.0 : outputLen > 0 ? 0.5 : 0.0,
        error_handling: errors.length === 0 ? 1.0 : completed ? 0.5 : 0.0,
        methodology: methJudge.score,
      };

      const overall = calculateOverall(scores);
      const feedback = [...toolJudge.feedback, ...effJudge.feedback, ...methJudge.feedback];

      const result: EvalResult = {
        id: `eval-${Date.now()}`,
        prompt: params.prompt,
        timestamp: new Date().toISOString(),
        duration: 0,
        toolCalls: tools,
        errors,
        outputLength: outputLen,
        scores,
        overall,
        grade: getGradeRaw(overall),
        feedback,
      };

      saveResult(result);

      return {
        overall: Math.round(overall * 100),
        grade: result.grade,
        scores: Object.fromEntries(Object.entries(scores).map(([k, v]) => [k, Math.round(v * 100)])),
        feedback,
      };
    },
  });

  // LLM tool: structured handoff validation (from exhaze's parallel-execute-v2)
  pi.registerTool({ name: "eval_handoff",
    description: "Validate a structured agent handoff. Check confidence calibration, evidence links, and flag confident hallucinations. Based on exhaze's parallel-execute-v2 protocol.",
    parameters: Type.Object({
      agentId: Type.String({ description: "Agent identifier" }),
      task: Type.String({ description: "What the agent was asked to do" }),
      reasoningChain: Type.Array(Type.Object({
        step: Type.Number(),
        action: Type.String(),
        reasoning: Type.String(),
        evidence: Type.Optional(Type.Array(Type.String(), { description: "File:line references or URLs" })),
        confidence: Type.Number({ description: "0-5 scale: 0-1 speculation, 2-3 probable, 4-5 verified" }),
      })),
      discoveredContext: Type.Optional(Type.Object({
        constraints: Type.Optional(Type.Array(Type.String())),
        dependencies: Type.Optional(Type.Array(Type.String())),
        assumptions: Type.Optional(Type.Array(Type.String())),
      })),
      openQuestions: Type.Optional(Type.Array(Type.String())),
    }),
    execute: async (params) => {
      const chain = params.reasoningChain;
      const warnings: string[] = [];
      const flags: string[] = [];
      let totalConfidence = 0;

      for (const step of chain) {
        totalConfidence += step.confidence;
        const hasEvidence = step.evidence && step.evidence.length > 0;

        // Flag confident hallucinations: high confidence (4+) with no evidence
        if (step.confidence >= 4 && !hasEvidence) {
          flags.push(`⚠ Step ${step.step}: CONFIDENT HALLUCINATION — confidence ${step.confidence}/5 but no evidence for "${step.action}"`);
        }
        // Flag speculation presented without marking
        if (step.confidence <= 1 && !step.reasoning.toLowerCase().includes("specul") && !step.reasoning.toLowerCase().includes("guess")) {
          warnings.push(`Step ${step.step}: Low confidence (${step.confidence}) — should be marked as speculation`);
        }
      }

      const avgConfidence = chain.length > 0 ? totalConfidence / chain.length : 0;
      const evidenceRate = chain.filter(s => s.evidence && s.evidence.length > 0).length / Math.max(1, chain.length);
      const assumptionCount = params.discoveredContext?.assumptions?.length || 0;

      // Score the handoff quality
      let score = 50;
      score += Math.round(evidenceRate * 30); // Up to 30 for evidence coverage
      score += flags.length === 0 ? 15 : 0;  // 15 for no hallucination flags
      score += assumptionCount > 0 ? 5 : -5; // 5 for stating assumptions explicitly

      return {
        quality: Math.min(100, Math.max(0, score)),
        avgConfidence: Math.round(avgConfidence * 10) / 10,
        evidenceRate: Math.round(evidenceRate * 100),
        hallucinations: flags,
        warnings,
        stepsAnalyzed: chain.length,
        openQuestions: params.openQuestions || [],
      };
    },
  });
}