// packages/ui-gen/src/evaluation/llm-evaluator.ts
//
// Parallel per-criterion LLM evaluator with cached mother prompt.
//
// Architecture:
//   1. Mother prompt (cached, sent once): user request + contract + design system + all criteria definitions
//   2. 7 parallel per-criterion LLM calls, each reusing the cached mother prompt
//   3. Each call returns a focused tool response for one criterion
//
// Tiers:
//   Tier 1 — Critical (always blocks): functionality, crash → { pass: boolean, issues?: string[] }
//   Tier 2 — Quality (LLM decides):    interactivity, accessibility, layout, loading, visual → { result: 'fail'|'warn'|'pass', issues?: string[] }
//
// Caching: Anthropic ephemeral cache, OpenAI auto-cache, Google explicit cache.
// Same system prompt across all 7 calls → first call caches, 6 remaining hit cache.

import type { EvalIssue, EvalResult, EvalCategory } from './types-public.js';
import { createAgent } from '../harness/llm-router';
import type { AgentConfig, LLMToolDef, LLMAgent } from '../harness/llm-router';
import type { JsonObject } from '@ggui-ai/protocol';
import { getCriterionById } from './types-public.js';

// =============================================================================
// Helpers
// =============================================================================

/**
 * Safely coerce LLM tool output to string[].
 *
 * LLMs sometimes return `issues` as a JSON string instead of an array.
 * Iterating a string goes character-by-character, creating thousands of
 * single-char "issues". This helper handles: string[] (pass-through),
 * string (JSON parse), or anything else (empty array).
 */
function coerceStringArray(value: unknown): string[] {
  if (Array.isArray(value)) {
    return value.filter((v): v is string => typeof v === 'string' && v.length > 1);
  }
  if (typeof value === 'string') {
    try {
      const parsed = JSON.parse(value);
      if (Array.isArray(parsed)) {
        return parsed.filter((v): v is string => typeof v === 'string' && v.length > 1);
      }
    } catch {
      // Not JSON — treat the whole string as a single issue
      if (value.length > 1) return [value];
    }
  }
  return [];
}

// =============================================================================
// Public interfaces
// =============================================================================

export interface LLMEvalContext {
  sourceCode: string;
  originalPrompt: string;
  designContext?: string;
  contract?: unknown;
}

export interface LLMEvalConfig {
  provider: 'claude' | 'openai' | 'google' | 'openrouter';
  model?: string;
  /**
   * Explicit per-call credentials/routing (#484) — see
   * `AgentConfig.routeOverride`. Threaded onto the agent this
   * evaluator constructs so it never falls back to reading
   * `process.env`. Absent (default) preserves existing behavior.
   */
  routeOverride?: AgentConfig['routeOverride'];
  /**
   * Provider-429-retry observer (#489) — see `AgentConfig.onRetry`.
   * Threaded onto the agent this evaluator constructs so a rate-limited
   * retry inside an evaluation turn's `apiCall()` reaches the same
   * caller the coding leg reaches. Absent (default) preserves existing
   * behavior (retries still happen; no observer fires).
   */
  onRetry?: AgentConfig['onRetry'];
}

/** A request-specific evaluation criterion generated by the LLM. */
export interface DynamicCriterion {
  name: string;
  description: string;
}

/** Pre-warmed evaluation context (generated during coding, consumed at eval time). */
export interface PreWarmedEvalContext {
  motherPrompt: string;
  dynamicCriteria: DynamicCriterion[];
}

// =============================================================================
// Mother prompt builder (cacheable context)
// =============================================================================

export interface EvalContext {
  originalPrompt: string;
  contract?: unknown;
  designSystemSummary?: string;
}

export function buildMotherPrompt(ctx: EvalContext): string {
  const parts: string[] = [];

  parts.push(`You are evaluating a generated React component built with the ggui design system.
The code has already passed automated checks (compilation, security, imports, design tokens).
Your job is to evaluate ONE specific quality criterion per call.`);

  parts.push(`## About This Prompt
This system prompt has 2 sections:
1. **Context** — the original user request and data contract
2. **Criteria Reference** — definitions for all 7 evaluation criteria (your call will focus on one)

## Input Messages
You will receive ONE user message per call containing:
1. Which criterion to evaluate (e.g., "Evaluate FUNCTIONALITY")
2. The source code with line numbers
3. Example outputs showing the expected tool response format

Respond with a single tool call (evaluate_{criterion}). Report ALL issues you find for that criterion, not just the first one. Each issue should be an actionable fix instruction.

## Issue-array discipline (applies to EVERY criterion)
The "issues" array is for confirmed defects ONLY. Before adding an entry, it must be something you are CERTAIN is missing or broken. NEVER add:
- a self-negating note ("…actually this is fine", "so no issue here", "no actual crash scenarios are present");
- a speculative / conditional note ("if the data is shaped X then…", "this may break when…");
- a "should be verified" / "double-check that…" note.
If you are not certain something is broken, it is a PASS — omit it entirely. An entry you describe as acceptable still counts against the score, so describing a non-issue is worse than saying nothing.`);

  parts.push(`## Original Request
${ctx.originalPrompt}`);

  if (ctx.contract) {
    parts.push(`## Data Contract (JSON Schema → TypeScript types)

The component's boilerplate was generated from this contract. All types are compiler-enforced:
- \`props\` → \`interface Props\` — every field must be rendered in the UI
- \`actionSpec\` (flat map keyed by action name) → \`useAction<Payload>(name)\` hooks — each must be wired to a UI element (button, form, etc.)
- \`actionSpec[name].nextStep\` (optional) — names an agent-side MCP tool the agent SHOULD invoke on its
  next turn after the action fires. Surfaces as advisory event metadata; the component does NOT call
  the named tool. Use it to judge whether button labels/icons match the underlying action's intent.
- \`streamSpec\` (flat map keyed by channel name) → \`useStream<T>(name)\` hooks — each must be consumed with null guard (\`.latest && ...\`)
- \`agentCapabilities.tools\` is a READ-ONLY catalog — the AGENT invokes these. The component never calls them.
  References surface only via \`actionSpec[*].nextStep\` and \`streamSpec[*].source.tool\`. Do NOT flag
  missing component-side calls for these.
- \`clientCapabilities.gadgets\` → a map keyed by npm PACKAGE name; each package maps export
  names to a descriptor. Two kinds, both contract features that MUST be used:
  • Built-in browser capabilities (\`useGeolocation\`, \`useCamera\`, \`useClipboardWrite\`,
    \`useMicrophone\`, \`useFilePicker\`, \`useClipboardPaste\`, \`useNotifications\`) — imported
    from \`@ggui-ai/gadgets\`.
  • Registered third-party gadgets — hooks or components — imported from their OWN package
    (e.g. \`import { useChartTheme } from '@ggui-samples/gadget-chart'\`). The boilerplate
    emits that import. Any export listed under \`clientCapabilities.gadgets\` IS part of the
    contract — NEVER flag it as "not part of the contract".

\`\`\`json
${JSON.stringify(ctx.contract).slice(0, 6000)}
\`\`\``);
  }

  if (ctx.designSystemSummary) {
    parts.push(`## Design System
${ctx.designSystemSummary}`);
  }

  parts.push(`## Design System Rules (important for evaluation)
- Spacing props (\`gap\`, \`padding\`, \`margin\`) take a t-shirt-scale name: \`gap="md"\`, \`padding="lg"\` (\`none|xs|sm|md|lg|xl|2xl\`). **These ARE design tokens** — each resolves to a \`--ggui-spacing-*\` variable. A raw \`var(--ggui-spacing-*)\` string is an accepted escape hatch. NEVER flag a scale name as "hardcoded". Only a numeric prop (\`padding={24}\`) or a raw CSS length (\`gap="13px"\`) bypasses the scale and should be warned.
- All colors must use CSS variables: \`color="var(--ggui-color-primary-600)"\` — hardcoded #hex or rgb() is a fail
- Component props that take enum/scale values — \`variant="primary"\`, \`size="lg"\`, \`gap="md"\`, \`padding="lg"\`, \`shadow="md"\`, \`radius="lg"\`, \`tone="muted"\`, \`surface="accent"\` — are all design system tokens and are always valid
- One \`export default function Component\` — helper components as named functions above it are fine

## Evaluation Criteria Reference

### Tier 1 — Critical (blocks delivery)

**functionality**: Does the component implement ALL features in the original request?
- Check every noun/verb in the request against the rendered output
- Missing feature = fail, even if other features work perfectly
- Partial implementation = fail (e.g., search UI exists but doesn't filter)

**crash**: Are there runtime crash scenarios?
- Uninitialized state used in render (e.g., items.map when items could be undefined)
- Missing event handlers referenced in JSX
- Destructuring undefined objects
- Array access without bounds checking
- Missing null/undefined guards before .map(), .filter(), .length

### Tier 2 — Quality (fail if severe, warn if improvement)

**interactivity**: Does it have interactive elements appropriate for its purpose?
- Forms should have submit buttons
- Lists should be navigable
- Editable fields should have save/cancel
- Interactive components should have hover/focus feedback

**accessibility**: Semantic HTML, labels, alt text, keyboard navigation, ARIA state.
- Form inputs must have associated labels (label prop or aria-label)
- Images need alt text
- Interactive elements need keyboard support
- Interactive controls expose their state (aria-checked / aria-selected / aria-expanded / aria-pressed), never styling alone
- Proper heading hierarchy (h1 > h2 > h3)

**layout**: Spacing, alignment, visual grouping.
- Consistent spacing between sections
- Related items grouped visually
- No overlapping or cramped elements
- Proper use of containers and cards for grouping

**loading**: Loading/error/empty states for dynamic data.
- Components receiving async data should show loading state
- Error boundaries or error messages for failure cases
- Empty states ("No items found") when data arrays could be empty

**visual**: Design system consistency.
- Uses CSS variables (var(--ggui-*)) for colors, not hardcoded values
- Consistent use of spacing tokens
- Proper use of component variants (primary, outline, ghost)
- Shadow, border-radius from design system tokens

## Primitive Accessibility (built-in — do NOT flag as missing)

ggui design-system primitives ship their own ARIA. The code you are
reading imports them from \`@ggui-ai/design\`; the source does NOT show
their internal markup. NEVER report a primitive below as missing a
role, aria-*, label, or keyboard support — it is already there:

- \`Input\` / \`Select\` / \`TextArea\` — render a real \`<label htmlFor>\` from
  the \`label\` prop, plus \`aria-invalid\` + \`aria-describedby\` + an error
  \`role="alert"\`. A primitive that has a \`label\` prop IS labelled.
- \`RadioGroup\` — \`role="radiogroup"\` + \`aria-labelledby\` from its \`label\`.
- \`Checkbox\` — wrapped in its own \`<label>\`.
- \`Toggle\` — \`role="switch"\` + \`aria-checked\` + \`aria-label\`.
- \`Progress\` — \`role="progressbar"\` + \`aria-valuenow/min/max\` + \`aria-label\`.
- \`Slider\` — full \`aria-valuenow/min/max\` slider semantics.
- \`Spinner\` — \`role="status"\` + \`aria-label="Loading"\`.
- \`Skeleton\` — \`aria-hidden\` (correctly decorative).
- \`Tabs\` — full \`role="tablist"/"tab"/"tabpanel"\` + \`aria-selected/controls\`.
- \`Stepper\` — \`<nav aria-label="Progress">\` + \`aria-current="step"\` on the
  active step; steps become real buttons when \`onStepClick\` is passed. It is
  display-only by design — the component owning the step index in its own
  \`useState\` is CORRECT, never a gap.
- \`Accordion\` — \`aria-expanded\` + \`aria-controls\` + \`role="region"\`.
- \`Alert\` — \`role="alert"\`; \`Toast\` — \`role="alert"\` + \`aria-live\`.
- \`Tooltip\` — \`role="tooltip"\`.
- \`Clickable\` (trait, applied via \`as={Clickable}\`) — \`role="button"\` +
  \`tabIndex={0}\` + Enter/Space keyboard activation.
- \`Icon\` — decorative by default (\`aria-hidden\`); exposes \`role="img"\` +
  the label ONLY when an \`aria-label\` prop is passed. A bare \`<Icon>\` next
  to text is CORRECTLY silent — never ask for \`aria-hidden\`, it is there.

REAL accessibility gaps worth flagging:
- A raw \`<div>\` / \`<span>\` used as a click target without \`as={Clickable}\`
  (no role, no keyboard) — or any hand-rolled interactive element.
- A hand-rolled control whose state (checked / selected / expanded /
  pressed) is conveyed ONLY by styling — e.g. a styled-div "checkbox"
  with a check icon + strikethrough label but no \`role="checkbox"\` /
  \`aria-checked\`. Note \`as={Clickable}\` adds button + keyboard
  semantics but NOT state — \`aria-checked\` / \`aria-pressed\` /
  \`aria-selected\` / \`aria-expanded\` must be passed explicitly.
- An \`<img>\` / \`<Image>\` with no alt text.
- \`<Input>\` / \`<Select>\` / \`<TextArea>\` used with NO \`label\` prop.
- An icon-only \`<Button>\` (no text child) with no \`aria-label\`.
- Live or streaming data (stream \`.latest\`, counters, real-time clocks)
  rendered in a plain element with no \`aria-live\` / \`role="status"\`.
- Inverted or skipped heading levels (an \`<Heading level={3}>\` rendered
  before the page's \`<Heading level={1}>\`).`);

  return parts.join('\n\n');
}

// =============================================================================
// Per-criterion tool definitions
// =============================================================================

/** Tier 1 tool schema: binary pass/fail with issues list */
function makeTier1Tool(criterion: string): LLMToolDef {
  return {
    name: `evaluate_${criterion}`,
    description: `Submit evaluation result for ${criterion}`,
    parameters: {
      type: 'object',
      required: ['pass'],
      properties: {
        pass: { type: 'boolean' },
        issues: {
          type: 'array',
          items: { type: 'string' },
          description: 'List of specific issues found',
        },
      },
    },
  };
}

/** Tier 2 tool schema: fail/warn/pass with issues list */
function makeTier2Tool(criterion: string): LLMToolDef {
  return {
    name: `evaluate_${criterion}`,
    description: `Submit evaluation result for ${criterion}`,
    parameters: {
      type: 'object',
      required: ['result'],
      properties: {
        result: {
          type: 'string',
          enum: ['fail', 'warn', 'pass'],
        },
        issues: {
          type: 'array',
          items: { type: 'string' },
          description: 'List of specific issues found',
        },
      },
    },
  };
}

// =============================================================================
// Per-criterion user prompts
// =============================================================================

function getCriterionPrompt(criterion: string, numberedSource: string): string {
  const sourceBlock = `\nSource code (with line numbers):\n\`\`\`tsx\n${numberedSource}\n\`\`\``;

  // Use criteria.ts as single source of truth when available
  const criterionDef = getCriterionById(criterion);
  if (criterionDef && criterionDef.tier > 0) {
    return `${criterionDef.evalInstruction}
${sourceBlock}

Return your evaluation using the evaluate_${criterion} tool.`;
  }

  // Fallback for criteria not yet in criteria.ts

  switch (criterion) {
    case 'functionality':
      return `Evaluate FUNCTIONALITY: Does this component implement ALL features from the request AND the data contract?

Check against BOTH sources:
1. **Original request** — each feature must be coded AND rendered in JSX
2. **Data contract** (if present in the context above) — verify:
   - ALL Props fields are rendered somewhere in the UI (not just accepted as params)
   - ALL useAction hooks are wired to clickable UI elements (buttons, forms, cards)
   - ALL useStream hooks have their \`.latest\` data rendered (with null guard)
   - ALL clientCapabilities hooks (useGeolocation / useCamera / etc.) are called in the component body and their values either render directly or thread into a context / action payload

A contract hook that is declared but never used in JSX is a MISSING feature.

**CRITICAL RULE:** The "issues" array must ONLY contain MISSING or BROKEN features. Do NOT list features that ARE implemented.

Design token usage (hardcoded colors, numeric spacing) is NOT a functionality issue — ignore it here.
${sourceBlock}

Return your evaluation using the evaluate_functionality tool.

Example outputs:
- All features present: {"pass": true}
- Missing features: {"pass": false, "issues": ["Add Task form is not rendered — only state logic exists", "useAction('refresh') is declared but never wired to any button"]}

Only missing or broken features go in the issues array. Nothing else.`;

    case 'crash':
      return `Evaluate CRASH SAFETY: Are there ACTUAL runtime crash scenarios in this component?

**CRITICAL RULE:** The "issues" array must ONLY contain code that WILL crash. Do NOT include safe code in the issues array — not even to explain why it's safe. If you checked a line and it has a guard, do not mention it at all.

**These WILL crash (include in issues):**
- .map() / .filter() / .length on a variable that is never initialized
- Accessing a property of a value that could be undefined without a guard
- Calling a function that is never defined
- useStream().latest.field WITHOUT null guard — .latest is T | null
- Optional Props field (has \`?\` in interface) accessed as props.field.x without guard — WILL crash if undefined
- Array item field that could be undefined: items.map(item => item.priority.toUpperCase()) when priority is optional

**These are SAFE (do NOT include in issues):**
- Optional chaining: props.items?.map() — SAFE, skip it
- Fallback values: items || [] — SAFE, skip it
- Default parameters: (items = []) — SAFE, skip it
- useState initializer: useState([]) — SAFE, skip it
- Null check: items && items.map() — SAFE, skip it
- stream.latest && stream.latest.field — SAFE, guarded
- stream.all.map(...) — SAFE, .all is always an array (never null)
- stream.all.length — SAFE, always a number
- toolResult.data && toolResult.data.field — SAFE, guarded

If everything is safe, return {"pass": true} with no issues array.
${sourceBlock}

Return your evaluation using the evaluate_crash tool.

Example outputs:
- Everything safe: {"pass": true}
- Has crash: {"pass": false, "issues": ["Line 41: selectedItem.name — selectedItem comes from .find() which returns undefined when no match, no guard"]}

Only unguarded crash risks go in the issues array. Nothing else.`;

    case 'interactivity':
      return `Evaluate INTERACTIVITY: Does this component have sufficient interactive elements for its purpose?

Consider: forms need submit buttons, lists need selection/navigation, editable content needs save/cancel, interactive elements need hover/focus feedback.

**Contract actions** (if present): every useAction hook from the data contract should be triggered by a visible, clickable UI element. A declared action with no triggering button/link/form is a missing interaction.

**Only list MISSING interactive elements in the issues array.** Do not list interactions that are present and working.
${sourceBlock}

Return your evaluation using the evaluate_interactivity tool.

Example outputs:
- Sufficient: {"result": "pass"}
- Minor gap: {"result": "warn", "issues": ["No hover feedback on list items — users can't tell items are clickable"]}
- Severe gap: {"result": "fail", "issues": ["Form has no submit button — users cannot complete the workflow"]}

Use 'fail' only for issues that block the component's core purpose.`;

    case 'accessibility':
      return `Evaluate ACCESSIBILITY: Check for missing labels, alt text, semantic HTML, keyboard navigation.

Look for: form inputs without labels, images without alt text, non-semantic elements used for interactive purposes, missing aria attributes, heading hierarchy issues.

**Only list MISSING accessibility features in the issues array.** Do not list things that are already accessible.
${sourceBlock}

Return your evaluation using the evaluate_accessibility tool.

Example outputs:
- Clean: {"result": "pass"}
- Minor issues: {"result": "warn", "issues": ["Email input missing label prop", "Task edit uses double-click with no keyboard alternative"]}
- Severe: {"result": "fail", "issues": ["No form inputs have labels — entire form is inaccessible to screen readers"]}

Use 'fail' only if the issue blocks delivery. Use 'warn' for improvements.`;

    case 'layout':
      return `Evaluate LAYOUT: Check spacing, alignment, visual grouping, and overall composition.

Look for: inconsistent spacing, cramped elements, missing visual grouping (cards/sections), overlapping concerns in single containers, poor responsive patterns.

**Only list ACTUAL layout problems in the issues array.** Do not list layout aspects that are working correctly.
${sourceBlock}

Return your evaluation using the evaluate_layout tool.

Example outputs:
- Clean: {"result": "pass"}
- Minor issues: {"result": "warn", "issues": ["Header and content area have no visual separation", "Button group lacks consistent spacing"]}
- Severe: {"result": "fail", "issues": ["All content rendered in a flat list with no visual structure — needs cards or sections"]}

Use 'fail' only for layouts that are fundamentally broken.`;

    case 'loading':
      return `Evaluate LOADING/EMPTY/ERROR STATES: Does the component handle async data, empty collections, and error cases?

Look for: arrays that could be empty but have no empty state message, missing error handling for operations that can fail.

**Contract-specific checks** (if present):
- useStream hooks deliver data over time — consider showing a "waiting for data" state before .latest arrives
- clientCapabilities hooks (e.g. useGeolocation) may return permission denials or undefined values — defensive null guards expected before threading into JSX

**Only list MISSING states in the issues array.** Do not list states that are already handled. Components that receive all data via props (no streams, no async capabilities) do NOT need loading states — return pass.
${sourceBlock}

Return your evaluation using the evaluate_loading tool.

Example outputs:
- Not applicable (props-driven): {"result": "pass"}
- Missing states: {"result": "warn", "issues": ["No empty state when items array is empty — should show 'No items found'"]}
- Severe: {"result": "fail", "issues": ["Component shows blank screen when data hasn't loaded yet — needs loading spinner"]}`;

    case 'visual':
      return `Evaluate VISUAL CONSISTENCY: Is the component using the design system correctly?

Look for: hardcoded colors instead of CSS variables, numeric/raw-CSS spacing props that bypass the t-shirt scale, style objects that bypass the design system.

**Only list ACTUAL design system violations in the issues array.** Do not list correct token usage. A t-shirt-scale spacing name (\`gap="md"\`, \`padding="lg"\`) IS correct token usage — never flag it as hardcoded; only numeric (\`padding={16}\`) or raw-CSS-length (\`gap="13px"\`) spacing is a violation. Note: intentional custom colors (e.g., status indicators, category colors) are acceptable when no semantic token fits.
${sourceBlock}

Return your evaluation using the evaluate_visual tool.

Example outputs:
- Clean: {"result": "pass"}
- Minor issues: {"result": "warn", "issues": ["Line 45: hardcoded '#f3f4f6' should use var(--ggui-color-surfaceVariant)", "Line 80: numeric padding={16} should use a scale name like padding='md'"]}
- Severe: {"result": "fail", "issues": ["Component uses entirely custom styling with no design system tokens"]}

Use 'fail' only for pervasive design system violations. Use 'warn' for isolated cases.`;

    default:
      return `Evaluate ${criterion.toUpperCase()}.${sourceBlock}\n\nReturn your evaluation using the evaluate_${criterion} tool.`;
  }
}

// =============================================================================
// Default models
// =============================================================================

function getDefaultEvalModel(provider: string): string {
  switch (provider) {
    case 'claude':
      return 'claude-haiku-4-5-20251001';
    case 'openai':
      return 'gpt-5.6-luna';
    case 'google':
      return 'gemini-3.5-flash-lite';
    default:
      return 'claude-haiku-4-5-20251001';
  }
}

// =============================================================================
// Per-criterion evaluation call
// =============================================================================

interface CriterionResult {
  criterion: string;
  tier: 1 | 2;
  issues: EvalIssue[];
  pass: boolean;
  tokens: { in: number; out: number };
  durationMs: number;
}

/** All valid categories for LLM evaluation */
const TIER_1_CRITERIA = ['functionality', 'crash'] as const;
const TIER_2_CRITERIA = ['interactivity', 'accessibility', 'layout', 'loading', 'visual'] as const;

async function evalCriterion(
  criterion: string,
  tier: 1 | 2,
  motherPrompt: string,
  numberedSource: string,
  agent: LLMAgent,
  model: string,
): Promise<CriterionResult> {
  const start = Date.now();
  const tool = tier === 1 ? makeTier1Tool(criterion) : makeTier2Tool(criterion);
  const userPrompt = getCriterionPrompt(criterion, numberedSource);

  try {
    const response = await agent.callTools(
      model,
      motherPrompt,
      userPrompt,
      [tool],
      'required',
    );

    const call = response.toolCalls[0];
    const durationMs = Date.now() - start;

    if (!call) {
      console.warn(`[eval] ${criterion}: no tool call returned`);
      return {
        criterion,
        tier,
        issues: [],
        pass: true,
        tokens: { in: response.inputTokens, out: response.outputTokens },
        durationMs,
      };
    }

    const raw = call.input as JsonObject;
    const issues: EvalIssue[] = [];
    let pass: boolean;

    if (tier === 1) {
      // Tier 1: binary pass/fail
      pass = raw.pass === true;
      const rawIssues = coerceStringArray(raw.issues);
      for (const desc of rawIssues) {
        issues.push({
          tier: 1,
          result: 'fail',
          category: criterion as EvalCategory,
          severity: 'critical',
          description: desc,
          fix: desc, // The issue description often includes the fix for tier 1
        });
      }
      // If pass is false but no issues listed, add a generic one
      if (!pass && issues.length === 0) {
        issues.push({
          tier: 1,
          result: 'fail',
          category: criterion as EvalCategory,
          severity: 'critical',
          description: `${criterion} check failed`,
          fix: `Fix ${criterion} issues`,
        });
      }
    } else {
      // Tier 2: fail/warn/pass
      const result = (raw.result as string) ?? 'pass';
      pass = result === 'pass';
      const rawIssues = coerceStringArray(raw.issues);
      const outcome = result === 'fail' ? 'fail' : 'warn';
      for (const desc of rawIssues) {
        issues.push({
          tier: 2,
          result: outcome as 'fail' | 'warn',
          category: criterion as EvalCategory,
          severity: result === 'fail' ? 'critical' : 'major',
          description: desc,
          fix: desc,
        });
      }
    }

    return {
      criterion,
      tier,
      issues,
      pass,
      tokens: { in: response.inputTokens, out: response.outputTokens },
      durationMs,
    };
  } catch (e) {
    const durationMs = Date.now() - start;
    console.error(`[eval] ${criterion}: error — ${e instanceof Error ? e.message : e}`);
    // On error, treat as pass to avoid blocking on flaky LLM calls
    return {
      criterion,
      tier,
      issues: [],
      pass: true,
      tokens: { in: 0, out: 0 },
      durationMs,
    };
  }
}

// =============================================================================
// Pre-warming
// =============================================================================

/**
 * Pre-warm evaluation context during the coding phase.
 *
 * Builds the mother prompt during coding so it's ready when eval starts.
 * The result can be passed to runLLMEvaluation() to avoid redundant work.
 */
export async function preWarmEval(
  context: LLMEvalContext,
  _config: LLMEvalConfig,
): Promise<PreWarmedEvalContext> {
  const motherPrompt = buildMotherPrompt({
    originalPrompt: context.originalPrompt,
    contract: context.contract,
    designSystemSummary: context.designContext,
  });

  return { motherPrompt, dynamicCriteria: [] };
}

// =============================================================================
// Response parsing (backward compat — used by tests)
// =============================================================================

/** Tier 1 categories — always tier 1 regardless of what LLM returns */
const TIER_1_CATEGORIES = new Set<string>(['functionality', 'crash']);

/** Tier 2 categories */
const TIER_2_CATEGORIES = new Set<string>([
  'interactivity',
  'accessibility',
  'layout',
  'loading',
  'visual',
]);

/** All valid categories for LLM evaluation */
const VALID_CATEGORIES = new Set<string>([
  ...TIER_1_CATEGORIES,
  ...TIER_2_CATEGORIES,
]);

/**
 * Parse the raw tool call input into a typed EvalResult.
 *
 * Exported for testing — not part of the public API.
 * Retained for backward compatibility with existing tests.
 */
export function parseEvalResponse(raw: JsonObject): EvalResult {
  const rawIssues = (raw.issues ?? []) as Array<JsonObject>;
  const rawPass = (raw.pass ?? []) as string[];

  const issues: EvalIssue[] = rawIssues
    .filter((i) => {
      const cat = i.category as string;
      return VALID_CATEGORIES.has(cat);
    })
    .map((i) => {
      const category = i.category as string;
      const isTier1 = TIER_1_CATEGORIES.has(category);

      // Enforce correct tier based on category
      const tier = isTier1 ? 1 : 2;

      // Tier 1 issues are always 'fail' (critical checks)
      // Tier 2 issues: LLM decides 'fail' or 'warn'
      const result = isTier1
        ? 'fail'
        : ((i.result as string) === 'fail' ? 'fail' : 'warn');

      return {
        tier: tier as 1 | 2,
        result: result as 'fail' | 'warn',
        category: category as EvalCategory,
        severity: (i.severity as 'critical' | 'major') || (isTier1 ? 'critical' : 'major'),
        description: (i.description as string) || '',
        fix: (i.fix as string) || '',
      };
    });

  return {
    issues,
    pass: rawPass,
  };
}

// =============================================================================
// Main evaluation function
// =============================================================================

/**
 * Run LLM evaluation with parallel per-criterion calls.
 *
 * Runs 7 static criteria (2 tier 1 + 5 tier 2) plus 0-3 dynamic criteria
 * generated from the user's request. All run in parallel with a shared
 * mother prompt (cached by the provider). Wall time ~2s for all criteria.
 *
 * @param preWarmedContext - Optional pre-warmed context from `preWarmEval()`.
 *   If provided, skips mother prompt building and dynamic criteria generation.
 */
export async function runLLMEvaluation(
  context: LLMEvalContext,
  config: LLMEvalConfig,
  preWarmedContext?: PreWarmedEvalContext | null,
): Promise<EvalResult & { inputTokens: number; outputTokens: number }> {
  const providerName = config.provider;
  const model = config.model || getDefaultEvalModel(providerName);
  const routerProvider =
    providerName === 'claude'
      ? ('anthropic' as const)
      : (providerName as 'openai' | 'google');

  // Use pre-warmed context if available, otherwise build fresh
  const motherPrompt = preWarmedContext?.motherPrompt ?? buildMotherPrompt({
    originalPrompt: context.originalPrompt,
    contract: context.contract,
    designSystemSummary: context.designContext,
  });

  // Add line numbers to source
  const numberedSource = context.sourceCode
    .split('\n')
    .map((line, i) => `${String(i + 1).padStart(4)} | ${line}`)
    .join('\n');

  // Create a single agent — all calls share the same client + cache.
  // Pass the full config (not just the bare provider string) so a
  // caller-supplied `routeOverride` (#484) reaches the constructed
  // `LLMAgent` instance and it never falls back to `process.env`.
  const agent = createAgent({
    provider: routerProvider,
    model,
    routeOverride: config.routeOverride,
    onRetry: config.onRetry,
  });

  // Dynamic criteria removed — they generated overly strict edge-case checks
  // that wasted coding turns on nitpicks. Will be replaced by a holistic
  // reviewer in auto-improve mode.

  const evalStart = Date.now();

  const totalCriteriaCount = TIER_1_CRITERIA.length + TIER_2_CRITERIA.length;

  // Run ALL criteria in parallel: 7 static (2 tier 1 + 5 tier 2)
  const results = await Promise.all([
    ...TIER_1_CRITERIA.map(c => evalCriterion(c, 1, motherPrompt, numberedSource, agent, model)),
    ...TIER_2_CRITERIA.map(c => evalCriterion(c, 2, motherPrompt, numberedSource, agent, model)),
  ]);

  // Merge results
  const allIssues: EvalIssue[] = [];
  const passCategories: string[] = [];
  let totalIn = 0;
  let totalOut = 0;
  let cachedTokensEstimate = 0;

  for (const r of results) {
    allIssues.push(...r.issues);
    if (r.pass) {
      passCategories.push(r.criterion);
    }
    totalIn += r.tokens.in;
    totalOut += r.tokens.out;
  }

  // Estimate cached tokens (mother prompt sent N times, cached N-1 times)
  const firstResult = results[0];
  if (firstResult && firstResult.tokens.in > 0) {
    cachedTokensEstimate = firstResult.tokens.in * (totalCriteriaCount - 1);
  }

  const wallMs = Date.now() - evalStart;

  // Log per-criterion results
  for (const r of results) {
    const issueCount = r.issues.length;
    const status = r.pass ? 'PASS' : r.tier === 1 ? 'FAIL' : (r.issues.some(i => i.result === 'fail') ? 'FAIL' : 'WARN');
    const issueLabel = issueCount > 0 ? ` (${issueCount} issue${issueCount > 1 ? 's' : ''})` : '';
    console.log(`[eval] ${r.criterion}: ${status}${issueLabel} ${(r.durationMs / 1000).toFixed(1)}s`);
  }
  console.log(
    `[eval] total: ${(wallMs / 1000).toFixed(1)}s wall (${totalCriteriaCount} parallel) | ` +
    `in=${totalIn} out=${totalOut} cached~=${cachedTokensEstimate} | ` +
    `issues=${allIssues.length} pass=${passCategories.length}`,
  );

  return {
    issues: allIssues,
    pass: passCategories,
    inputTokens: totalIn,
    outputTokens: totalOut,
  };
}