"use client" import * as React from 'react' import { AlertTriangle, Check, CheckCircle2, ClipboardCheck, FlaskConical, MinusCircle, Plus, X, XCircle } from 'lucide-react' import { Button } from '@open-mercato/ui/primitives/button' import { Spinner } from '@open-mercato/ui/primitives/spinner' import { StatusBadge } from '@open-mercato/ui/primitives/status-badge' import { JsonDisplay } from '@open-mercato/ui/backend/JsonDisplay' import { SectionHeader } from '@open-mercato/ui/backend/SectionHeader' import { apiCall, apiCallOrThrow } from '@open-mercato/ui/backend/utils/apiCall' import { flash } from '@open-mercato/ui/backend/FlashMessages' import { useGuardedMutation } from '@open-mercato/ui/backend/injection/useGuardedMutation' import { useT } from '@open-mercato/shared/lib/i18n/context' import { mapRunDetail, type EvalResultView, type RunDetailView } from '../../components/types' import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@open-mercato/ui/primitives/table' type PanelState = 'loading' | 'ready' | 'forbidden' /** * Post-run evaluation watch panel for the Playground. Every run is scored online * against its agent's assertions; this surfaces those verdicts, and — when the * input matched an approved golden case — the run's actual-vs-expected diff plus * the golden's own rules. On no match, offers to capture the run as a golden case. */ export function PlaygroundEvalPanel({ runId, agentId }: { runId: string; agentId: string }) { const t = useT() const [detail, setDetail] = React.useState(null) const [state, setState] = React.useState('loading') const [savedGolden, setSavedGolden] = React.useState(false) const [savingGolden, setSavingGolden] = React.useState(false) const { runMutation, retryLastMutation } = useGuardedMutation<{ retryLastMutation: () => Promise }>({ contextId: 'agent_orchestrator.playground.eval', }) React.useEffect(() => { let cancelled = false setState('loading') setDetail(null) setSavedGolden(false) async function load() { const call = await apiCall>( `/api/agent_orchestrator/runs/${encodeURIComponent(runId)}`, undefined, { fallback: {} }, ) if (cancelled) return if (!call.ok || !call.result) { setState('forbidden') return } setDetail(mapRunDetail(call.result)) setState('ready') } load() return () => { cancelled = true } }, [runId]) const online = (detail?.evalResults ?? []).filter((row) => !row.matchedEvalCaseId) const golden = (detail?.evalResults ?? []).filter((row) => row.matchedEvalCaseId) // Only surface golden verdicts that ADD information over the online list: an // assertion the online plane skipped (expected-based, so only the golden plane // could score it) or one with no online counterpart. Expected-agnostic // assertions produce identical verdicts on both planes — don't list them twice. const onlineByKey = new Map(online.map((row) => [row.assertionKey, row])) const goldenExtra = golden.filter((row) => { const counterpart = onlineByKey.get(row.assertionKey) return !counterpart || counterpart.passed === null }) const applied = online.filter((row) => row.passed !== null) const matched = detail?.goldenCase ?? null // Overall run verdict: a failed gate fails the run; a failed warn downgrades a // pass to "pass with warnings"; no applicable assertions → not evaluated. const gateFailed = applied.some((row) => row.severity === 'gate' && row.passed === false) const warnFailed = applied.some((row) => row.severity === 'warn' && row.passed === false) const verdict: RunVerdict = applied.length === 0 ? 'none' : gateFailed ? 'failed' : warnFailed ? 'warnings' : 'passed' const saveGolden = React.useCallback(async () => { setSavingGolden(true) try { await runMutation({ operation: async () => { await apiCallOrThrow(`/api/agent_orchestrator/runs/${encodeURIComponent(runId)}/eval-case`, { method: 'POST' }) }, context: { retryLastMutation }, mutationPayload: { runId }, }) setSavedGolden(true) flash(t('agent_orchestrator.playground.eval.savedGolden', 'Saved as a draft golden case.'), 'success') } catch (err) { flash(err instanceof Error ? err.message : t('agent_orchestrator.playground.eval.saveGoldenError', 'Could not save golden case.'), 'error') } finally { setSavingGolden(false) } }, [runId, runMutation, retryLastMutation, t]) return (
{t('agent_orchestrator.playground.eval.title', 'Evaluation')}
{state === 'ready' ? (
{matched ? ( {t('agent_orchestrator.playground.eval.goldenBadge', 'Golden matched')} ) : null}
) : null}
{state === 'loading' ? (
{t('agent_orchestrator.playground.eval.loading', 'Scoring this run…')}
) : state === 'forbidden' ? (

{t('agent_orchestrator.playground.eval.forbidden', 'You don’t have permission to view this run’s evaluation.')}

) : ( <>
{online.length === 0 ? (

{t('agent_orchestrator.playground.eval.noAssertions', 'No assertions apply to this agent yet.')}

) : ( )}
{matched ? (
{goldenIcon(detail?.run.goldenPassed ?? null)}
{t(goldenTitleKey(detail?.run.goldenPassed ?? null), goldenTitleFallback(detail?.run.goldenPassed ?? null))} {matched.id.slice(0, 8)}
{goldenExtra.length > 0 ? : null}

{t('agent_orchestrator.playground.eval.actual', 'Actual')}

{t('agent_orchestrator.playground.eval.expected', 'Expected (golden)')}

) : (

{t('agent_orchestrator.playground.eval.noMatch', 'No golden case matches this input yet.')}

{savedGolden ? ( {t('agent_orchestrator.playground.eval.savedGoldenShort', 'Saved as draft golden')} ) : ( )}
)} )}
) } type RunVerdict = 'passed' | 'warnings' | 'failed' | 'none' function VerdictBadge({ verdict, t }: { verdict: RunVerdict; t: ReturnType }) { const meta = { passed: { variant: 'success' as const, Icon: CheckCircle2, key: 'agent_orchestrator.playground.eval.verdict.passed', fallback: 'Passed' }, warnings: { variant: 'warning' as const, Icon: AlertTriangle, key: 'agent_orchestrator.playground.eval.verdict.warnings', fallback: 'Pass with warnings' }, failed: { variant: 'error' as const, Icon: XCircle, key: 'agent_orchestrator.playground.eval.verdict.failed', fallback: 'Failed' }, none: { variant: 'neutral' as const, Icon: MinusCircle, key: 'agent_orchestrator.playground.eval.verdict.none', fallback: 'Not evaluated' }, }[verdict] return ( {t(meta.key, meta.fallback)} ) } function AssertionTable({ rows, t }: { rows: EvalResultView[]; t: ReturnType }) { return (
{rows.map((row) => { const meta = verdictMeta(row.passed) const { actual, expected } = evidenceActualExpected(row.evidence) const actualText = row.passed === null ? t('agent_orchestrator.playground.eval.skipped', 'skipped') : actual ?? t(meta.labelKey, meta.labelFallback) return ( {row.assertionKey} {t(`agent_orchestrator.playground.eval.severity.${row.severity}`, row.severity)} {actualText} {expected ?? '—'} ) })}
) } const OP_SYMBOL: Record = { eq: '=', ne: '≠', gt: '>', gte: '≥', lt: '<', lte: '≤', contains: '⊇', in: '∈' } function formatEvidenceValue(value: unknown): string { if (value === '') return '∅' if (value === null || value === undefined) return '—' if (typeof value === 'number') return Number.isInteger(value) ? String(value) : value.toFixed(2) if (typeof value === 'string') return value try { return JSON.stringify(value) } catch { return String(value) } } /** * Turns a scorer's `evidence` into a human-readable (actual, expected) pair so * each assertion row shows WHAT it checked and WHAT it observed — not a bare * 0–1 score. Covers the json_path_compare and confidence_threshold evidence * shapes; unknown shapes fall back to just the actual, or the verdict word. */ function evidenceActualExpected(evidence: unknown): { actual: string | null; expected: string | null } { if (evidence && typeof evidence === 'object' && !Array.isArray(evidence)) { const rec = evidence as Record if ('operator' in rec && 'actual' in rec) { const op = OP_SYMBOL[String(rec.operator)] ?? String(rec.operator) return { actual: formatEvidenceValue(rec.actual), expected: `${op} ${formatEvidenceValue(rec.value)}` } } if ('confidence' in rec && 'threshold' in rec) { const dir = rec.direction === 'lte' ? '≤' : '≥' return { actual: formatEvidenceValue(rec.confidence), expected: `${dir} ${formatEvidenceValue(rec.threshold)}` } } if ('actual' in rec) return { actual: formatEvidenceValue(rec.actual), expected: null } } return { actual: null, expected: null } } function verdictMeta(passed: boolean | null): { Icon: React.ComponentType<{ className?: string }> className: string labelKey: string labelFallback: string } { if (passed === null) return { Icon: MinusCircle, className: 'text-muted-foreground', labelKey: 'agent_orchestrator.playground.eval.skipped', labelFallback: 'skipped' } if (passed) return { Icon: Check, className: 'text-status-success-icon', labelKey: 'agent_orchestrator.playground.eval.pass', labelFallback: 'pass' } return { Icon: X, className: 'text-status-error-icon', labelKey: 'agent_orchestrator.playground.eval.fail', labelFallback: 'fail' } } function goldenIcon(passed: boolean | null) { if (passed === true) return if (passed === false) return return } function goldenTitleKey(passed: boolean | null): string { if (passed === true) return 'agent_orchestrator.playground.eval.goldenPassed' if (passed === false) return 'agent_orchestrator.playground.eval.goldenFailed' return 'agent_orchestrator.playground.eval.goldenAdvisory' } function goldenTitleFallback(passed: boolean | null): string { if (passed === true) return 'Matched a golden case — passed' if (passed === false) return 'Matched a golden case — failed' return 'Matched a golden case — advisory (no gate)' }