/** * Evaluator Catalog page (#55 Phase 4). Route: /eval/evaluators * * Browse the catalog of reusable scorer definitions (built-ins + your own), * filter by scorer type / status / kind, create new ones, and run the * draft → published → verified lifecycle. Built-ins are read-only. */ import React, { useState, useCallback } from 'react'; import { useApi } from '../../hooks/useApi'; import { listEvaluators, createEvaluator, publishEvaluator, verifyEvaluator, deleteEvaluator, } from '../../api/eval'; import type { EvaluatorDefinition, ScorerType } from '../../api/eval'; const SCORER_TYPES: ScorerType[] = ['compliance', 'llm_judge', 'regex', 'exact_match', 'contains', 'composite', 'custom']; function placeholderConfig(t: ScorerType): string { if (t === 'compliance') return JSON.stringify({ rules: [{ id: 'no_delete', type: 'tool_denylist', tools: ['delete_*'] }] }, null, 2); if (t === 'llm_judge') return JSON.stringify({ rubric: 'Did the agent ...? Score 1.0 if compliant, 0.0 otherwise.' }, null, 2); if (t === 'regex') return JSON.stringify({ pattern: '^OK' }, null, 2); return JSON.stringify({}, null, 2); } function Badge({ children, color }: { children: React.ReactNode; color: string }): React.ReactElement { return {children}; } export function Evaluators(): React.ReactElement { const [scorerType, setScorerType] = useState(''); const [status, setStatus] = useState(''); const [kind, setKind] = useState(''); // '' | 'builtin' | 'custom' const [showCreate, setShowCreate] = useState(false); const [name, setName] = useState(''); const [type, setType] = useState('compliance'); const [config, setConfig] = useState(placeholderConfig('compliance')); const [tags, setTags] = useState(''); const [busy, setBusy] = useState(false); const [formErr, setFormErr] = useState(null); const [actionErr, setActionErr] = useState(null); const { data, loading, error, refetch } = useApi( () => listEvaluators({ scorerType: scorerType || undefined, status: status || undefined, builtin: kind === '' ? undefined : kind === 'builtin', }), [scorerType, status, kind], ); const evaluators = data?.evaluators ?? []; const handleCreate = useCallback(async () => { setFormErr(null); if (!name.trim()) { setFormErr('Name is required'); return; } let parsed: Record; try { parsed = JSON.parse(config); } catch { setFormErr('configTemplate must be valid JSON'); return; } setBusy(true); try { await createEvaluator({ name: name.trim(), scorerType: type, configTemplate: parsed, tags: tags.split(',').map((t) => t.trim()).filter(Boolean) }); setShowCreate(false); setName(''); setTags(''); setConfig(placeholderConfig(type)); refetch(); } catch (e) { setFormErr(e instanceof Error ? e.message : 'Create failed'); } finally { setBusy(false); } }, [name, type, config, tags, refetch]); const act = useCallback(async (fn: () => Promise) => { setActionErr(null); try { await fn(); refetch(); } catch (e) { setActionErr(e instanceof Error ? e.message : 'Action failed'); refetch(); } }, [refetch]); return (

Evaluator Catalog

Reusable scorer definitions — reference one by id from session scoring. Built-ins are read-only.

{/* Filters */}
{error &&
Error loading evaluators: {error}
} {actionErr && (
Action failed: {actionErr}
)} {/* Create form */} {showCreate && (

New Evaluator

{formErr &&
{formErr}
}
setName(e.target.value)} placeholder="e.g., No PII exfiltration" className="w-full rounded border border-gray-300 px-3 py-2 text-sm" />
setTags(e.target.value)} placeholder="pii, security" className="w-full rounded border border-gray-300 px-3 py-2 text-sm" />