export type LearningFeedbackVerdict = 'accept' | 'edit' | 'reject' | null; export type LearningFeedbackReason = 'missing_citation' | 'wrong_format' | 'wrong_category' | 'too_long' | 'missed_clause' | 'needs_review' | 'great' | null; export interface LearningMetadataRow { tenant_id: string; vertical: string; outcome_slug: string; prompt_version: string; model_id: string; effort: 'low' | 'medium' | 'high' | 'xhigh' | 'unknown'; quality_score: number; passed: boolean; escalated: boolean; cost_cents: number; input_tokens: number; output_tokens: number; duration_ms: number; guardrail_tier: string; feedback_verdict: LearningFeedbackVerdict; feedback_reason: LearningFeedbackReason; timestamp: string; } export interface LearningAggregate { key: string; vertical: string; outcome_slug: string; prompt_version: string; model_id: string; effort: string; sample_count: number; distinct_tenant_count: number; pass_rate: number; mean_quality_score: number; escalation_rate: number; mean_cost_cents: number; mean_latency_ms: number; feedback_distribution: { accept: number; edit: number; reject: number; accept_rate: number; reject_rate: number }; reason_counts: Record; revision_suggestion: string; } export interface LearningProposal { proposal_id: string; status: 'shadow'; vertical: string; outcome_slug: string; proposed: { prompt_version: string; model_id: string; effort: string; fit_score: number }; current: { prompt_version: string; model_id: string; effort: string; fit_score: number } | null; sample_count: number; distinct_tenant_count: number; revision_suggestion: string; human_promotion_required: true; } const ALLOWED = new Set(['tenant_id','vertical','outcome_slug','prompt_version','model_id','effort','quality_score','passed','escalated','cost_cents','input_tokens','output_tokens','duration_ms','guardrail_tier','feedback_verdict','feedback_reason','timestamp']); const REASONS = new Set(['missing_citation','wrong_format','wrong_category','too_long','missed_clause','needs_review','great', null]); const VERDICTS = new Set(['accept','edit','reject', null]); const CONTENT = /^(prompt|completion|content|messages|input|inputs|output|outputs|text|raw|transcript|document|free_text|edited_text|email|phone|address|api_key|secret)$/i; export function assertLearningMetadataOnly(value: Record): void { for (const key of Object.keys(value)) { if (!ALLOWED.has(key)) throw new Error('learning field not allowed: ' + key); if (key !== 'prompt_version' && CONTENT.test(key)) throw new Error('learning data-plane field not allowed: ' + key); const child = value[key]; if (child && typeof child === 'object') throw new Error('learning nested value not allowed: ' + key); if (typeof child === 'string' && (child.length > 160 || child.includes('\n'))) throw new Error('learning value looks like content: ' + key); } } export function normalizeLearningMetadataRow(input: Record): LearningMetadataRow { assertLearningMetadataOnly(input); const verdict = clean(input.feedback_verdict, null, 20) as LearningFeedbackVerdict; const reason = clean(input.feedback_reason, null, 40) as LearningFeedbackReason; if (!VERDICTS.has(verdict)) throw new Error('invalid learning verdict'); if (!REASONS.has(reason)) throw new Error('invalid learning reason'); const effort = clean(input.effort, 'unknown', 20); return { tenant_id: clean(input.tenant_id, 'anonymous', 96), vertical: clean(input.vertical, 'unknown', 48), outcome_slug: clean(input.outcome_slug, 'unknown', 96), prompt_version: clean(input.prompt_version, 'unknown', 48), model_id: clean(input.model_id, 'unknown', 120), effort: ['low','medium','high','xhigh'].includes(effort) ? effort as LearningMetadataRow['effort'] : 'unknown', quality_score: Math.min(1, number(input.quality_score, 0)), passed: input.passed !== false, escalated: input.escalated === true, cost_cents: Math.round(number(input.cost_cents, 0)), input_tokens: Math.round(number(input.input_tokens, 0)), output_tokens: Math.round(number(input.output_tokens, 0)), duration_ms: Math.round(number(input.duration_ms, 0)), guardrail_tier: clean(input.guardrail_tier, 'unknown', 64), feedback_verdict: verdict, feedback_reason: reason, timestamp: normalizeTimestamp(input.timestamp), }; } export function aggregateLearningMetadata(rows: Record[]): LearningAggregate[] { const groups = new Map(); for (const raw of rows) { const row = normalizeLearningMetadataRow(raw); const key = [row.vertical,row.outcome_slug,row.prompt_version,row.model_id,row.effort].join('|'); const group = groups.get(key) ?? { key, vertical: row.vertical, outcome_slug: row.outcome_slug, prompt_version: row.prompt_version, model_id: row.model_id, effort: row.effort, sample_count: 0, tenants: new Set(), passed: 0, escalated: 0, cost: 0, latency: 0, quality: 0, verdicts: { accept: 0, edit: 0, reject: 0 }, reasons: {} as Record }; group.sample_count += 1; group.tenants.add(row.tenant_id); if (row.passed) group.passed += 1; if (row.escalated) group.escalated += 1; group.cost += row.cost_cents; group.latency += row.duration_ms; group.quality += row.quality_score; if (row.feedback_verdict) group.verdicts[row.feedback_verdict] += 1; if (row.feedback_reason) group.reasons[row.feedback_reason] = (group.reasons[row.feedback_reason] || 0) + 1; groups.set(key, group); } return [...groups.values()].map((g) => { const totalVerdicts = Math.max(1, g.verdicts.accept + g.verdicts.edit + g.verdicts.reject); return { key: g.key, vertical: g.vertical, outcome_slug: g.outcome_slug, prompt_version: g.prompt_version, model_id: g.model_id, effort: g.effort, sample_count: g.sample_count, distinct_tenant_count: g.tenants.size, pass_rate: round(g.passed / g.sample_count), mean_quality_score: round(g.quality / g.sample_count), escalation_rate: round(g.escalated / g.sample_count), mean_cost_cents: round(g.cost / g.sample_count), mean_latency_ms: round(g.latency / g.sample_count), feedback_distribution: { ...g.verdicts, accept_rate: round(g.verdicts.accept / totalVerdicts), reject_rate: round(g.verdicts.reject / totalVerdicts) }, reason_counts: g.reasons, revision_suggestion: revisionSuggestion(g.reasons) }; }); } export function proposeLearningMetadataImprovements(aggregates: LearningAggregate[], current: Record = {}, opts: { minRuns?: number; minTenants?: number } = {}): LearningProposal[] { const minRuns = opts.minRuns ?? 20, minTenants = opts.minTenants ?? 5; const byOutcome = new Map(); for (const aggregate of aggregates) { const key = aggregate.vertical + '|' + aggregate.outcome_slug; byOutcome.set(key, [...(byOutcome.get(key) || []), aggregate]); } const proposals: LearningProposal[] = []; for (const [key, rows] of byOutcome.entries()) { const eligible = rows.filter((r) => r.sample_count >= minRuns && r.distinct_tenant_count >= minTenants).sort((a, b) => fitScore(b) - fitScore(a)); if (!eligible.length) continue; const best = eligible[0]!; const baseline = current[key] || rows[0]!; if (best.key === baseline.key) continue; proposals.push({ proposal_id: 'ag_lp_' + Math.abs(hash(best.key)).toString(16), status: 'shadow', vertical: best.vertical, outcome_slug: best.outcome_slug, proposed: pick(best), current: baseline ? pick(baseline) : null, sample_count: best.sample_count, distinct_tenant_count: best.distinct_tenant_count, revision_suggestion: best.revision_suggestion, human_promotion_required: true }); } return proposals; } export function revisionSuggestion(reasons: Record): string { const top = Object.entries(reasons).sort((a, b) => b[1] - a[1])[0]?.[0]; return ({ missing_citation: 'tighten cite-or-refuse instruction', wrong_format: 'make output schema stricter', wrong_category: 'add category examples', too_long: 'shorten answer format', missed_clause: 'expand clause watchlist', needs_review: 'lower reviewer cascade threshold', great: 'preserve current template' } as Record)[top || ''] || 'keep collecting metadata'; } function normalizeTimestamp(value: unknown) { const time = value == null ? Date.now() : new Date(String(value)).getTime(); return new Date(Number.isFinite(time) ? time : Date.now()).toISOString(); } function pick(a: LearningAggregate) { return { prompt_version: a.prompt_version, model_id: a.model_id, effort: a.effort, fit_score: fitScore(a) }; } function fitScore(a: LearningAggregate) { return round(a.pass_rate * 0.32 + a.mean_quality_score * 0.28 + a.feedback_distribution.accept_rate * 0.24 - a.feedback_distribution.reject_rate * 0.18 - a.escalation_rate * 0.08 - Math.min(0.12, a.mean_cost_cents / 2000) - Math.min(0.08, a.mean_latency_ms / 120000)); } function clean(value: unknown, fallback: any, max: number) { const raw = value == null ? fallback : String(value).toLowerCase().replace(/[^a-z0-9_./:-]/g, '_').slice(0, max); return raw || fallback; } function number(value: unknown, fallback: number) { const n = Number(value); return Number.isFinite(n) && n >= 0 ? n : fallback; } function round(value: number) { return Math.round(value * 10000) / 10000; } function hash(value: string) { let h = 0; for (let i = 0; i < value.length; i++) h = Math.imul(31, h) + value.charCodeAt(i) | 0; return h; } export interface PortableAiTeamOutcome { vertical: string; name: string; outcome_slug: string; prompt_version: string; caps?: Record; effort?: string; guardrail_tier?: string; enabled?: boolean; } export interface PortableAiTeamExport { schema_version: 1; export_type: 'agentguard_ai_team'; exported_at: string; zero_data_plane: true; no_lock_in: true; outcomes: PortableAiTeamOutcome[]; } export function exportAiTeamConfig(outcomes: PortableAiTeamOutcome[]): PortableAiTeamExport { return { schema_version: 1, export_type: 'agentguard_ai_team', exported_at: new Date().toISOString(), zero_data_plane: true, no_lock_in: true, outcomes: outcomes.map((outcome) => ({ vertical: clean(outcome.vertical, 'unknown', 48) as string, name: String(outcome.name || 'AI teammate').slice(0, 120), outcome_slug: clean(outcome.outcome_slug, 'unknown', 96) as string, prompt_version: clean(outcome.prompt_version, 'unknown', 48) as string, caps: outcome.caps, effort: outcome.effort, guardrail_tier: outcome.guardrail_tier, enabled: outcome.enabled !== false, })), }; }