import { mkdirSync, writeFileSync } from 'fs'; import { dirname, join } from 'path'; import { homedir } from 'os'; import type { OutcomeRuntimeReceipt } from './types'; export interface OutcomeLearningProposal { vertical: string; outcome: string; currentPromptVersion: string; proposedPromptVersion: string; recommendedModel: string; sampleSize: number; metricDeltas: { passRateDelta: number; qualityScoreDelta: number; costCentsDelta: number; escalationRateDelta: number; }; apply: 'manual_only'; } interface Aggregate { vertical: string; outcome: string; promptVersion: string; model: string; count: number; pass: number; quality: number; cost: number; escalations: number; } const DATA_PLANE_KEYS = /^(prompt|completion|content|messages|input|inputs|output|outputs|text|raw|transcript)$/i; export function proposeOutcomeImprovements(receipts: OutcomeRuntimeReceipt[]): OutcomeLearningProposal[] { for (const receipt of receipts) assertMetadataOnly(receipt); const aggregates = aggregate(receipts); const groups = new Map(); for (const row of aggregates) { const key = row.vertical + ':' + row.outcome; const existing = groups.get(key) ?? []; existing.push(row); groups.set(key, existing); } const proposals: OutcomeLearningProposal[] = []; for (const rows of groups.values()) { if (rows.length < 2) continue; const current = rows.slice().sort((a, b) => b.count - a.count)[0]!; const best = rows.slice().sort((a, b) => scoreAggregate(b) - scoreAggregate(a))[0]!; if (best === current || best.count < 3) continue; proposals.push({ vertical: current.vertical, outcome: current.outcome, currentPromptVersion: current.promptVersion, proposedPromptVersion: best.promptVersion, recommendedModel: best.model, sampleSize: best.count, metricDeltas: { passRateDelta: rate(best.pass, best.count) - rate(current.pass, current.count), qualityScoreDelta: avg(best.quality, best.count) - avg(current.quality, current.count), costCentsDelta: avg(best.cost, best.count) - avg(current.cost, current.count), escalationRateDelta: rate(best.escalations, best.count) - rate(current.escalations, current.count), }, apply: 'manual_only', }); } return proposals; } export function persistLearningProposals( receipts: OutcomeRuntimeReceipt[], path = join(homedir(), '.agentguard', 'learning-proposals.json'), ): OutcomeLearningProposal[] { const proposals = proposeOutcomeImprovements(receipts); mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, JSON.stringify({ generatedAt: new Date().toISOString(), proposals }, null, 2)); return proposals; } function aggregate(receipts: OutcomeRuntimeReceipt[]): Aggregate[] { const map = new Map(); for (const receipt of receipts) { const model = receipt.reviewer?.model ?? receipt.drafter.model; const key = [receipt.vertical, receipt.outcome, receipt.promptVersion, model].join(':'); const row = map.get(key) ?? { vertical: receipt.vertical, outcome: receipt.outcome, promptVersion: receipt.promptVersion, model, count: 0, pass: 0, quality: 0, cost: 0, escalations: 0, }; row.count += 1; if (receipt.pass) row.pass += 1; row.quality += receipt.qualityScore; row.cost += receipt.totalCostCents; if (receipt.triggerFired.length > 0) row.escalations += 1; map.set(key, row); } return [...map.values()]; } function scoreAggregate(row: Aggregate): number { return rate(row.pass, row.count) * 0.5 + avg(row.quality, row.count) * 0.35 - Math.min(0.15, avg(row.cost, row.count) / 1000) - rate(row.escalations, row.count) * 0.1; } function rate(value: number, count: number): number { return count === 0 ? 0 : value / count; } function avg(value: number, count: number): number { return count === 0 ? 0 : value / count; } function assertMetadataOnly(value: unknown, path = 'receipt'): void { if (value == null) return; if (Array.isArray(value)) { value.forEach((child, index) => assertMetadataOnly(child, path + '[' + index + ']')); return; } if (typeof value !== 'object') return; for (const [key, child] of Object.entries(value as Record)) { if (DATA_PLANE_KEYS.test(key)) { throw new Error('Learning loop received data-plane field: ' + path + '.' + key); } assertMetadataOnly(child, path + '.' + key); } }