import type { OutcomeQualityGateConfig, OutcomeTemplate } from './types'; export interface OutcomeQualityGateResult { score: number; riskScore: number; passed: boolean; issues: string[]; triggerFired: string[]; } const DEFAULT_PASS_THRESHOLD = 0.7; const DEFAULT_REVIEWER_TRIGGER = 0.55; const GENERIC_RISK_TERMS = [ 'unlimited liability', 'indemnification', 'non-compete', 'material weakness', 'coverage denied', 'claim denied', 'wire instructions', 'refund denied', 'regulatory filing', ]; export function scoreOutcomeOutput( outputText: string, template: OutcomeTemplate, providerScore?: number, ): OutcomeQualityGateResult { const config = template.qualityGate ?? {}; const issues: string[] = []; const text = outputText.trim(); let score = bounded(providerScore ?? heuristicQualityScore(text, template, issues)); if (text.length === 0) { score = 0; issues.push('empty_output'); } const riskScore = triageRisk(text, config, template.vertical); const threshold = config.passThreshold ?? DEFAULT_PASS_THRESHOLD; const reviewerThreshold = config.reviewerTriggerScore ?? DEFAULT_REVIEWER_TRIGGER; const triggerFired: string[] = []; if (riskScore >= reviewerThreshold) triggerFired.push('high_risk_classifier_score_above:' + reviewerThreshold); const keywordTriggers = keywordHits(text, [...GENERIC_RISK_TERMS, ...(config.riskKeywords ?? [])]); if (keywordTriggers.length) triggerFired.push('keyword_watchlist:' + keywordTriggers.join(',')); if (score < threshold) triggerFired.push('quality_score_below:' + threshold); return { score, riskScore, passed: score >= threshold, issues: [...new Set(issues)], triggerFired: [...new Set(triggerFired)], }; } export function shouldRunReviewer(result: OutcomeQualityGateResult): boolean { return result.triggerFired.length > 0; } export function triageRisk( outputText: string, config: OutcomeQualityGateConfig = {}, vertical = 'general', ): number { const lower = outputText.toLowerCase(); const terms = [...GENERIC_RISK_TERMS, ...(config.riskKeywords ?? []), ...verticalRiskTerms(vertical)]; const hits = keywordHits(lower, terms).length; const lengthPenalty = outputText.length > 6000 ? 0.08 : 0; const disclaimerPenalty = /(legal|tax|insurance) advice/i.test(outputText) ? -0.05 : 0; return bounded((hits * 0.18) + lengthPenalty + disclaimerPenalty); } function heuristicQualityScore(text: string, template: OutcomeTemplate, issues: string[]): number { let score = 0.74; if (template.disclaimer && !text.toLowerCase().includes(shortDisclaimerAnchor(template.disclaimer))) { issues.push('missing_disclaimer'); score -= 0.12; } if (/cite|citation|source|provenance/i.test(template.systemPrompt) && !/(source|citation|provenance|cited)/i.test(text)) { issues.push('missing_source_reference'); score -= 0.12; } if (template.outputFormat.toLowerCase().includes('json')) { try { JSON.parse(text); } catch { issues.push('invalid_json_output'); score -= 0.2; } } return bounded(score); } function shortDisclaimerAnchor(disclaimer: string): string { return disclaimer.toLowerCase().split('.').at(0)?.slice(0, 24) ?? ''; } function keywordHits(text: string, terms: string[]): string[] { const lower = text.toLowerCase(); return terms.filter((term) => term && lower.includes(term.toLowerCase())); } function verticalRiskTerms(vertical: string): string[] { if (vertical === 'law') return ['conflict of interest', 'privilege waived', 'statute of limitations']; if (vertical === 'accounting') return ['material misstatement', 'gaap departure', 'tax penalty']; if (vertical === 'insurance') return ['exclusion', 'endorsement', 'coverage gap']; if (vertical === 'realestate' || vertical === 'real-estate') return ['fair housing', 'respa', 'disclosure defect']; if (vertical === 'ecommerce') return ['chargeback', 'ftc', 'platform violation']; return []; } function bounded(value: number): number { if (!Number.isFinite(value)) return 0; return Math.max(0, Math.min(1, value)); }