/** * ThinkHive SDK v3.0 - Calibration API * * Prediction accuracy tracking with Brier scores and calibration metrics */ import type { CalibrationStatus, CalibrationBucket, PredictionType } from '../core/types'; /** * Calibration metrics */ export interface CalibrationMetrics { agentId: string; predictionType: PredictionType; /** Brier score (lower is better, <0.1 is good) */ brierScore: number; /** Expected Calibration Error */ ece: number; /** Maximum Calibration Error */ mce: number; /** Sample count */ sampleCount: number; /** Is the model well-calibrated */ isCalibrated: boolean; /** Reliability diagram data */ reliabilityDiagram: CalibrationBucket[]; /** Last updated */ lastUpdated: string; } /** * Calibration API client for prediction accuracy tracking */ export declare const calibration: { /** * Get calibration status for an agent * * @example * ```typescript * const status = await calibration.status('agent_123', 'churn_risk'); * console.log(`Brier score: ${status.brierScore}`); * console.log(`Is calibrated: ${status.isCalibrated}`); * ``` */ status(agentId: string, predictionType: PredictionType): Promise; /** * Get all calibration metrics for an agent * * @example * ```typescript * const metrics = await calibration.allMetrics('agent_123'); * for (const m of metrics) { * console.log(`${m.predictionType}: Brier=${m.brierScore}`); * } * ``` */ allMetrics(agentId: string): Promise; /** * Trigger recalibration for an agent * * @example * ```typescript * const result = await calibration.retrain('agent_123', { * predictionTypes: ['churn_risk', 'escalation_risk'], * }); * console.log(`Retrained: ${result.success}`); * ``` */ retrain(agentId: string, options?: { predictionTypes?: PredictionType[]; minSamples?: number; }): Promise<{ success: boolean; retrainedTypes: PredictionType[]; skippedTypes: Array<{ type: PredictionType; reason: string; }>; newMetrics: CalibrationMetrics[]; }>; }; /** * Calculate Brier score from predictions and outcomes * Lower is better, <0.1 is considered good * * Supports two calling conventions: * - calculateBrierScore(0.8, true) — single prediction (returns squared error) * - calculateBrierScore([{predicted: 0.8, actual: 1}, ...]) — batch (returns mean squared error) */ export declare function calculateBrierScore(predictionsOrProbability: Array<{ predicted: number; actual: number; }> | number, outcome?: boolean | number): number; /** * Calculate Expected Calibration Error (ECE) * Measures how well-calibrated predictions are across confidence buckets * * Supports two calling conventions: * - calculateECE([{predicted, actual}, ...]) — raw predictions, auto-bucketed * - calculateECE([{averageConfidence, accuracy, count}, ...]) — pre-computed buckets */ export declare function calculateECE(predictionsOrBuckets: Array<{ predicted: number; actual: number; }> | Array<{ averageConfidence: number; accuracy: number; count: number; }>, numBuckets?: number): number; /** * Check if a model is well-calibrated based on Brier score * Accepts a number or an object with brierScore property */ export declare function isWellCalibrated(brierScoreOrStatus: number | { brierScore: number; }): boolean; /** * Get calibration quality label */ export declare function getCalibrationQuality(brierScore: number): 'excellent' | 'good' | 'fair' | 'poor'; /** * Format Brier score for display */ export declare function formatBrierScore(score: number): string;