/** * @wundr/governance - Alignment Monitoring Evaluator Agent * * Implements policy compliance, reward alignment, and drift detection * for AI alignment monitoring and governance. */ /** * Types of evaluator agents available for alignment monitoring */ export type EvaluatorType = 'policy_compliance' | 'reward_alignment' | 'drift_detection'; /** * Evaluation frequency options */ export type EvaluationFrequency = 'per_commit' | 'hourly' | 'daily'; /** * Actions to take when violations are detected */ export type ViolationAction = 'block_on_violation' | 'escalate_to_guardian' | 'alert_architect'; /** * Configuration for creating an evaluator agent */ export interface EvaluatorConfig { /** Type of evaluator to create */ readonly evaluatorType: EvaluatorType; /** How often evaluations should run */ readonly frequency: EvaluationFrequency; /** Threshold score below which violations are triggered (0-1) */ readonly threshold: number; /** Action to take when violations occur */ readonly action: ViolationAction; /** Optional name for the evaluator instance */ readonly name?: string; /** Optional additional configuration */ readonly options?: EvaluatorOptions; } /** * Additional configuration options for evaluators */ export interface EvaluatorOptions { /** Enable verbose logging */ readonly verbose?: boolean; /** Custom policy rules to evaluate */ readonly customPolicies?: readonly PolicyRule[]; /** Patterns to monitor for drift detection */ readonly driftPatterns?: readonly string[]; /** Timeout for evaluation in milliseconds */ readonly timeoutMs?: number; /** Maximum number of issues to report */ readonly maxIssues?: number; } /** * A policy rule for compliance checking */ export interface PolicyRule { /** Unique identifier for the rule */ readonly id: string; /** Human-readable name */ readonly name: string; /** Description of what the rule checks */ readonly description: string; /** Severity level of violations */ readonly severity: 'low' | 'medium' | 'high' | 'critical'; /** Whether the rule is currently active */ readonly enabled: boolean; /** Category of the rule */ readonly category: string; } /** * Context provided to evaluations */ export interface EvaluationContext { /** Unique identifier for this evaluation */ readonly evaluationId: string; /** Timestamp when evaluation started */ readonly timestamp: Date; /** Source of the evaluation trigger */ readonly source: 'commit' | 'scheduled' | 'manual' | 'webhook'; /** Repository or project being evaluated */ readonly repository?: string; /** Branch being evaluated */ readonly branch?: string; /** Commit SHA if applicable */ readonly commitSha?: string; /** Files changed (for commit-based evaluations) */ readonly changedFiles?: readonly string[]; /** Additional metadata */ readonly metadata?: Record; } /** * Result of an evaluation */ export interface EvaluationResult { /** Whether the evaluation passed all checks */ readonly passed: boolean; /** Overall score (0-1, where 1 is perfect) */ readonly score: number; /** List of issues found */ readonly issues: readonly string[]; /** Recommendations for improvement */ readonly recommendations: readonly string[]; /** Action to take based on the result, null if none required */ readonly action: string | null; /** Timestamp of the evaluation */ readonly timestamp: Date; /** Type of evaluator that produced this result */ readonly evaluatorType: EvaluatorType; /** Duration of evaluation in milliseconds */ readonly durationMs: number; } /** * Reward score for alignment checking */ export interface RewardScore { /** Overall alignment score (0-1) */ readonly alignmentScore: number; /** Score for helpfulness objective */ readonly helpfulnessScore: number; /** Score for harmlessness objective */ readonly harmlessnessScore: number; /** Score for honesty objective */ readonly honestyScore: number; /** Additional dimension scores */ readonly dimensionScores?: Record; /** Explanation of the scores */ readonly explanation?: string; } /** * Result of policy compliance check */ export interface ComplianceResult { /** Whether all policies are compliant */ readonly compliant: boolean; /** Overall compliance score (0-1) */ readonly score: number; /** List of policy violations */ readonly violations: readonly PolicyViolation[]; /** Policies that passed */ readonly passedPolicies: readonly string[]; /** Policies that were skipped */ readonly skippedPolicies: readonly string[]; /** Timestamp of the check */ readonly timestamp: Date; } /** * A policy violation found during compliance checking */ export interface PolicyViolation { /** ID of the violated policy */ readonly policyId: string; /** Name of the violated policy */ readonly policyName: string; /** Severity of the violation */ readonly severity: 'low' | 'medium' | 'high' | 'critical'; /** Description of the violation */ readonly description: string; /** Location of the violation (file, line, etc.) */ readonly location?: string; /** Suggested fix for the violation */ readonly suggestedFix?: string; } /** * Result of reward alignment check */ export interface AlignmentResult { /** Whether alignment is within acceptable bounds */ readonly aligned: boolean; /** Overall alignment score (0-1) */ readonly score: number; /** Alignment gaps identified */ readonly gaps: readonly AlignmentGap[]; /** Reward decomposition */ readonly rewardBreakdown: RewardBreakdown; /** Recommendations for improving alignment */ readonly recommendations: readonly string[]; /** Timestamp of the check */ readonly timestamp: Date; } /** * An alignment gap identified during checking */ export interface AlignmentGap { /** Dimension where gap was found */ readonly dimension: string; /** Expected score */ readonly expected: number; /** Actual score */ readonly actual: number; /** Gap magnitude (expected - actual) */ readonly gap: number; /** Priority for addressing this gap */ readonly priority: 'low' | 'medium' | 'high'; } /** * Breakdown of reward components */ export interface RewardBreakdown { /** Base reward score */ readonly baseReward: number; /** Penalty for violations */ readonly violationPenalty: number; /** Bonus for exceeding expectations */ readonly alignmentBonus: number; /** Final computed reward */ readonly finalReward: number; } /** * Result of drift detection */ export interface DriftResult { /** Whether drift was detected */ readonly driftDetected: boolean; /** Overall drift score (0 = no drift, 1 = maximum drift) */ readonly driftScore: number; /** Individual drift indicators */ readonly driftIndicators: readonly DriftIndicator[]; /** Historical comparison data */ readonly historicalComparison?: HistoricalComparison; /** Recommendations for addressing drift */ readonly recommendations: readonly string[]; /** Timestamp of the check */ readonly timestamp: Date; } /** * A drift indicator showing change in a specific pattern */ export interface DriftIndicator { /** Pattern or metric that drifted */ readonly pattern: string; /** Baseline value */ readonly baseline: number; /** Current value */ readonly current: number; /** Change magnitude */ readonly change: number; /** Change as percentage */ readonly changePercent: number; /** Direction of change */ readonly direction: 'increase' | 'decrease' | 'stable'; /** Severity of the drift */ readonly severity: 'low' | 'medium' | 'high' | 'critical'; } /** * Historical comparison data for drift analysis */ export interface HistoricalComparison { /** Period start for comparison */ readonly periodStart: Date; /** Period end for comparison */ readonly periodEnd: Date; /** Number of samples in the period */ readonly sampleCount: number; /** Average score during the period */ readonly averageScore: number; /** Standard deviation during the period */ readonly standardDeviation: number; } /** * EvaluatorAgent - Alignment monitoring evaluator for AI governance * * Provides policy compliance checking, reward alignment verification, * and drift detection for maintaining AI system alignment. * * @example * ```typescript * const evaluator = new EvaluatorAgent({ * evaluatorType: 'policy_compliance', * frequency: 'per_commit', * threshold: 0.8, * action: 'block_on_violation', * }); * * const result = await evaluator.evaluate(context); * if (evaluator.shouldTriggerAction(result)) { * console.log('Action required:', evaluator.getRecommendedAction(result)); * } * ``` */ export declare class EvaluatorAgent { private readonly evaluatorType; private readonly frequency; private readonly threshold; private readonly action; private readonly name; private readonly options; /** Default policies for compliance checking */ private static readonly DEFAULT_POLICIES; /** Default patterns for drift detection */ private static readonly DEFAULT_DRIFT_PATTERNS; /** * Creates a new EvaluatorAgent instance * * @param config - Configuration for the evaluator * @throws Error if threshold is not between 0 and 1 */ constructor(config: EvaluatorConfig); /** * Performs an evaluation based on the evaluator type * * @param context - The evaluation context * @returns Promise resolving to the evaluation result */ evaluate(context: EvaluationContext): Promise; /** * Checks policy compliance for the given context * * @param context - The evaluation context * @returns Promise resolving to the compliance result */ checkPolicyCompliance(context: EvaluationContext): Promise; /** * Checks reward alignment against expected objectives * * @param context - The evaluation context * @param rewardScore - The reward score to check * @returns Promise resolving to the alignment result */ checkRewardAlignment(context: EvaluationContext, rewardScore: RewardScore): Promise; /** * Detects drift in monitored patterns * * @param context - The evaluation context * @param patterns - Patterns to monitor for drift * @returns Promise resolving to the drift result */ detectDrift(context: EvaluationContext, patterns: string[]): Promise; /** * Determines if an action should be triggered based on evaluation result * * @param result - The evaluation result * @returns True if an action should be triggered */ shouldTriggerAction(result: EvaluationResult): boolean; /** * Gets the recommended action string based on the evaluation result * * @param result - The evaluation result * @returns The recommended action string */ getRecommendedAction(result: EvaluationResult): string; /** * Evaluates a single policy against the context */ private evaluatePolicy; /** * Simulates a policy check - returns true if violation detected * In production, this would be replaced with actual policy evaluation logic */ private simulatePolicyCheck; /** * Evaluates a drift pattern against baselines */ private evaluateDriftPattern; /** * Calculates drift severity based on percentage change */ private calculateDriftSeverity; /** * Calculates overall drift score from indicators */ private calculateDriftScore; /** * Generates recommendations from compliance result */ private generateComplianceRecommendations; /** * Maps violation action to human-readable string */ private mapActionToString; /** * Calculates severity level from evaluation result */ private calculateSeverity; /** * Creates a simple hash from context for deterministic simulation */ private hashContext; /** Gets the evaluator type */ getEvaluatorType(): EvaluatorType; /** Gets the evaluation frequency */ getFrequency(): EvaluationFrequency; /** Gets the threshold */ getThreshold(): number; /** Gets the configured action */ getAction(): ViolationAction; /** Gets the evaluator name */ getName(): string; } /** * Factory function to create an EvaluatorAgent with convenient defaults * * @param type - Type of evaluator to create * @param options - Optional configuration overrides * @returns A new EvaluatorAgent instance * * @example * ```typescript * // Create a policy compliance evaluator * const policyEvaluator = createEvaluator('policy_compliance'); * * // Create a drift detector with custom threshold * const driftDetector = createEvaluator('drift_detection', { * threshold: 0.9, * frequency: 'hourly', * }); * ``` */ export declare function createEvaluator(type: EvaluatorType, options?: Partial>): EvaluatorAgent; /** * Creates a set of evaluators for comprehensive alignment monitoring * * @returns An object containing all three evaluator types */ export declare function createEvaluatorSuite(): { policyCompliance: EvaluatorAgent; rewardAlignment: EvaluatorAgent; driftDetection: EvaluatorAgent; }; /** * Runs all evaluators and aggregates results * * @param evaluators - Array of evaluators to run * @param context - The evaluation context * @returns Promise resolving to aggregated results */ export declare function runEvaluatorSuite(evaluators: readonly EvaluatorAgent[], context: EvaluationContext): Promise<{ passed: boolean; overallScore: number; results: readonly EvaluationResult[]; criticalIssues: readonly string[]; }>; //# sourceMappingURL=evaluator-agent.d.ts.map