/** * Tool Selection Metrics (TC-010) * * Tracks tool selection patterns to measure the effectiveness of the * 5-tool interface consolidation. Helps identify: * - Which tools are most used * - Whether deprecated tools are still being called (confusion indicator) * - First-browse success rate * - Tool sequence patterns (repeated attempts) */ /** * Core tools that should be visible by default */ export declare const CORE_TOOLS: readonly ["smart_browse", "batch_browse", "execute_api_call", "session_management", "api_auth"]; /** * Debug tools (visible with DEBUG_MODE) */ export declare const DEBUG_TOOLS: readonly ["capture_screenshot", "export_har", "debug_traces"]; /** * Admin tools (visible with ADMIN_MODE) */ export declare const ADMIN_TOOLS: readonly ["get_performance_metrics", "usage_analytics", "get_analytics_dashboard", "get_system_status", "get_browser_providers", "tier_management", "content_tracking"]; /** * Deprecated tools that indicate confusion if called */ export declare const DEPRECATED_TOOLS: readonly ["get_domain_intelligence", "get_domain_capabilities", "get_learning_stats", "get_learning_effectiveness", "skill_management", "get_api_auth_status", "configure_api_auth", "complete_oauth", "get_auth_guidance", "delete_api_auth", "list_configured_auth", "browse"]; export type CoreTool = typeof CORE_TOOLS[number]; export type DebugTool = typeof DEBUG_TOOLS[number]; export type AdminTool = typeof ADMIN_TOOLS[number]; export type DeprecatedTool = typeof DEPRECATED_TOOLS[number]; export type AllTools = CoreTool | DebugTool | AdminTool | DeprecatedTool; /** * A single tool invocation event */ export interface ToolInvocationEvent { /** Unique event ID */ id: string; /** Timestamp of invocation */ timestamp: number; /** Tool name */ tool: string; /** Whether the invocation succeeded */ success: boolean; /** Error message if failed */ error?: string; /** Duration in milliseconds */ durationMs: number; /** Session ID to track sequences within a session */ sessionId?: string; /** Optional tenant ID for multi-tenant deployments */ tenantId?: string; /** Whether this was a deprecated tool */ isDeprecated: boolean; /** Tool category */ category: 'core' | 'debug' | 'admin' | 'deprecated' | 'unknown'; } /** * Aggregated tool usage statistics */ export interface ToolUsageStats { /** Total invocations */ totalInvocations: number; /** Breakdown by tool */ byTool: Record; /** Breakdown by category */ byCategory: CategoryStats; /** First-browse success rate (smart_browse success on first try per session) */ firstBrowseSuccessRate: number; /** Deprecated tool usage rate (confusion indicator) */ deprecatedUsageRate: number; /** Session stats */ sessionStats: { totalSessions: number; avgToolsPerSession: number; sessionsWithDeprecatedUsage: number; }; /** Time range of data */ periodStart: number; periodEnd: number; } export interface ToolStats { /** Number of invocations */ invocations: number; /** Success count */ successCount: number; /** Failure count */ failureCount: number; /** Success rate (0-1) */ successRate: number; /** Average duration in ms */ avgDurationMs: number; /** Last used timestamp */ lastUsed: number; } export interface CategoryStats { core: { invocations: number; successRate: number; }; debug: { invocations: number; successRate: number; }; admin: { invocations: number; successRate: number; }; deprecated: { invocations: number; successRate: number; }; unknown: { invocations: number; successRate: number; }; } /** * Query options for tool selection metrics */ export interface ToolMetricsQueryOptions { /** Filter by tool name */ tool?: string; /** Filter by category */ category?: 'core' | 'debug' | 'admin' | 'deprecated' | 'unknown'; /** Filter by session */ sessionId?: string; /** Filter by tenant */ tenantId?: string; /** Time period */ period?: 'hour' | 'day' | 'week' | 'month' | 'all'; /** Custom start time */ startTime?: number; /** Custom end time */ endTime?: number; } /** * Categorize a tool name */ export declare function categorize(tool: string): 'core' | 'debug' | 'admin' | 'deprecated' | 'unknown'; /** * Check if a tool is deprecated */ export declare function isDeprecated(tool: string): boolean; export declare class ToolSelectionMetrics { private events; private persistentStore; private maxEvents; private initialized; constructor(options?: { persistPath?: string; maxEvents?: number; debounceMs?: number; }); /** * Initialize the metrics tracker (load persisted events) */ initialize(): Promise; /** * Record a tool invocation */ record(event: Omit): Promise; /** * Generate a unique event ID */ private generateId; /** * Get period boundaries based on time period */ private getPeriodBoundaries; /** * Filter events based on query options */ private filterEvents; /** * Get tool usage statistics */ getStats(options?: ToolMetricsQueryOptions): Promise; /** * Return empty stats */ private emptyStats; /** * Get confusion indicators * Returns metrics that suggest LLM confusion with tools */ getConfusionIndicators(options?: ToolMetricsQueryOptions): Promise<{ deprecatedToolCalls: { tool: string; count: number; suggestion: string; }[]; repeatedFailures: { tool: string; failureCount: number; lastError?: string; }[]; toolHopping: { sessionsWithMultipleTools: number; avgToolSwitches: number; }; overallConfusionScore: number; recommendations: string[]; }>; /** * Get suggestion for deprecated tool */ private getSuggestionForDeprecatedTool; /** * Get event count */ getEventCount(): number; /** * Reset metrics data */ reset(): Promise; /** * Flush pending data to storage */ flush(): Promise; } /** * Get or create the global tool selection metrics instance */ export declare function getToolSelectionMetrics(): ToolSelectionMetrics; /** * Reset the global metrics instance (for testing) */ export declare function resetToolSelectionMetricsInstance(): void; //# sourceMappingURL=tool-selection-metrics.d.ts.map