/**
 * Model Integration Benchmark
 *
 * Runs real API calls against multiple LLM providers via Factory.ai droid exec CLI
 * to compare model performance on UAM memory-enhanced tasks.
 *
 * Assumptions:
 * - FACTORY_API_KEY is set in environment for Factory.ai API access
 * - Models: Claude Opus 4.5, GLM 4.7, GPT 5.2
 * - droid CLI is installed and accessible
 * - UAM CLI is available for memory initialization
 *
 * What this handles:
 * - Full UAM setup (init, analyze, generate, memory start, prepopulate)
 * - CLAUDE.md reading and context injection
 * - Real API calls to multiple LLM providers via droid exec
 * - Task execution comparison across models with/without UAM
 * - Performance metrics collection (latency, success, tokens)
 * - Result aggregation and reporting
 *
 * What this does NOT handle:
 * - Rate limiting (caller responsibility)
 * - Cost tracking (would require billing API)
 * - Streaming responses (uses completion mode)
 */
interface ModelConfig {
    id: string;
    name: string;
    provider: string;
    apiModel: string;
}
interface BenchmarkTaskDef {
    id: string;
    name: string;
    description: string;
    prompt: string;
    difficulty: 'easy' | 'medium' | 'hard';
    category: string;
    expectedPatterns: string[];
    maxTokens: number;
}
interface TaskResult {
    taskId: string;
    modelId: string;
    success: boolean;
    latencyMs: number;
    tokensUsed: number;
    response: string;
    error?: string;
    matchedPatterns: string[];
}
interface ModelBenchmarkResult {
    modelId: string;
    modelName: string;
    tasksRun: number;
    tasksSucceeded: number;
    successRate: number;
    avgLatencyMs: number;
    totalTokens: number;
    results: TaskResult[];
}
interface BenchmarkReport {
    timestamp: string;
    models: ModelBenchmarkResult[];
    comparison: {
        bestOverall: string;
        fastestModel: string;
        mostAccurate: string;
        byDifficulty: Record<string, {
            model: string;
            successRate: number;
        }>;
    };
    memoryComparison?: {
        withMemory: ModelBenchmarkResult[];
        withoutMemory: ModelBenchmarkResult[];
        improvement: Record<string, {
            successDelta: number;
            speedupRatio: number;
        }>;
    };
}
interface UAMSetupResult {
    initialized: boolean;
    memoryStarted: boolean;
    memoryPrepopulated: boolean;
    claudeMdLoaded: boolean;
    errors: string[];
}
/**
 * Initialize UAM system for benchmark testing
 * Runs: uam init, uam analyze, uam generate, uam memory start, uam memory prepopulate
 */
declare function setupUAM(verbose?: boolean): Promise<UAMSetupResult>;
/**
 * Load UAM memory context from CLAUDE.md and short-term memory
 */
declare function loadUAMMemoryContext(): string;
declare const MODELS: ModelConfig[];
declare const BENCHMARK_TASKS: BenchmarkTaskDef[];
export interface BenchmarkOptions {
    apiKey?: string;
    modelIds?: string[];
    compareMemory?: boolean;
    parallelModels?: number;
}
export declare function runModelBenchmark(apiKey?: string, modelIds?: string[], compareMemory?: boolean, parallelModels?: number): Promise<BenchmarkReport>;
export declare function runModelBenchmark(options: BenchmarkOptions): Promise<BenchmarkReport>;
export { MODELS, BENCHMARK_TASKS, setupUAM, loadUAMMemoryContext };
export type { ModelConfig, BenchmarkTaskDef, TaskResult, ModelBenchmarkResult, BenchmarkReport, UAMSetupResult };
//# sourceMappingURL=model-integration.d.ts.map