{"version":3,"file":"live-provider.d.ts","sourceRoot":"","sources":["../../../src/core/evaluation/live-provider.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAA6B,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,KAAK,EAAE,gBAAgB,EAAmB,MAAM,YAAY,CAAC;AAEpE,MAAM,WAAW,yBAAyB;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,uBAAuB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,uBAAuB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,0BAA0B;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,uBAAuB,CAAC;CAC/B;AAED,MAAM,WAAW,wBAAwB;IACxC,QAAQ,CAAC,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAAC;CAC9G;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,EACV,kBAAkB,GAClB,aAAa,GACb,cAAc,GACd,iBAAiB,GACjB,iBAAiB,GACjB,WAAW,CAAC;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,YAAY,IAAI,EAAE,uBAAuB,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAKnF;CACD;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,yBAAyB,CAsB5B;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC9C,OAAO,EAAE,yBAAyB,CAAC;IACnC,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,SAAS,EAAE,OAAO,CAAC;CACnB,GAAG,IAAI,CA6BP;AAOD,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,yBAAyB,GAAG,wBAAwB,CA+C3G;AAED,wBAAgB,2BAA2B,IAAI,wBAAwB,CAUtE;AAED,wBAAgB,0BAA0B,CACzC,OAAO,EAAE,yBAAyB,EAClC,MAAM,EAAE,wBAAwB,EAChC,MAAM,EAAE,gBAAgB,EACxB,MAAM,CAAC,EAAE,WAAW,GAClB,kBAAkB,CA8CpB","sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport type { EvaluationExecutionResult, EvaluationExecutor } from \"./runner.js\";\nimport type { EvaluationBudget, EvaluationEvent } from \"./types.js\";\n\nexport interface EvaluationProviderProfile {\n\tprofileId: string;\n\tprovider: string;\n\tconfiguredModel: string;\n\tresolvedModel: string;\n\tbaseUrl: string;\n\tapiKeyEnv: string;\n\tapiKey?: string;\n}\n\nexport interface EvaluationProviderUsage {\n\tinputTokens: number;\n\toutputTokens: number;\n\tmodelCalls: number;\n\testimatedCostUsd?: number;\n\tproviderReportedCostUsd?: number;\n}\n\nexport interface EvaluationProviderResponse {\n\ttext: string;\n\tusage: EvaluationProviderUsage;\n}\n\nexport interface EvaluationProviderClient {\n\tcomplete(input: { prompt: string; model: string; signal?: AbortSignal }): Promise<EvaluationProviderResponse>;\n}\n\nexport class EvaluationProviderError extends Error {\n\treadonly code:\n\t\t| \"PREFLIGHT_FAILED\"\n\t\t| \"AUTH_FAILED\"\n\t\t| \"RATE_LIMITED\"\n\t\t| \"BUDGET_EXCEEDED\"\n\t\t| \"PROVIDER_FAILED\"\n\t\t| \"CANCELLED\";\n\treadonly provider: string;\n\tconstructor(code: EvaluationProviderError[\"code\"], provider: string, message: string) {\n\t\tsuper(message);\n\t\tthis.name = \"EvaluationProviderError\";\n\t\tthis.code = code;\n\t\tthis.provider = provider;\n\t}\n}\n\nexport function resolveProviderProfile(input: {\n\tprofileId: string;\n\tconfiguredModel: string;\n\tresolvedModel?: string;\n\tprovider?: string;\n}): EvaluationProviderProfile {\n\tconst profileId = input.profileId.trim();\n\tif (!profileId) throw new EvaluationProviderError(\"PREFLIGHT_FAILED\", \"unknown\", \"provider profile is required\");\n\tconst provider = input.provider ?? (profileId === \"openrouter\" ? \"openrouter\" : profileId);\n\tconst defaults: Record<string, { baseUrl: string; apiKeyEnv: string }> = {\n\t\topenrouter: { baseUrl: \"https://openrouter.ai/api/v1\", apiKeyEnv: \"OPENROUTER_API_KEY\" },\n\t\topenai: { baseUrl: \"https://api.openai.com/v1\", apiKeyEnv: \"OPENAI_API_KEY\" },\n\t\tanthropic: { baseUrl: \"https://api.anthropic.com/v1\", apiKeyEnv: \"ANTHROPIC_API_KEY\" },\n\t};\n\tconst defaultProfile = defaults[provider];\n\tif (!defaultProfile)\n\t\tthrow new EvaluationProviderError(\"PREFLIGHT_FAILED\", provider, `unsupported provider profile: ${profileId}`);\n\tconst apiKey = process.env[defaultProfile.apiKeyEnv];\n\treturn {\n\t\tprofileId,\n\t\tprovider,\n\t\tconfiguredModel: input.configuredModel,\n\t\tresolvedModel: input.resolvedModel ?? input.configuredModel,\n\t\tbaseUrl: process.env.JENSEN_EVAL_PROVIDER_BASE_URL ?? defaultProfile.baseUrl,\n\t\tapiKeyEnv: defaultProfile.apiKeyEnv,\n\t\tapiKey,\n\t};\n}\n\nexport function preflightLiveEvaluation(input: {\n\tprofile: EvaluationProviderProfile;\n\tbudget?: EvaluationBudget;\n\tconfirmed: boolean;\n}): void {\n\tif (!input.confirmed)\n\t\tthrow new EvaluationProviderError(\n\t\t\t\"PREFLIGHT_FAILED\",\n\t\t\tinput.profile.provider,\n\t\t\t\"live evaluation requires explicit confirmation\",\n\t\t);\n\tif (!input.profile.apiKey)\n\t\tthrow new EvaluationProviderError(\n\t\t\t\"PREFLIGHT_FAILED\",\n\t\t\tinput.profile.provider,\n\t\t\t`credential ${input.profile.apiKeyEnv} is unavailable`,\n\t\t);\n\tif (!input.profile.resolvedModel)\n\t\tthrow new EvaluationProviderError(\n\t\t\t\"PREFLIGHT_FAILED\",\n\t\t\tinput.profile.provider,\n\t\t\t\"resolved model identity is required\",\n\t\t);\n\tif (!input.budget?.maximumCostUsd || input.budget.maximumCostUsd <= 0)\n\t\tthrow new EvaluationProviderError(\n\t\t\t\"PREFLIGHT_FAILED\",\n\t\t\tinput.profile.provider,\n\t\t\t\"a positive maximum cost is required\",\n\t\t);\n\tif (!input.budget.maximumModelCalls || input.budget.maximumModelCalls < 1)\n\t\tthrow new EvaluationProviderError(\"PREFLIGHT_FAILED\", input.profile.provider, \"maximum model calls is required\");\n\tif (!input.budget.maximumWallTimeMs || input.budget.maximumWallTimeMs < 1)\n\t\tthrow new EvaluationProviderError(\"PREFLIGHT_FAILED\", input.profile.provider, \"maximum wall time is required\");\n}\n\nfunction estimateCost(provider: string, usage: { inputTokens: number; outputTokens: number }): number {\n\tconst rate = provider === \"openrouter\" ? 0.00001 : 0.00002;\n\treturn (usage.inputTokens + usage.outputTokens) * rate;\n}\n\nexport function createOpenAiCompatibleProvider(profile: EvaluationProviderProfile): EvaluationProviderClient {\n\treturn {\n\t\tasync complete(input) {\n\t\t\tif (!profile.apiKey)\n\t\t\t\tthrow new EvaluationProviderError(\"AUTH_FAILED\", profile.provider, \"provider credential is unavailable\");\n\t\t\tconst response = await fetch(`${profile.baseUrl.replace(/\\/$/, \"\")}/chat/completions`, {\n\t\t\t\tmethod: \"POST\",\n\t\t\t\theaders: {\n\t\t\t\t\tAuthorization: `Bearer ${profile.apiKey}`,\n\t\t\t\t\t\"Content-Type\": \"application/json\",\n\t\t\t\t\t...(profile.provider === \"openrouter\"\n\t\t\t\t\t\t? { \"HTTP-Referer\": \"https://github.com/apholdings/jensen-code\" }\n\t\t\t\t\t\t: {}),\n\t\t\t\t},\n\t\t\t\tbody: JSON.stringify({ model: input.model, messages: [{ role: \"user\", content: input.prompt }] }),\n\t\t\t\tsignal: input.signal,\n\t\t\t});\n\t\t\tif (!response.ok) {\n\t\t\t\tconst code =\n\t\t\t\t\tresponse.status === 401 || response.status === 403\n\t\t\t\t\t\t? \"AUTH_FAILED\"\n\t\t\t\t\t\t: response.status === 429\n\t\t\t\t\t\t\t? \"RATE_LIMITED\"\n\t\t\t\t\t\t\t: \"PROVIDER_FAILED\";\n\t\t\t\tthrow new EvaluationProviderError(\n\t\t\t\t\tcode,\n\t\t\t\t\tprofile.provider,\n\t\t\t\t\t`provider request failed with HTTP ${response.status}`,\n\t\t\t\t);\n\t\t\t}\n\t\t\tconst payload = (await response.json()) as {\n\t\t\t\tchoices?: Array<{ message?: { content?: string } }>;\n\t\t\t\tusage?: { prompt_tokens?: number; completion_tokens?: number };\n\t\t\t};\n\t\t\tconst inputTokens = payload.usage?.prompt_tokens ?? 0;\n\t\t\tconst outputTokens = payload.usage?.completion_tokens ?? 0;\n\t\t\treturn {\n\t\t\t\ttext: payload.choices?.[0]?.message?.content ?? \"\",\n\t\t\t\tusage: {\n\t\t\t\t\tinputTokens,\n\t\t\t\t\toutputTokens,\n\t\t\t\t\tmodelCalls: 1,\n\t\t\t\t\testimatedCostUsd: estimateCost(profile.provider, { inputTokens, outputTokens }),\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t};\n}\n\nexport function createDeterministicProvider(): EvaluationProviderClient {\n\treturn {\n\t\tasync complete(input) {\n\t\t\tconst inputTokens = input.prompt.length;\n\t\t\treturn {\n\t\t\t\ttext: `deterministic evaluation response for ${input.model}`,\n\t\t\t\tusage: { inputTokens, outputTokens: 8, modelCalls: 1, estimatedCostUsd: 0 },\n\t\t\t};\n\t\t},\n\t};\n}\n\nexport function createLiveProviderExecutor(\n\tprofile: EvaluationProviderProfile,\n\tclient: EvaluationProviderClient,\n\tbudget: EvaluationBudget,\n\tsignal?: AbortSignal,\n): EvaluationExecutor {\n\treturn {\n\t\tasync execute(input): Promise<EvaluationExecutionResult> {\n\t\t\tconst started = Date.now();\n\t\t\tconst events: EvaluationEvent[] = [];\n\t\t\tlet totalCost = 0;\n\t\t\tlet modelCalls = 0;\n\t\t\tconst response = await client.complete({\n\t\t\t\tprompt: input.scenario.task.prompt,\n\t\t\t\tmodel: profile.resolvedModel,\n\t\t\t\tsignal,\n\t\t\t});\n\t\t\tmodelCalls += response.usage.modelCalls;\n\t\t\ttotalCost += response.usage.providerReportedCostUsd ?? response.usage.estimatedCostUsd ?? 0;\n\t\t\tif (budget.maximumModelCalls !== undefined && modelCalls > budget.maximumModelCalls)\n\t\t\t\tthrow new EvaluationProviderError(\"BUDGET_EXCEEDED\", profile.provider, \"maximum model calls exceeded\");\n\t\t\tif (budget.maximumCostUsd !== undefined && totalCost > budget.maximumCostUsd)\n\t\t\t\tthrow new EvaluationProviderError(\"BUDGET_EXCEEDED\", profile.provider, \"maximum evaluation cost exceeded\");\n\t\t\tif (budget.maximumWallTimeMs !== undefined && Date.now() - started > budget.maximumWallTimeMs)\n\t\t\t\tthrow new EvaluationProviderError(\n\t\t\t\t\t\"BUDGET_EXCEEDED\",\n\t\t\t\t\tprofile.provider,\n\t\t\t\t\t\"maximum evaluation wall time exceeded\",\n\t\t\t\t);\n\t\t\tevents.push({\n\t\t\t\teventId: randomUUID(),\n\t\t\t\ttype: \"provider.response\",\n\t\t\t\ttimestamp: new Date().toISOString(),\n\t\t\t\tdetails: {\n\t\t\t\t\tprovider: profile.provider,\n\t\t\t\t\tconfiguredModel: profile.configuredModel,\n\t\t\t\t\tresolvedModel: profile.resolvedModel,\n\t\t\t\t\tmodelCalls,\n\t\t\t\t\testimatedCostUsd: response.usage.estimatedCostUsd ?? 0,\n\t\t\t\t\tproviderReportedCostUsd: response.usage.providerReportedCostUsd ?? 0,\n\t\t\t\t\tresponseBytes: Buffer.byteLength(response.text),\n\t\t\t\t},\n\t\t\t});\n\t\t\treturn {\n\t\t\t\tevents,\n\t\t\t\tsemanticResults: [],\n\t\t\t\tworkspaceRoot: input.workspaceRoot,\n\t\t\t\tusage: response.usage,\n\t\t\t};\n\t\t},\n\t};\n}\n"]}