{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/core/evaluation/runner.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AAUvE,OAAO,EAAE,KAAK,2BAA2B,EAA4B,MAAM,eAAe,CAAC;AAC3F,OAAO,EAEN,KAAK,wBAAwB,EAC7B,KAAK,yBAAyB,EAG9B,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAEN,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,wBAAwB,EAC7B,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,yBAAyB;IACzC,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,yBAAyB,CAAC;IAC5C,eAAe,CAAC,EAAE,wBAAwB,EAAE,CAAC;IAC7C,KAAK,CAAC,EAAE;QACP,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,uBAAuB,CAAC,EAAE,MAAM,CAAC;KACjC,CAAC;CACF;AAED,MAAM,WAAW,kBAAkB;IAClC,OAAO,CAAC,KAAK,EAAE;QACd,QAAQ,EAAE,kBAAkB,CAAC;QAC7B,IAAI,EAAE,cAAc,CAAC;QACrB,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,oBAAoB;IACpC,IAAI,EAAE,cAAc,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,CAAC;IAChD,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,YAAY,CAAC,EAAE;QAAE,OAAO,EAAE,yBAAyB,CAAC;QAAC,MAAM,EAAE,wBAAwB,CAAA;KAAE,CAAC;IACxF,QAAQ,CAAC,EAAE,2BAA2B,CAAC;CACvC;AAED,wBAAsB,aAAa,CAClC,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,EAAE,oBAAoB,GAC3B,OAAO,CAAC,kBAAkB,CAAC,CAyL7B","sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport type { BenchmarkEvaluationResult } from \"../benchmark/types.js\";\nimport { createArtifact } from \"./artifacts.js\";\nimport { evaluateAssertions } from \"./assertions.js\";\nimport { cleanupFixture, materializeFixture } from \"./fixtures.js\";\nimport {\n\tcreateCandidateIdentity,\n\tcreateEnvironmentIdentity,\n\thasExplicitLiveOptIn,\n\tscenarioContentHash,\n} from \"./identity.js\";\nimport { type EvaluationCandidateLauncher, resolveCandidateLauncher } from \"./launcher.js\";\nimport {\n\tcreateLiveProviderExecutor,\n\ttype EvaluationProviderClient,\n\ttype EvaluationProviderProfile,\n\tpreflightLiveEvaluation,\n\tresolveProviderProfile,\n} from \"./live-provider.js\";\nimport { calculateMetrics } from \"./metrics.js\";\nimport { createEvaluationSandbox } from \"./sandbox.js\";\nimport {\n\tEVALUATOR_VERSION,\n\ttype EvaluationArtifact,\n\ttype EvaluationBudget,\n\ttype EvaluationEvent,\n\ttype EvaluationMode,\n\ttype EvaluationRun,\n\ttype EvaluationScenario,\n\ttype SemanticEvaluationResult,\n} from \"./types.js\";\n\nexport interface EvaluationExecutionResult {\n\tevents: EvaluationEvent[];\n\tworkspaceRoot?: string;\n\tbenchmarkReport?: BenchmarkEvaluationResult;\n\tsemanticResults?: SemanticEvaluationResult[];\n\tusage?: {\n\t\tinputTokens?: number;\n\t\toutputTokens?: number;\n\t\tmodelCalls?: number;\n\t\testimatedCostUsd?: number;\n\t\tproviderReportedCostUsd?: number;\n\t};\n}\n\nexport interface EvaluationExecutor {\n\texecute(input: {\n\t\tscenario: EvaluationScenario;\n\t\tmode: EvaluationMode;\n\t\tworkspaceRoot?: string;\n\t}): Promise<EvaluationExecutionResult>;\n}\n\nexport interface EvaluationRunOptions {\n\tmode: EvaluationMode;\n\tcandidate?: Partial<EvaluationRun[\"candidate\"]>;\n\texecutor?: EvaluationExecutor;\n\tretainFailedWorkspace?: boolean;\n\tlive?: boolean;\n\tconfirmLive?: boolean;\n\tbudget?: EvaluationBudget;\n\tsignal?: AbortSignal;\n\tliveProvider?: { profile: EvaluationProviderProfile; client: EvaluationProviderClient };\n\tlauncher?: EvaluationCandidateLauncher;\n}\n\nexport async function runEvaluation(\n\tscenario: EvaluationScenario,\n\toptions: EvaluationRunOptions,\n): Promise<EvaluationArtifact> {\n\tif (!scenario.candidatePolicy.allowedModes.includes(options.mode))\n\t\tthrow new Error(`scenario does not allow ${options.mode} mode`);\n\tif (options.mode === \"live\" && !scenario.candidatePolicy.allowLiveProvider)\n\t\tthrow new Error(\"live provider evaluation is disabled by scenario policy\");\n\tif (\n\t\toptions.mode === \"live\" &&\n\t\t!hasExplicitLiveOptIn({\n\t\t\tmode: options.mode,\n\t\t\tlive: options.live,\n\t\t\tconfirmed: options.confirmLive,\n\t\t\tbudget: options.budget,\n\t\t})\n\t)\n\t\tthrow new Error(\n\t\t\t\"live evaluation requires JENSEN_EVAL_LIVE=1 or --confirm-live plus positive cost, model-call, and wall-time budgets\",\n\t\t);\n\tconst startedAt = new Date().toISOString();\n\tconst candidate = createCandidateIdentity(options.candidate);\n\tconst run: EvaluationRun = {\n\t\tevaluationRunId: randomUUID(),\n\t\tscenarioId: scenario.scenarioId,\n\t\tscenarioVersion: scenario.scenarioVersion,\n\t\tscenarioContentHash: scenarioContentHash(scenario),\n\t\tmode: options.mode,\n\t\tcandidate,\n\t\tenvironmentIdentity: createEnvironmentIdentity(\"pending\", { gitCommit: candidate.gitCommit }),\n\t\tstartedAt,\n\t\tstatus: \"running\",\n\t};\n\tlet fixture: Awaited<ReturnType<typeof materializeFixture>> | undefined;\n\tlet sandbox: Awaited<ReturnType<typeof createEvaluationSandbox>> | undefined;\n\tlet failed = false;\n\tlet artifact: EvaluationArtifact | undefined;\n\tlet launcher = options.launcher;\n\ttry {\n\t\tif (options.mode === \"sandbox\" || options.mode === \"live\") {\n\t\t\tif (!launcher) launcher = await resolveCandidateLauncher();\n\t\t\tsandbox = await createEvaluationSandbox({\n\t\t\t\tevaluationRunId: run.evaluationRunId,\n\t\t\t\tfixture: scenario.fixture,\n\t\t\t\tpolicy: scenario.candidatePolicy,\n\t\t\t\tretainOnFailure: options.retainFailedWorkspace,\n\t\t\t\tsignal: options.signal,\n\t\t\t\tlauncher,\n\t\t\t});\n\t\t\trun.sandboxIdentity = sandbox.identity;\n\t\t} else if (options.mode === \"fixture\") {\n\t\t\tfixture = await materializeFixture(scenario.fixture, { retainOnFailure: options.retainFailedWorkspace });\n\t\t}\n\t\tlet executor = options.executor;\n\t\tif (options.mode === \"live\") {\n\t\t\tconst profile =\n\t\t\t\toptions.liveProvider?.profile ??\n\t\t\t\tresolveProviderProfile({\n\t\t\t\t\tprofileId: candidate.providerProfile,\n\t\t\t\t\tconfiguredModel: candidate.configuredModel,\n\t\t\t\t\tresolvedModel: candidate.resolvedModel,\n\t\t\t\t\tprovider: candidate.provider,\n\t\t\t\t});\n\t\t\tpreflightLiveEvaluation({\n\t\t\t\tprofile,\n\t\t\t\tbudget: options.budget,\n\t\t\t\tconfirmed: options.confirmLive === true || process.env.JENSEN_EVAL_LIVE === \"1\",\n\t\t\t});\n\t\t\tif (!executor)\n\t\t\t\texecutor = createLiveProviderExecutor(\n\t\t\t\t\tprofile,\n\t\t\t\t\toptions.liveProvider?.client ?? {\n\t\t\t\t\t\tcomplete: async () => {\n\t\t\t\t\t\t\tthrow new Error(\"live provider client is not configured\");\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\toptions.budget ?? {},\n\t\t\t\t\toptions.signal,\n\t\t\t\t);\n\t\t}\n\t\tconst execution = executor\n\t\t\t? await executor.execute({ scenario, mode: options.mode, workspaceRoot: sandbox?.root ?? fixture?.root })\n\t\t\t: sandbox\n\t\t\t\t? await executeSandboxCandidate(sandbox, scenario, options.signal)\n\t\t\t\t: defaultExecution(scenario, options.mode);\n\t\trun.environmentIdentity = createEnvironmentIdentity(\n\t\t\tsandbox?.identity.fixtureHash ?? fixture?.fixtureHash ?? \"offline\",\n\t\t\t{\n\t\t\t\tgitCommit: candidate.gitCommit,\n\t\t\t},\n\t\t);\n\t\tconst events = [...(sandbox?.events ?? []), ...execution.events];\n\t\tconst metrics = calculateMetrics({\n\t\t\trun: { ...run, completedAt: new Date().toISOString() },\n\t\t\tevents,\n\t\t\tspecs: scenario.metrics,\n\t\t\tbenchmarkReport: execution.benchmarkReport,\n\t\t});\n\t\tconst assertionResults = await evaluateAssertions(scenario.assertions, {\n\t\t\tworkspaceRoot: execution.workspaceRoot ?? fixture?.root,\n\t\t\tevents: execution.events,\n\t\t\tmetrics: Object.fromEntries(metrics.map((metric) => [metric.metricId, metric.value])),\n\t\t\tevidenceIds: new Set(events.map((event) => event.eventId)),\n\t\t\ttimeoutMs: scenario.timeoutMs,\n\t\t});\n\t\tconst hasInvalid = assertionResults.some(\n\t\t\t(assertion) => assertion.status === \"invalid\" || assertion.status === \"not_evaluated\",\n\t\t);\n\t\tconst safetyFailure = assertionResults.some(\n\t\t\t(assertion) =>\n\t\t\t\t(assertion.severity === \"critical\" || assertion.severity === \"high\") && assertion.status !== \"pass\",\n\t\t);\n\t\tconst deterministicPass =\n\t\t\tassertionResults.length > 0 && assertionResults.every((assertion) => assertion.status === \"pass\");\n\t\tconst completedAt = new Date().toISOString();\n\t\tconst completedRun = { ...run, completedAt, status: \"completed\" as const };\n\t\tartifact = createArtifact({\n\t\t\trun: completedRun,\n\t\t\tscenario: {\n\t\t\t\tscenarioId: scenario.scenarioId,\n\t\t\t\tscenarioVersion: scenario.scenarioVersion,\n\t\t\t\tscenarioContentHash: run.scenarioContentHash,\n\t\t\t},\n\t\t\tcandidate,\n\t\t\tassertions: assertionResults,\n\t\t\tmetrics,\n\t\t\tsemanticResults: execution.semanticResults ?? [],\n\t\t\tevents,\n\t\t\tusage: execution.usage,\n\t\t\tverdict: hasInvalid ? \"invalid\" : safetyFailure || !deterministicPass ? \"fail\" : \"pass\",\n\t\t\tevidenceIds: assertionResults.flatMap((assertion) => assertion.evidenceIds),\n\t\t\tprovenance: {\n\t\t\t\tcreatedAt: completedAt,\n\t\t\t\tsourceRunIds: [run.evaluationRunId],\n\t\t\t\tevaluatorVersion: EVALUATOR_VERSION,\n\t\t\t},\n\t\t});\n\t} catch (error) {\n\t\tfailed = true;\n\t\tconst completedAt = new Date().toISOString();\n\t\tconst cancelled = options.signal?.aborted === true;\n\t\tartifact = createArtifact({\n\t\t\trun: { ...run, completedAt, status: cancelled ? \"cancelled\" : \"failed\" },\n\t\t\tscenario: {\n\t\t\t\tscenarioId: scenario.scenarioId,\n\t\t\t\tscenarioVersion: scenario.scenarioVersion,\n\t\t\t\tscenarioContentHash: run.scenarioContentHash,\n\t\t\t},\n\t\t\tcandidate,\n\t\t\tassertions: [\n\t\t\t\t{\n\t\t\t\t\tassertionId: \"runtime\",\n\t\t\t\t\tstatus: \"invalid\",\n\t\t\t\t\texpected: true,\n\t\t\t\t\tobserved: undefined,\n\t\t\t\t\tevidenceIds: [],\n\t\t\t\t\treasonCode: error instanceof Error ? error.message : \"execution_failed\",\n\t\t\t\t\tseverity: \"high\",\n\t\t\t\t},\n\t\t\t],\n\t\t\tmetrics: [],\n\t\t\tsemanticResults: [],\n\t\t\tverdict: cancelled ? \"cancelled\" : \"invalid\",\n\t\t\tevidenceIds: [],\n\t\t\tprovenance: {\n\t\t\t\tcreatedAt: completedAt,\n\t\t\t\tsourceRunIds: [run.evaluationRunId],\n\t\t\t\tevaluatorVersion: EVALUATOR_VERSION,\n\t\t\t},\n\t\t});\n\t} finally {\n\t\tif (sandbox) {\n\t\t\tif (failed && options.retainFailedWorkspace) await sandbox.retain().catch(() => undefined);\n\t\t\tawait sandbox.cleanup();\n\t\t} else if (fixture) await cleanupFixture(fixture);\n\t\tif (artifact && sandbox) {\n\t\t\tconst { artifactHash: _artifactHash, ...unsigned } = artifact;\n\t\t\tconst events = [...(artifact.events ?? []), ...sandbox.events].filter(\n\t\t\t\t(event, index, all) => all.findIndex((candidate) => candidate.eventId === event.eventId) === index,\n\t\t\t);\n\t\t\tartifact = createArtifact({\n\t\t\t\t...unsigned,\n\t\t\t\trun: { ...artifact.run, sandboxIdentity: sandbox.identity, eventCount: events.length },\n\t\t\t\tevents,\n\t\t\t});\n\t\t}\n\t}\n\treturn artifact!;\n}\n\nfunction defaultExecution(scenario: EvaluationScenario, mode: EvaluationMode): EvaluationExecutionResult {\n\tif (mode === \"offline\") return { events: [] };\n\tconst now = new Date().toISOString();\n\tconst events: EvaluationEvent[] = [];\n\tfor (const assertion of scenario.assertions) {\n\t\tif (\n\t\t\t(assertion.kind === \"event_present\" || assertion.kind === \"event_absent\") &&\n\t\t\tassertion.pattern &&\n\t\t\tassertion.kind === \"event_present\"\n\t\t)\n\t\t\tevents.push({ eventId: randomUUID(), type: assertion.pattern, timestamp: now, details: { status: \"pass\" } });\n\t}\n\treturn { events };\n}\n\nasync function executeSandboxCandidate(\n\tsandbox: Awaited<ReturnType<typeof createEvaluationSandbox>>,\n\tscenario: EvaluationScenario,\n\tsignal?: AbortSignal,\n): Promise<EvaluationExecutionResult> {\n\tconst launcher = sandbox.policy.authorizedLauncher;\n\tif (!launcher) throw new Error(\"sandbox candidate requires an authorized launcher\");\n\tconst args = [...launcher.invocationPrefix];\n\tconst processResult = await sandbox.runProcess(launcher.executablePath, args, { signal });\n\tconst fixtureEvents = defaultExecution(scenario, \"sandbox\").events;\n\treturn {\n\t\tevents: [\n\t\t\t...fixtureEvents,\n\t\t\t{\n\t\t\t\teventId: randomUUID(),\n\t\t\t\ttype: \"candidate.process\",\n\t\t\t\ttimestamp: new Date().toISOString(),\n\t\t\t\tdetails: {\n\t\t\t\t\texitCode: processResult.exitCode ?? -1,\n\t\t\t\t\toutputBytes: Buffer.byteLength(processResult.stdout),\n\t\t\t\t\tlauncher: launcher.executableIdentity,\n\t\t\t\t},\n\t\t\t},\n\t\t],\n\t\tworkspaceRoot: sandbox.root,\n\t};\n}\n"]}