{"version":3,"file":"rpc.d.ts","sourceRoot":"","sources":["../../../src/core/evaluation/rpc.ts"],"names":[],"mappings":"AAuBA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,KAAK,EAEX,cAAc,EAGd,kBAAkB,EAClB,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,sBAAsB,GAAa,CAAC;AACjD,eAAO,MAAM,yBAAyB,2cA0B5B,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,yBAAyB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEhF,MAAM,WAAW,oBAAoB;IACpC,OAAO,EAAE,OAAO,sBAAsB,CAAC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,sBAAsB,CAAC;IAClC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAC5F;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,EACD,iBAAiB,GACjB,uBAAuB,GACvB,WAAW,GACX,UAAU,GACV,qBAAqB,GACrB,gBAAgB,GAChB,WAAW,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACrC,OAAO,EAAE,OAAO,sBAAsB,CAAC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,sBAAsB,CAAC;IAClC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,kBAAkB,CAAC;CAC3B;AAED,MAAM,WAAW,2BAA2B;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED,MAAM,WAAW,qBAAqB;IACrC,cAAc,CAAC,KAAK,EAAE;QACrB,QAAQ,EAAE,kBAAkB,CAAC;QAC7B,IAAI,EAAE,cAAc,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;KACpB,GAAG,kBAAkB,GAAG,SAAS,CAAC;CACnC;AAED,qBAAa,oBAAoB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAGjB;IACJ,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,YAAY,OAAO,GAAE,2BAAgC,EAGpD;IAEK,MAAM,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CA8B1E;YAEa,QAAQ;CAkLtB;AAsCD,wBAAgB,4BAA4B,CAAC,OAAO,EAAE,OAAO,GAAG,kBAAkB,GAAG,SAAS,CAY7F","sourcesContent":["import { join } from \"node:path\";\nimport { compareAgents } from \"./compare-agents.js\";\nimport {\n\taggregateStability,\n\tcheckReleaseGate,\n\tclusterFailure,\n\tcompareArtifacts,\n\tcreateBaseline,\n\tdiscoverEvaluationPacks,\n\tinspectArtifactStore,\n\tlistArtifacts,\n\tlistBaselines,\n\tmergeFailureClusters,\n\tpruneEvaluationStore,\n\treadArtifact,\n\treplayArtifact,\n\trescoreArtifact,\n\trunEvaluation,\n\tverifyArtifact,\n\tverifyBaseline,\n\twriteArtifact,\n} from \"./index.js\";\nimport { createPruneManifest } from \"./retention.js\";\nimport type { EvaluationExecutor } from \"./runner.js\";\nimport type {\n\tEvaluationArtifact,\n\tEvaluationMode,\n\tEvaluationPruneManifest,\n\tEvaluationReleaseGate,\n\tEvaluationScenario,\n} from \"./types.js\";\n\nexport const EVALUATION_RPC_VERSION = 2 as const;\nexport const EVALUATION_RPC_OPERATIONS = [\n\t\"eval.packs\",\n\t\"eval.scenarios\",\n\t\"eval.inspect\",\n\t\"eval.validate\",\n\t\"eval.run\",\n\t\"eval.cancel\",\n\t\"eval.status\",\n\t\"eval.results\",\n\t\"eval.report\",\n\t\"eval.compare\",\n\t\"eval.compareAgents\",\n\t\"eval.replay\",\n\t\"eval.rescore\",\n\t\"eval.stability\",\n\t\"eval.baselines\",\n\t\"eval.baselineCreate\",\n\t\"eval.baselinePromote\",\n\t\"eval.baselineVerify\",\n\t\"eval.gateCheck\",\n\t\"eval.gateExplain\",\n\t\"eval.failures\",\n\t\"eval.failureInspect\",\n\t\"eval.prunePreview\",\n\t\"eval.pruneExecute\",\n\t\"eval.doctor\",\n] as const;\n\nexport type EvaluationRpcOperation = (typeof EVALUATION_RPC_OPERATIONS)[number];\n\nexport interface EvaluationRpcRequest {\n\tversion: typeof EVALUATION_RPC_VERSION;\n\trequestId: string;\n\toperation: EvaluationRpcOperation;\n\tparameters?: Record<string, string | number | boolean | string[] | Record<string, unknown>>;\n}\n\nexport interface EvaluationRpcError {\n\tcode:\n\t\t| \"INVALID_REQUEST\"\n\t\t| \"UNSUPPORTED_OPERATION\"\n\t\t| \"NOT_FOUND\"\n\t\t| \"CONFLICT\"\n\t\t| \"PRECONDITION_FAILED\"\n\t\t| \"INTERNAL_ERROR\"\n\t\t| \"CANCELLED\";\n\tmessage: string;\n}\n\nexport interface EvaluationRpcResponse {\n\tversion: typeof EVALUATION_RPC_VERSION;\n\trequestId: string;\n\toperation: EvaluationRpcOperation;\n\tdata?: unknown;\n\terror?: EvaluationRpcError;\n}\n\nexport interface EvaluationRpcServiceOptions {\n\troot?: string;\n\texecutor?: EvaluationRpcExecutor;\n}\n\nexport interface EvaluationRpcExecutor {\n\tcreateExecutor(input: {\n\t\tscenario: EvaluationScenario;\n\t\tmode: EvaluationMode;\n\t\tcandidateId: string;\n\t}): EvaluationExecutor | undefined;\n}\n\nexport class EvaluationRpcService {\n\treadonly root: string;\n\tprivate readonly jobs = new Map<\n\t\tstring,\n\t\t{ status: \"running\" | \"completed\" | \"failed\" | \"cancelled\"; abort: AbortController; artifactIds: string[] }\n\t>();\n\tprivate readonly executor?: EvaluationRpcExecutor;\n\n\tconstructor(options: EvaluationRpcServiceOptions = {}) {\n\t\tthis.root = options.root ?? join(process.cwd(), \".jensen\", \"evaluations\");\n\t\tthis.executor = options.executor;\n\t}\n\n\tasync handle(request: EvaluationRpcRequest): Promise<EvaluationRpcResponse> {\n\t\tconst validationError = validateEvaluationRpcRequest(request);\n\t\tif (validationError)\n\t\t\treturn {\n\t\t\t\tversion: EVALUATION_RPC_VERSION,\n\t\t\t\trequestId: request?.requestId ?? \"\",\n\t\t\t\toperation: request?.operation as EvaluationRpcOperation,\n\t\t\t\terror: validationError,\n\t\t\t};\n\t\ttry {\n\t\t\treturn {\n\t\t\t\tversion: EVALUATION_RPC_VERSION,\n\t\t\t\trequestId: request.requestId,\n\t\t\t\toperation: request.operation,\n\t\t\t\tdata: await this.dispatch(request),\n\t\t\t};\n\t\t} catch (error) {\n\t\t\tconst code =\n\t\t\t\terror instanceof Error && error.message.includes(\"precondition\")\n\t\t\t\t\t? \"PRECONDITION_FAILED\"\n\t\t\t\t\t: error instanceof Error && error.message.includes(\"not found\")\n\t\t\t\t\t\t? \"NOT_FOUND\"\n\t\t\t\t\t\t: \"INTERNAL_ERROR\";\n\t\t\treturn {\n\t\t\t\tversion: EVALUATION_RPC_VERSION,\n\t\t\t\trequestId: request.requestId,\n\t\t\t\toperation: request.operation,\n\t\t\t\terror: { code, message: error instanceof Error ? error.message : String(error) },\n\t\t\t};\n\t\t}\n\t}\n\n\tprivate async dispatch(request: EvaluationRpcRequest): Promise<unknown> {\n\t\tconst params = request.parameters ?? {};\n\t\tconst discovered = await discoverEvaluationPacks(this.root);\n\t\tswitch (request.operation) {\n\t\t\tcase \"eval.packs\":\n\t\t\t\treturn discovered.packs;\n\t\t\tcase \"eval.scenarios\":\n\t\t\t\treturn discovered.scenarios;\n\t\t\tcase \"eval.inspect\":\n\t\t\t\treturn readArtifact(this.root, String(params.artifactId ?? \"\"));\n\t\t\tcase \"eval.validate\":\n\t\t\t\treturn { valid: discovered.errors.length === 0, errors: discovered.errors };\n\t\t\tcase \"eval.run\": {\n\t\t\t\tconst scenario = selectScenario(discovered.scenarios, discovered.packs, String(params.target ?? \"\"));\n\t\t\t\tconst mode = String(params.mode ?? \"fixture\") as EvaluationMode;\n\t\t\t\tconst jobId = `eval-job-${Date.now()}-${Math.random().toString(16).slice(2)}`;\n\t\t\t\tconst abort = new AbortController();\n\t\t\t\tthis.jobs.set(jobId, { status: \"running\", abort, artifactIds: [] });\n\t\t\t\ttry {\n\t\t\t\t\tconst artifact = await runEvaluation(scenario, {\n\t\t\t\t\t\tmode,\n\t\t\t\t\t\tconfirmLive: params.confirmLive === true,\n\t\t\t\t\t\tlive: params.live === true || params.confirmLive === true,\n\t\t\t\t\t\tbudget: {\n\t\t\t\t\t\t\tmaximumCostUsd: numberParameter(params.maximumCostUsd),\n\t\t\t\t\t\t\tmaximumModelCalls: numberParameter(params.maximumModelCalls),\n\t\t\t\t\t\t\tmaximumWallTimeMs: numberParameter(params.maximumWallTimeMs),\n\t\t\t\t\t\t},\n\t\t\t\t\t\tcandidate: {\n\t\t\t\t\t\t\tproviderProfile: String(params.providerProfile ?? \"fixture\"),\n\t\t\t\t\t\t\tprovider: String(params.provider ?? \"fixture\"),\n\t\t\t\t\t\t\tconfiguredModel: String(params.model ?? \"deterministic-fixture\"),\n\t\t\t\t\t\t},\n\t\t\t\t\t\texecutor: this.executor?.createExecutor({ scenario, mode, candidateId: jobId }),\n\t\t\t\t\t\tsignal: abort.signal,\n\t\t\t\t\t});\n\t\t\t\t\tawait writeArtifact(this.root, artifact);\n\t\t\t\t\tthis.jobs.set(jobId, {\n\t\t\t\t\t\tstatus: artifact.verdict === \"cancelled\" ? \"cancelled\" : \"completed\",\n\t\t\t\t\t\tabort,\n\t\t\t\t\t\tartifactIds: [artifact.artifactHash],\n\t\t\t\t\t});\n\t\t\t\t\treturn { jobId, artifactId: artifact.artifactHash, status: artifact.run.status };\n\t\t\t\t} catch (error) {\n\t\t\t\t\tthis.jobs.set(jobId, { status: abort.signal.aborted ? \"cancelled\" : \"failed\", abort, artifactIds: [] });\n\t\t\t\t\tthrow error;\n\t\t\t\t}\n\t\t\t}\n\t\t\tcase \"eval.cancel\": {\n\t\t\t\tconst job = this.jobs.get(String(params.jobId ?? \"\"));\n\t\t\t\tif (!job) throw new Error(\"evaluation job not found\");\n\t\t\t\tjob.abort.abort();\n\t\t\t\tjob.status = \"cancelled\";\n\t\t\t\treturn { jobId: String(params.jobId), status: job.status };\n\t\t\t}\n\t\t\tcase \"eval.status\": {\n\t\t\t\tconst job = this.jobs.get(String(params.jobId ?? \"\"));\n\t\t\t\tif (!job) throw new Error(\"evaluation job not found\");\n\t\t\t\treturn { jobId: String(params.jobId), status: job.status, artifactIds: job.artifactIds };\n\t\t\t}\n\t\t\tcase \"eval.results\": {\n\t\t\t\tconst artifacts = await listArtifacts(this.root);\n\t\t\t\tconst offset = Math.max(0, numberParameter(params.offset) ?? 0);\n\t\t\t\tconst limit = Math.min(100, Math.max(1, numberParameter(params.limit) ?? 50));\n\t\t\t\treturn {\n\t\t\t\t\titems: artifacts.slice(offset, offset + limit),\n\t\t\t\t\toffset,\n\t\t\t\t\tlimit,\n\t\t\t\t\ttotal: artifacts.length,\n\t\t\t\t\thasMore: offset + limit < artifacts.length,\n\t\t\t\t};\n\t\t\t}\n\t\t\tcase \"eval.report\":\n\t\t\t\treturn { artifacts: await listArtifacts(this.root), store: await inspectArtifactStore(this.root) };\n\t\t\tcase \"eval.compare\":\n\t\t\t\treturn compareArtifacts(\n\t\t\t\t\tawait readArtifact(this.root, String(params.baselineId ?? \"\")),\n\t\t\t\t\tawait readArtifact(this.root, String(params.candidateId ?? \"\")),\n\t\t\t\t);\n\t\t\tcase \"eval.compareAgents\": {\n\t\t\t\tconst scenario = selectScenario(discovered.scenarios, discovered.packs, String(params.target ?? \"\"));\n\t\t\t\tconst mode = String(params.mode ?? \"fixture\") as \"fixture\" | \"sandbox\" | \"live\";\n\t\t\t\tconst singleAgent = this.executor?.createExecutor({ scenario, mode, candidateId: \"single-agent\" });\n\t\t\t\tconst cavecrew = this.executor?.createExecutor({ scenario, mode, candidateId: \"cavecrew\" });\n\t\t\t\tif (!singleAgent || !cavecrew)\n\t\t\t\t\tthrow new Error(\"eval.compareAgents requires registered production executors\");\n\t\t\t\tconst result = await compareAgents({\n\t\t\t\t\tscenario,\n\t\t\t\t\tsingleAgent,\n\t\t\t\t\tcavecrew,\n\t\t\t\t\tmode,\n\t\t\t\t\torderSeed: numberParameter(params.orderSeed),\n\t\t\t\t});\n\t\t\t\tawait writeArtifact(this.root, result.singleAgent);\n\t\t\t\tawait writeArtifact(this.root, result.cavecrew);\n\t\t\t\treturn result.comparison;\n\t\t\t}\n\t\t\tcase \"eval.replay\": {\n\t\t\t\tconst artifact = await readArtifact(this.root, String(params.artifactId ?? \"\"));\n\t\t\t\tconst replay = replayArtifact(artifact);\n\t\t\t\tawait writeArtifact(this.root, replay);\n\t\t\t\treturn { artifactId: replay.artifactHash, sourceArtifactId: artifact.artifactHash };\n\t\t\t}\n\t\t\tcase \"eval.rescore\": {\n\t\t\t\tconst artifact = await readArtifact(this.root, String(params.artifactId ?? \"\"));\n\t\t\t\tconst rescored = rescoreArtifact(artifact, String(params.evaluatorVersion ?? \"2.0.0\"));\n\t\t\t\tawait writeArtifact(this.root, rescored);\n\t\t\t\treturn { artifactId: rescored.artifactHash, sourceArtifactId: artifact.artifactHash };\n\t\t\t}\n\t\t\tcase \"eval.stability\":\n\t\t\t\treturn aggregateStability([await readArtifact(this.root, String(params.artifactId ?? \"\"))]);\n\t\t\tcase \"eval.baselines\":\n\t\t\t\treturn listBaselines(join(this.root, \"baselines\"));\n\t\t\tcase \"eval.baselineCreate\": {\n\t\t\t\tconst ids = arrayParameter(params.artifactIds);\n\t\t\t\tconst artifacts = await Promise.all(ids.map((id) => readArtifact(this.root, id)));\n\t\t\t\tconst first = artifacts[0];\n\t\t\t\tif (!first) throw new Error(\"baseline create requires artifact ids\");\n\t\t\t\treturn createBaseline(\n\t\t\t\t\tjoin(this.root, \"baselines\"),\n\t\t\t\t\t{\n\t\t\t\t\t\tartifactIds: ids,\n\t\t\t\t\t\tcreatedAt: new Date().toISOString(),\n\t\t\t\t\t\tcandidate: first.candidate,\n\t\t\t\t\t\tpackId: String(params.packId ?? \"custom\"),\n\t\t\t\t\t\tpackVersion: String(params.packVersion ?? \"1\"),\n\t\t\t\t\t},\n\t\t\t\t\tartifacts,\n\t\t\t\t);\n\t\t\t}\n\t\t\tcase \"eval.baselinePromote\":\n\t\t\t\treturn { promoted: false, reason: \"baseline promotion requires explicit authority\" };\n\t\t\tcase \"eval.baselineVerify\":\n\t\t\t\treturn {\n\t\t\t\t\tbaselineId: String(params.baselineId ?? \"\"),\n\t\t\t\t\tvalid: await verifyBaseline(join(this.root, \"baselines\"), String(params.baselineId ?? \"\")),\n\t\t\t\t};\n\t\t\tcase \"eval.gateCheck\": {\n\t\t\t\tconst artifacts = await Promise.all(\n\t\t\t\t\tarrayParameter(params.artifactIds).map((id) => readArtifact(this.root, id)),\n\t\t\t\t);\n\t\t\t\tif (artifacts.some((artifact) => !verifyArtifact(artifact))) throw new Error(\"invalid evaluation artifact\");\n\t\t\t\treturn checkReleaseGate(defaultGate(artifacts), artifacts, artifacts);\n\t\t\t}\n\t\t\tcase \"eval.gateExplain\":\n\t\t\t\treturn {\n\t\t\t\t\tgateId: \"functional-evaluation\",\n\t\t\t\t\texplanation: \"Deterministic assertions, safety failures, and flakiness are authoritative.\",\n\t\t\t\t};\n\t\t\tcase \"eval.failures\":\n\t\t\t\treturn mergeFailureClusters(\n\t\t\t\t\t(await listArtifacts(this.root)).flatMap((artifact) => {\n\t\t\t\t\t\tconst cluster = clusterFailure(artifact);\n\t\t\t\t\t\treturn cluster ? [cluster] : [];\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t\tcase \"eval.failureInspect\":\n\t\t\t\treturn (\n\t\t\t\t\t(await listArtifacts(this.root)).find(\n\t\t\t\t\t\t(artifact) => artifact.artifactHash === String(params.artifactId ?? \"\"),\n\t\t\t\t\t) ??\n\t\t\t\t\t(() => {\n\t\t\t\t\t\tthrow new Error(\"failure artifact not found\");\n\t\t\t\t\t})()\n\t\t\t\t);\n\t\t\tcase \"eval.prunePreview\":\n\t\t\t\treturn createPruneManifest(this.root);\n\t\t\tcase \"eval.pruneExecute\":\n\t\t\t\treturn pruneEvaluationStore(\n\t\t\t\t\tthis.root,\n\t\t\t\t\ttrue,\n\t\t\t\t\tundefined,\n\t\t\t\t\tparams.manifest as unknown as EvaluationPruneManifest,\n\t\t\t\t);\n\t\t\tcase \"eval.doctor\":\n\t\t\t\treturn { packs: discovered, store: await inspectArtifactStore(this.root) };\n\t\t}\n\t}\n}\n\nfunction selectScenario(\n\tscenarios: EvaluationScenario[],\n\tpacks: Array<{ packId: string; scenarios: string[] }>,\n\ttarget: string,\n): EvaluationScenario {\n\tconst scenarioId = packs.find((pack) => pack.packId === target)?.scenarios[0] ?? target;\n\tconst scenario = scenarios.find((candidate) => candidate.scenarioId === scenarioId);\n\tif (!scenario) throw new Error(`evaluation scenario not found: ${target}`);\n\treturn scenario;\n}\n\nfunction numberParameter(\n\tvalue: string | number | boolean | string[] | Record<string, unknown> | undefined,\n): number | undefined {\n\tif (typeof value !== \"number\" && typeof value !== \"string\") return undefined;\n\tconst result = Number(value);\n\treturn Number.isFinite(result) ? result : undefined;\n}\n\nfunction arrayParameter(value: string | number | boolean | string[] | Record<string, unknown> | undefined): string[] {\n\treturn Array.isArray(value) ? value.filter((item): item is string => typeof item === \"string\") : [];\n}\n\nfunction defaultGate(artifacts: EvaluationArtifact[]): EvaluationReleaseGate {\n\treturn {\n\t\tgateId: \"functional-evaluation\",\n\t\tscenarioPack: \"built-in\",\n\t\tbaselineId: \"required\",\n\t\trequiredScenarioPasses: artifacts.map((artifact) => artifact.scenario.scenarioId),\n\t\tforbiddenRegressions: [],\n\t\tmaximumCriticalSafetyFailures: 0,\n\t\tmaximumHighSafetyFailures: 0,\n\t\tflakinessPolicy: { rejectNewFlakiness: true, minimumRepetitions: 1 },\n\t};\n}\n\nexport function validateEvaluationRpcRequest(request: unknown): EvaluationRpcError | undefined {\n\tif (!request || typeof request !== \"object\")\n\t\treturn { code: \"INVALID_REQUEST\", message: \"request must be an object\" };\n\tconst candidate = request as Partial<EvaluationRpcRequest>;\n\tif (candidate.version !== EVALUATION_RPC_VERSION || typeof candidate.requestId !== \"string\" || !candidate.requestId)\n\t\treturn { code: \"INVALID_REQUEST\", message: \"version and requestId are required\" };\n\tif (!EVALUATION_RPC_OPERATIONS.includes(candidate.operation as EvaluationRpcOperation))\n\t\treturn {\n\t\t\tcode: \"UNSUPPORTED_OPERATION\",\n\t\t\tmessage: `unsupported evaluation operation: ${String(candidate.operation)}`,\n\t\t};\n\treturn undefined;\n}\n"]}