import { randomUUID } from "node:crypto"; import { join } from "node:path"; import type { AgentToolResult, ExtensionContext, ToolDefinition, } from "@earendil-works/pi-coding-agent"; import { defaultCodemodeSettings } from "../config/settings.ts"; import { RESERVED_TOOL_NAMES } from "../bridge/reserved.ts"; import { buildEvalPrompt } from "../prompt/eval-prompt.ts"; import { abortError, CellExecution, defaultTimeoutFactory, } from "./cell-execution.ts"; import { CellHandler, type CellState } from "./cell-handler.ts"; import { EvalDetachedCellManager } from "./detached-cell-manager.ts"; import { detachedKernelBusyError, executeEvalControl, resultAfterDetach, } from "./detached-eval-result.ts"; import { evalTimeoutBehavior, isEvalControlRequest, parseEvalRequest, } from "./eval-request.ts"; import type { CreateEvalToolOptions, EvalCellInvocation, } from "./eval-tool-options.ts"; import { describeTimeoutState } from "./interrupt-note.ts"; import { SessionStore } from "./session-store.ts"; import { createEvalInputSchema, type EvalInputSchema, type EvalRenderState, type EvalToolDetails, enabledLanguageList, } from "./types.ts"; export type { EvalTimeoutFactory } from "./cell-execution.ts"; export type { CreateEvalToolOptions } from "./eval-tool-options.ts"; export type { EnabledEvalLanguages, EvalKernel, EvalKernelManager, } from "./types.ts"; /** Update model-facing metadata while preserving the tool definition identity. */ export function refreshEvalTool( tool: ReturnType, options: CreateEvalToolOptions ): void { const refreshed = createEvalTool(options); tool.description = refreshed.description; tool.promptSnippet = refreshed.promptSnippet; tool.promptGuidelines = refreshed.promptGuidelines; tool.parameters = refreshed.parameters; if (refreshed.renderCall === undefined) delete tool.renderCall; else tool.renderCall = refreshed.renderCall; if (refreshed.renderResult === undefined) delete tool.renderResult; else tool.renderResult = refreshed.renderResult; } /** * Reads the active tool metadata for the typed-declaration prompt section. * * pi 0.83.0 forbids action methods during extension loading: the baseline * eval registration runs at factory time, so `getAllTools()` throws there. * That registration has no active session tools yet, so degrading to no * declarations is correct — the section renders when the runtime re-registers * the eval tool on `session_start`/`model_select`, where the call succeeds. */ function listActiveTools( options: CreateEvalToolOptions ): | readonly { readonly name: string; readonly parameters?: unknown }[] | undefined { if (options.listTools === undefined) { return; } try { return options.listTools(); } catch {} } export function createEvalTool(options: CreateEvalToolOptions): ToolDefinition< EvalInputSchema, EvalToolDetails, EvalRenderState > & { readonly name: "eval"; } { const parameters = createEvalInputSchema(options.enabledLanguages); const prompt = buildEvalPrompt(options.enabledLanguages, { spawns: options.spawns ?? false, ...(options.spawnDefaultAgent === undefined ? {} : { spawnDefaultAgent: options.spawnDefaultAgent }), ...(options.modelId === undefined ? {} : { modelId: options.modelId }), ...(options.hostLine === undefined ? {} : { hostLine: options.hostLine }), waitTool: options.waitTool === true, tools: options.toolSnapshot?.declaredTools ?? listActiveTools(options), toolSnapshot: options.toolSnapshot, reservedCollisionNote: (() => { const collisions = listActiveTools(options)?.filter((tool) => RESERVED_TOOL_NAMES.has(tool.name)).map((tool) => tool.name) ?? []; return collisions.length === 0 ? undefined : `Warning: ${collisions.join(", ")} is reserved by the kernel bridge and cannot be called as a host tool from cells.`; })(), }); const fallbackCellManager = new EvalDetachedCellManager({ artifactsDir: options.artifactsDir, }); return { name: "eval", label: "Eval", description: prompt.description, promptSnippet: prompt.promptSnippet, promptGuidelines: [...prompt.promptGuidelines], parameters, executionMode: "sequential", ...(options.renderers?.renderCall === undefined ? {} : { renderCall: options.renderers.renderCall }), ...(options.renderers?.renderResult === undefined ? {} : { renderResult: options.renderers.renderResult }), async execute(toolCallId, params, signal, onUpdate, ctx) { if (options.isActive !== undefined && !options.isActive()) { throw new Error("No active codemode session"); } const cellManager = options.cellManager ?? fallbackCellManager; const languages = enabledLanguageList(options.enabledLanguages); const request = parseEvalRequest(params); if (isEvalControlRequest(request)) { return await executeEvalControl(cellManager, request); } if (options.budget !== undefined && options.budget.exhausted()) { throw new RangeError( `Eval budget exhausted: ${options.budget.weightedTokensUsed()} weighted tokens used, ${options.budget.remainingTokens()} remaining. No new cells can run in this session.` ); } if (options.proxyExecutor) { return await options.proxyExecutor(request, signal); } if (!languages.includes(request.language)) { throw new RangeError( `Unsupported eval language "${request.language}". Enabled languages: ${languages.join(", ")}` ); } const busy = cellManager.busyFor(request.language); if (busy !== undefined) { throw detachedKernelBusyError(busy); } options.executionTracker?.assertEvalExecutionAllowed(); const lifecycleController = new AbortController(); const combinedSignal = signal ? AbortSignal.any([signal, lifecycleController.signal]) : lifecycleController.signal; const execution = runEvalCell(options, cellManager, { cellId: toolCallId, input: request, signal: combinedSignal, onUpdate, ctx, }); return options.executionTracker ? await options.executionTracker.trackEvalExecution( execution, lifecycleController ) : await execution; }, }; } async function runEvalCell( options: CreateEvalToolOptions, cellManager: EvalDetachedCellManager, invocation: EvalCellInvocation ): Promise> { if (invocation.signal.aborted) { throw abortError(invocation.signal.reason); } const timeoutMs = Math.floor( (invocation.input.timeout ?? options.cellTimeoutSeconds) * 1000 ); const timeoutBehavior = evalTimeoutBehavior(invocation.input, invocation.ctx); const bridgeAbortController = new AbortController(); const cellSignal = AbortSignal.any([ invocation.signal, bridgeAbortController.signal, ]); const bridgeContext: ExtensionContext = { ...invocation.ctx, signal: cellSignal, }; const state: CellState = { input: invocation.input, signal: cellSignal, onUpdate: invocation.onUpdate, toolCalls: [], pendingBridgeCalls: [], statusEvents: [], active: true, output: "", phase: undefined, durationMs: 0, status: "pending", }; // One store instance per cell run: the rejection path discards through it // even when the CellHandler was never created, and detached cells commit // through the same handler+store when they later complete. const sessionStore = options.sessionStore ?? new SessionStore(); const cell = cellManager.create(invocation.cellId, invocation.input); let execution: CellExecution; execution = new CellExecution({ callerSignal: invocation.signal, cellId: invocation.cellId, timeoutMs, timeoutFactory: options.timeoutFactory ?? defaultTimeoutFactory, onTimeout: (error) => { if (timeoutBehavior === "detach" && cellManager.detach(cell)) { execution.detach(); return; } execution.cancel(error); }, onAbort: (error) => { state.active = false; bridgeAbortController.abort(error); }, }); const running = executeCell( options, invocation, cellManager, cell, state, execution, bridgeContext, bridgeAbortController, timeoutBehavior, timeoutMs, sessionStore ); const finalized = running.then( (result) => { cellManager.complete(cell, result); return result; }, (error: unknown) => { // Cells that reject (user cancellation, timeout, session disposal) never // reach CellHandler.finalize, so discard their staged store writes here; // CellHandler.finalizeCancellation already discarded for disposed errors. sessionStore.discard(invocation.input.language); cellManager.fail( cell, error instanceof Error ? error : new Error(String(error)) ); throw error; } ); const outcome = await Promise.race([ finalized.then((result) => ({ kind: "result" as const, result })), execution.detached.then(() => ({ kind: "detached" as const })), ]); if (outcome.kind === "detached") { return resultAfterDetach( cellManager.peek(invocation.cellId), invocation.input ); } return outcome.result; } async function executeCell( options: CreateEvalToolOptions, invocation: EvalCellInvocation, cellManager: EvalDetachedCellManager, cell: Parameters[0], state: CellState, execution: CellExecution, bridgeContext: ExtensionContext, bridgeAbortController: AbortController, timeoutBehavior: "detach" | "error", timeoutMs: number, sessionStore: SessionStore ): Promise> { let handler: CellHandler | undefined; try { const kernel = await execution.wait( options.kernelManager.getKernel(invocation.input.language, (message) => { if (!state.active || handler === undefined) { return; } const pending = handler.handle(message); void pending.catch((error: unknown) => execution.cancel(error)); }) ); execution.setKernel(kernel); const activeHandler = new CellHandler(kernel, state, { executeTool: options.executeTool, ...(options.listTools === undefined ? {} : { listTools: options.listTools }), settings: options.settings ?? defaultCodemodeSettings, ...(options.complete === undefined ? {} : { complete: options.complete }), ctx: bridgeContext, pauseTimeout: () => execution.pause(), resumeTimeout: () => execution.resume(), ...(options.artifactsDir === undefined ? {} : { artifactPath: join( options.artifactsDir, `eval-${randomUUID()}.log` ), }), ...(options.imageResizer === undefined ? {} : { imageResizer: options.imageResizer }), sessionStore, }); handler = activeHandler; cellManager.markRunning(cell, kernel, () => activeHandler.liveResult()); if ( "setContext" in options.kernelManager && typeof options.kernelManager.setContext === "function" ) { options.kernelManager.setContext(bridgeContext); } // Subprocess preludes (py/rb) call tools over the HTTP bridge; register // this cell's hooks with the session manager so bridge tool calls get the // same enrichment and agent() progress status as the JS kernel path. if ( "setBridgeCellSession" in options.kernelManager && typeof options.kernelManager.setBridgeCellSession === "function" ) { options.kernelManager.setBridgeCellSession( invocation.input.language, activeHandler.bridgeSession() ); } if (invocation.input.reset) { await execution.wait(kernel.reset()); } const result = await execution.wait( kernel.run({ cellId: invocation.cellId, code: invocation.input.code, ...(timeoutBehavior === "error" ? { timeoutMs } : {}), ...(invocation.input.strings === undefined ? {} : { strings: invocation.input.strings }), }) ); if (result.ok && state.pendingBridgeCalls.length > 0) { await execution.wait(Promise.all(state.pendingBridgeCalls)); } return await handler.finalize(result); } catch (error) { if ( handler && error instanceof Error && error.name === "CodemodeSessionDisposedError" ) { return await handler.finalizeCancellation(error); } if (error instanceof Error && error.name === "TimeoutError") { throw await describeTimeoutState(error, execution); } throw error; } finally { state.active = false; bridgeAbortController.abort(); if ( "setBridgeCellSession" in options.kernelManager && typeof options.kernelManager.setBridgeCellSession === "function" ) { options.kernelManager.setBridgeCellSession( invocation.input.language, undefined ); } execution.finish(); if (handler) { await handler.flushOutput(); } } }