/** * TestSession — orchestrates a test run with playbook, mock tools, and mock UI. * * 1. Creates a real pi environment (extensions, tools, hooks, session) * 2. Replaces streamFn with playbook * 3. Intercepts tool.execute() for mockTools * 4. Injects mock UI context * 5. Collects events * 6. Runs conversation script */ import * as fs from "node:fs"; import * as path from "node:path"; import * as os from "node:os"; import { createAgentSession, DefaultResourceLoader, SessionManager, SettingsManager, type AgentSessionEvent, } from "@earendil-works/pi-coding-agent"; import type { AgentTool } from "@earendil-works/pi-agent-core"; import { getModel } from "@earendil-works/pi-ai"; import { createPlaybookStreamFn, type PlaybookState } from "./playbook.js"; import { interceptToolExecution } from "./mock-tools.js"; import { createMockUIContext } from "./mock-ui.js"; import { createEventCollector } from "./events.js"; import { formatPlaybookDiagnostic } from "./diagnostics.js"; import type { TestSessionOptions, TestSession, Turn, ToolCallRecord, ToolResultRecord, } from "./types.js"; export async function createTestSession(options: TestSessionOptions = {}): Promise { const propagateErrors = options.propagateErrors ?? true; const ownsTmpDir = !options.cwd; const cwd = options.cwd ?? fs.mkdtempSync(path.join(os.tmpdir(), "pi-test-harness-")); // Ensure cwd exists if (!fs.existsSync(cwd)) { fs.mkdirSync(cwd, { recursive: true }); } // Build resource loader with extensions const settingsManager = SettingsManager.inMemory(); const loader = new DefaultResourceLoader({ cwd, agentDir: cwd, // Use cwd as agent dir to avoid touching real ~/.pi settingsManager, additionalExtensionPaths: options.extensions?.map((p) => path.resolve(cwd, p)) ?? [], extensionFactories: options.extensionFactories, systemPromptOverride: options.systemPrompt ? () => options.systemPrompt! : undefined, }); await loader.reload(); // Use a real model definition (never actually called — playbook replaces streamFn) const playbookModel = getModel("openai", "gpt-4o"); // Create real session with in-memory persistence const { session, extensionsResult } = await createAgentSession({ cwd, agentDir: cwd, model: playbookModel, sessionManager: SessionManager.inMemory(), settingsManager, resourceLoader: loader, }); // Override getApiKey to bypass real auth checks (on both agent and session) (session.agent as any).getApiKey = async () => "test-key"; // The session also validates via _modelRegistry.getApiKey — patch it const origModelRegistry = (session as any)._modelRegistry; if (origModelRegistry) { origModelRegistry.getApiKey = async () => "test-key"; origModelRegistry.getApiKeyForProvider = async () => "test-key"; if (typeof origModelRegistry.getApiKeyAndHeaders === "function") { origModelRegistry.getApiKeyAndHeaders = async () => ({ ok: true, apiKey: "test-key", headers: {} }); } if (typeof origModelRegistry.hasConfiguredAuth === "function") { origModelRegistry.hasConfiguredAuth = () => true; } if (typeof origModelRegistry.isUsingOAuth === "function") { origModelRegistry.isUsingOAuth = () => false; } } // Check for extension load errors if (extensionsResult.errors.length > 0) { session.dispose(); if (ownsTmpDir && fs.existsSync(cwd)) { fs.rmSync(cwd, { recursive: true, force: true }); } const errors = extensionsResult.errors .map((e) => ` ${e.path}: ${e.error}`) .join("\n"); throw new Error(`Extension load errors:\n${errors}`); } // Event collection const events = createEventCollector(); let currentStep = 0; // Subscribe to session events session.subscribe((event: AgentSessionEvent) => { events.all.push(event); // Collect tool call events if (event.type === "tool_execution_start") { const record: ToolCallRecord = { step: currentStep, toolName: event.toolName, input: (event as any).args ?? {}, blocked: false, }; events.toolCalls.push(record); } if (event.type === "tool_execution_end") { // Compute result text once from serialized event result const resultText = (event as any).result?.content ?.filter((c: any) => c.type === "text") ?.map((c: any) => c.text) ?.join("\n") ?? ""; // Record native tool results when the harness wrapper has no entry // (e.g., pre-execute blocks that never reach the wrapped tool) const toolCallId = (event as any).toolCallId as string | undefined; const hasRecord = toolCallId ? events.toolResults.some((r) => r.toolCallId === toolCallId) : false; if (!hasRecord && (event as any).result) { const record: ToolResultRecord = { step: currentStep, toolName: event.toolName, toolCallId: toolCallId ?? "", text: resultText, content: (event as any).result?.content ?? [], isError: event.isError, details: (event as any).result?.details, mocked: false, }; events.toolResults.push(record); } if (event.isError) { // Check if this was a block (look at the most recent tool call) const lastCall = events.toolCalls[events.toolCalls.length - 1]; if (lastCall && lastCall.toolName === event.toolName) { // Detect block via result text. We cannot use isBlockedError() here // because the AgentSessionEvent only carries the serialized result // content — not the original Error object. Pi does not yet export a // typed block error, so message-string matching is the only option // at this layer. Keep in sync with isBlockedError() in mock-tools.ts. if (resultText.includes("blocked") || resultText.includes("Plan mode")) { lastCall.blocked = true; lastCall.blockReason = resultText; } } } } // Collect messages if (event.type === "message_end") { events.messages.push(event.message); } }); // Playbook state (initialized on run()) let playbookState: PlaybookState | null = null; // Mock UI context const mockUI = createMockUIContext(options.mockUI, events.ui); // Inject mock UI context via bindExtensions await session.bindExtensions({ uiContext: mockUI, onError: (err) => { console.error(`[pi-test-harness] Extension error: ${err.event} — ${err.error}`); }, }); // Capture original tools before any wrapping — used in run() to avoid double-wrap const originalTools: AgentTool[] = [...((session.agent as any).state.tools as AgentTool[])]; const testSession: TestSession = { session, cwd, events, get playbook() { return { consumed: playbookState?.consumed ?? 0, remaining: playbookState?.remaining ?? 0, }; }, async run(...turns: Turn[]): Promise { // Create playbook streamFn const { streamFn, state } = createPlaybookStreamFn(turns); playbookState = state; // Replace the model with the playbook (session.agent as any).streamFn = streamFn; (session.agent as any).getApiKey = () => "test-key"; // Always wrap tools for event collection; if no mocks configured, pass empty map const effectiveMockTools = options.mockTools ?? {}; const currentTools = originalTools; const runner = session.extensionRunner; const interceptedTools = interceptToolExecution( currentTools, effectiveMockTools, events.toolResults, state, propagateErrors, runner, ); const agent = session.agent as any; if (typeof agent.setTools === "function") { agent.setTools(interceptedTools); } else { agent.state.tools = interceptedTools; } // Run each turn for (const turn of turns) { currentStep = state.consumed; await session.prompt(turn.prompt); await (session.agent as any).waitForIdle(); } // Auto-assert: playbook fully consumed if (state.remaining > 0) { // Collect remaining actions for diagnostics const allActions = turns.flatMap((t) => t.actions); const remaining = allActions.slice(state.consumed); const diagnostic = formatPlaybookDiagnostic("remaining", state, remaining); throw new Error(diagnostic); } }, /** * Dispose the test session and clean up the temp directory (if owned). * * Note: `session.dispose()` does NOT fire `session_shutdown`. That event is * dispatched by pi at Node.js process exit. Extensions that open resources in * `session_start` (e.g., SQLite databases) keep those resources open until the * process exits. Use `safeRmSync` when cleaning up extension-owned files in * afterEach hooks on Windows to avoid EPERM errors. */ dispose(): void { session.dispose(); if (ownsTmpDir && fs.existsSync(cwd)) { fs.rmSync(cwd, { recursive: true, force: true }); } }, }; return testSession; }