// Regression coverage for the Phase 2 safety invariants in ../index.ts: // - bare: true is rejected at runtime (never reaches the runner) // - timeoutMs is clamped to [MIN_TIMEOUT_MS, MAX_TIMEOUT_MS] even if a caller // bypasses the tool schema and passes 0 or an oversized value directly // - no more than DEFAULT_MAX_ACTIVE_RUNS runs may be active at once // - prompt-facing tool metadata explicitly forbids cross-plugin misuse with // pi-subagents (e.g. subagent_wait) and busy-polling get/list instead of // waiting for the guaranteed completion hook (see forensic findings from // Pi session 019fd397-3a29-7572-a57b-059309b321b0) // // Run with: node --test scripts/index_safety.test.ts import assert from "node:assert/strict"; import { mkdtemp, rm, writeFile, chmod } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; import claudeCodeExtension from "../index.ts"; const SIX_HOURS_MS = 6 * 60 * 60 * 1000; const THIRTY_SECONDS_MS = 30_000; // Minimal fake of ExtensionAPI: index.ts only calls on/registerTool/sendMessage. function makeFakePi() { const tools = new Map(); const handlers = new Map(); const messages: Array<{ message: any; options?: any }> = []; const pi = { on(event: string, handler: unknown) { handlers.set(event, handler); }, registerTool(tool: any) { tools.set(tool.name, tool); }, sendMessage(message: any, options?: any) { messages.push({ message, options }); }, }; return { pi: pi as any, tools, handlers, messages }; } async function withFakeClaudeOnPath(scriptBody: string, fn: (env: NodeJS.ProcessEnv) => Promise): Promise { const dir = await mkdtemp(join(tmpdir(), "pi-claude-code-safety-test-")); const binDir = join(dir, "bin"); await import("node:fs/promises").then(fs => fs.mkdir(binDir, { recursive: true })); const fakeClaude = join(binDir, "claude"); await writeFile(fakeClaude, scriptBody); await chmod(fakeClaude, 0o755); try { return await fn({ ...process.env, PATH: `${binDir}:${process.env.PATH}` }); } finally { await rm(dir, { recursive: true, force: true }); } } async function waitFor(predicate: () => boolean, timeoutMs = 2_000): Promise { const deadline = Date.now() + timeoutMs; while (!predicate()) { if (Date.now() >= deadline) throw new Error("Timed out waiting for completion message"); await new Promise(resolve => setTimeout(resolve, 10)); } } test("rejects bare: true before validating the runner or working directory", async () => { const { pi, tools } = makeFakePi(); claudeCodeExtension(pi); const spawnTool = tools.get("spawn_claude_code"); assert.ok(spawnTool, "spawn_claude_code must be registered"); const missingCwd = join(tmpdir(), `bare-must-precede-cwd-validation-${process.pid}-${Date.now()}`); await assert.rejects( () => spawnTool.execute("call-1", { prompt: "do the thing", bare: true, cwd: missingCwd }, undefined, undefined, { cwd: process.cwd() }), /bare: true is disabled/, ); }); test("does not reject when bare is omitted or false", async () => { // Uses a fake `claude` that exits immediately so this stays fast/deterministic. await withFakeClaudeOnPath( "#!/usr/bin/env bash\nprintf '%s\\n' '{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"result\":\"ok\"}'\nexit 0\n", async (env) => { const { pi, tools, messages } = makeFakePi(); claudeCodeExtension(pi); const spawnTool = tools.get("spawn_claude_code"); const origEnv = { ...process.env }; Object.assign(process.env, env); try { const result = await spawnTool.execute("call-1", { prompt: "do the thing", bare: false }, undefined, undefined, { cwd: process.cwd() }); assert.ok(result.details.runId, "expected a run to start when bare is false"); await waitFor(() => messages.some(({ message }) => message.customType === "claude-code-complete")); } finally { process.env = origEnv; } }, ); }); test("steers exactly one completion into the active run with relevance guidance", async () => { await withFakeClaudeOnPath( "#!/usr/bin/env bash\nprintf '%s\\n' '{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"result\":\"ok\"}'\nexit 0\n", async (env) => { const { pi, tools, messages } = makeFakePi(); claudeCodeExtension(pi); const spawnTool = tools.get("spawn_claude_code"); const origEnv = { ...process.env }; Object.assign(process.env, env); try { const started = await spawnTool.execute( "call-steer", { prompt: "finish deterministically" }, undefined, undefined, { cwd: process.cwd() }, ); await waitFor(() => messages.some(({ message }) => message.customType === "claude-code-complete")); const completions = messages.filter(({ message }) => message.customType === "claude-code-complete"); assert.equal(completions.length, 1); assert.deepEqual(completions[0]?.options, { triggerTurn: true, deliverAs: "steer" }); assert.equal(completions[0]?.message.details.runId, started.details.runId); assert.equal(completions[0]?.message.details.terminationReason, "completed"); assert.match(completions[0]?.message.content, /stale, superseded, or irrelevant/i); assert.match(completions[0]?.message.content, /independently verify/i); } finally { process.env = origEnv; } }, ); }); test("clamps timeoutMs to the non-disableable ceiling even if 0 is passed directly", async () => { await withFakeClaudeOnPath( "#!/usr/bin/env bash\nprintf '%s\\n' '{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"result\":\"ok\"}'\nexit 0\n", async (env) => { const { pi, tools, messages } = makeFakePi(); claudeCodeExtension(pi); const spawnTool = tools.get("spawn_claude_code"); const origEnv = { ...process.env }; Object.assign(process.env, env); try { // A caller that bypasses the tool-parameter schema (e.g. a stale saved // call, or a client that does not enforce JSON Schema minimums) must // still be clamped up to MIN_TIMEOUT_MS, never left at 0/disabled. const zero = await spawnTool.execute("call-zero", { prompt: "x", timeoutMs: 0 }, undefined, undefined, { cwd: process.cwd() }); assert.equal(zero.details.timeoutMs, THIRTY_SECONDS_MS); // And an oversized value must be clamped down to the 6h ceiling. const huge = await spawnTool.execute("call-huge", { prompt: "x", timeoutMs: 100 * 60 * 60 * 1000 }, undefined, undefined, { cwd: process.cwd() }); assert.equal(huge.details.timeoutMs, SIX_HOURS_MS); await waitFor(() => messages.filter(({ message }) => message.customType === "claude-code-complete").length === 2); } finally { process.env = origEnv; } }, ); }); test("concurrent spawn_claude_code calls never exceed the max active run limit (TOCTOU race)", async () => { // Adversarial: fire far more concurrent spawn calls than the limit allows, // all in the same tick (no sequential awaiting), to prove the capacity // check and slot reservation are atomic. Before the fix, two concurrent // calls could both read the same "under limit" count before either had // reserved a slot (the reservation happened after an `await mkdir`), so // more than DEFAULT_MAX_ACTIVE_RUNS runs could be spawned. await withFakeClaudeOnPath( "#!/usr/bin/env bash\nsleep 30\n", async (env) => { const { pi, tools } = makeFakePi(); claudeCodeExtension(pi); const spawnTool = tools.get("spawn_claude_code"); const stopTool = tools.get("stop_claude_code_run"); const origEnv = { ...process.env }; Object.assign(process.env, env); const CONCURRENCY = 10; const MAX_ACTIVE = 3; try { const results = await Promise.allSettled( Array.from({ length: CONCURRENCY }, (_, i) => spawnTool.execute(`race-${i}`, { prompt: `task ${i}`, timeoutMs: THIRTY_SECONDS_MS }, undefined, undefined, { cwd: process.cwd() }), ), ); const fulfilled = results.filter((r): r is PromiseFulfilledResult => r.status === "fulfilled"); const rejected = results.filter((r): r is PromiseRejectedResult => r.status === "rejected"); assert.equal(fulfilled.length, MAX_ACTIVE, `expected exactly ${MAX_ACTIVE} runs to launch, got ${fulfilled.length}`); assert.equal(rejected.length, CONCURRENCY - MAX_ACTIVE); for (const r of rejected) { assert.match(String(r.reason?.message ?? r.reason), /Refusing to spawn Claude Code: 3 active runs already exist \(max 3\)/); } // Every fulfilled result must be a distinct, real run (no duplicate ids // from a double-reservation, which is exactly what the race would cause). const runIds = fulfilled.map(r => r.value.details.runId); assert.equal(new Set(runIds).size, runIds.length, "run ids must be unique"); // claude_cli_runner.sh installs its SIGTERM trap and assigns CLAUDE_PID // in separate statements after backgrounding the child; stopping before // that assignment lands means the trap fires as a no-op (empty // CLAUDE_PID) and the fake worker then runs unsupervised to completion, // hanging this test for tens of seconds. A short grace window lets every // runner reach the assignment before we send SIGTERM, so cleanup is fast // and deterministic instead of racing the shell's own startup. await new Promise(resolve => setTimeout(resolve, 250)); for (const runId of runIds) { await stopTool.execute("stop", { runId }, undefined, undefined, { cwd: process.cwd() }).catch(() => {}); } } finally { process.env = origEnv; await new Promise(resolve => setTimeout(resolve, 300)); } }, ); }); test("refuses to spawn beyond the max active run limit", async () => { // A fake `claude` that sleeps keeps the runner (and therefore the run) active // for the duration of the test, so the 4th spawn attempt should be rejected. await withFakeClaudeOnPath( "#!/usr/bin/env bash\nsleep 30\n", async (env) => { const { pi, tools } = makeFakePi(); claudeCodeExtension(pi); const spawnTool = tools.get("spawn_claude_code"); const stopTool = tools.get("stop_claude_code_run"); const origEnv = { ...process.env }; Object.assign(process.env, env); const started: string[] = []; try { for (let i = 0; i < 3; i += 1) { const result = await spawnTool.execute(`call-${i}`, { prompt: `task ${i}`, timeoutMs: THIRTY_SECONDS_MS }, undefined, undefined, { cwd: process.cwd() }); started.push(result.details.runId); } await assert.rejects( () => spawnTool.execute("call-overflow", { prompt: "one too many" }, undefined, undefined, { cwd: process.cwd() }), /Refusing to spawn Claude Code: 3 active runs already exist \(max 3\)/, ); // See the race test above: give each runner's CLAUDE_PID assignment a // moment to land before SIGTERM, so stop can't race the shell startup. await new Promise(resolve => setTimeout(resolve, 250)); } finally { for (const runId of started) { await stopTool.execute("stop", { runId }, undefined, undefined, { cwd: process.cwd() }).catch(() => {}); } process.env = origEnv; // Give SIGTERM/SIGKILL escalation a moment to land before the temp dir // (and its fake claude binary) is removed by the caller. await new Promise(resolve => setTimeout(resolve, 300)); } }, ); }); test("persists a shutdown record that a replacement extension can recover", async () => { await withFakeClaudeOnPath( "#!/usr/bin/env bash\nsleep 30\n", async (env) => { const first = makeFakePi(); claudeCodeExtension(first.pi); const spawnTool = first.tools.get("spawn_claude_code"); const originalEnv = { ...process.env }; Object.assign(process.env, env); let runDir = ""; try { const started = await spawnTool.execute("shutdown-run", { prompt: "long task" }, undefined, undefined, { cwd: process.cwd() }); runDir = join(started.details.outputFile, ".."); await new Promise(resolve => setTimeout(resolve, 250)); await first.handlers.get("session_shutdown")(); const replacement = makeFakePi(); claudeCodeExtension(replacement.pi); const getTool = replacement.tools.get("get_claude_code_run"); const recovered = await getTool.execute("recover", { runId: started.details.runId }); assert.equal(recovered.details.terminationReason, "shutdown"); assert.equal(recovered.details.recovered, true); assert.equal(recovered.details.running, false); } finally { process.env = originalEnv; if (runDir) await rm(runDir, { recursive: true, force: true }); } }, ); }); test("an abrupt crash (no session_shutdown) is never recovered as a clean shutdown", async () => { // Regression for a blocking PR #85 finding: scheduleLedger used to default a // missing terminationReason to "shutdown" when it seeded the ledger before // spawning. If Pi crashed before session_shutdown ever ran, that seeded // record was the only thing on disk, so a replacement extension recovered // the run as terminationReason "shutdown"/running:false even though the // detached worker could still be executing. Simulate exactly that: spawn a // long-running worker and simulate a crash by never invoking the // session_shutdown handler, then load a replacement extension instance. await withFakeClaudeOnPath( "#!/usr/bin/env bash\nsleep 30\n", async (env) => { const first = makeFakePi(); claudeCodeExtension(first.pi); const spawnTool = first.tools.get("spawn_claude_code"); const originalEnv = { ...process.env }; Object.assign(process.env, env); let runDir = ""; let runId = ""; try { const started = await spawnTool.execute("crash-run", { prompt: "long task" }, undefined, undefined, { cwd: process.cwd() }); runId = started.details.runId; runDir = join(started.details.outputFile, ".."); // Give scheduleLedger's pre-spawn write time to land, then simulate an // abrupt process crash: no session_shutdown handler is ever invoked. const replacement = makeFakePi(); claudeCodeExtension(replacement.pi); const getTool = replacement.tools.get("get_claude_code_run"); await assert.rejects( () => getTool.execute("recover", { runId }), /Unknown Claude Code run/, "a pending (non-shutdown) ledger record must not be recoverable", ); } finally { const stopTool = first.tools.get("stop_claude_code_run"); await stopTool.execute("stop", { runId }, undefined, undefined, { cwd: process.cwd() }).catch(() => {}); await new Promise(resolve => setTimeout(resolve, 300)); process.env = originalEnv; if (runDir) await rm(runDir, { recursive: true, force: true }); } }, ); }); test("spawn_claude_code metadata warns against cross-plugin subagent_wait misuse and busy-polling", () => { // Forensic finding from Pi session 019fd397-3a29-7572-a57b-059309b321b0: a // Pi agent passed a spawn_claude_code run ID to subagent_wait (a pi-subagents // tool this extension has no relationship with), then busy-polled // get_claude_code_run 31 times despite the guaranteed completion hook. The // tool metadata must make both mistakes explicitly unambiguous to the model. const { pi, tools } = makeFakePi(); claudeCodeExtension(pi); const spawnTool = tools.get("spawn_claude_code"); assert.ok(spawnTool, "spawn_claude_code must be registered"); assert.match(spawnTool.description, /subagent_wait/); assert.match(spawnTool.description, /pi-subagents/); assert.match(spawnTool.description, /do not poll/i); assert.match(spawnTool.promptSnippet, /subagent_wait/); const guidelines: string[] = spawnTool.promptGuidelines; assert.ok(guidelines.some(g => /subagent_wait/.test(g) && /pi-subagents/.test(g)), "must forbid passing run IDs to subagent_wait/pi-subagents"); assert.ok(guidelines.some(g => /busy-poll/i.test(g)), "must forbid busy-polling get/list"); assert.ok(guidelines.some(g => /inactivity alert/.test(g) && /explicit user request/.test(g) && /stuck/.test(g)), "must scope mid-flight inspection to alert/user-request/suspected-stuck"); assert.ok(guidelines.some(g => /completion hook fires/.test(g) && /once/.test(g)), "must instruct a single post-completion get_claude_code_run call"); }); test("get_claude_code_run and list_claude_code_runs metadata are explicit that they are one-shot, not a wait", () => { const { pi, tools } = makeFakePi(); claudeCodeExtension(pi); const getTool = tools.get("get_claude_code_run"); assert.ok(getTool, "get_claude_code_run must be registered"); assert.match(getTool.description, /one-shot/i); assert.match(getTool.description, /NOT a wait or polling mechanism/i); assert.match(getTool.description, /pi-subagents/); assert.match(getTool.promptSnippet, /not a wait\/poll/i); const getGuidelines: string[] = getTool.promptGuidelines; assert.ok(getGuidelines.some(g => /busy-polling/i.test(g)), "must warn against busy-polling"); assert.ok(getGuidelines.some(g => /subagent_wait/.test(g) && /pi-subagents/.test(g)), "must forbid cross-plugin run ID misuse"); const listTool = tools.get("list_claude_code_runs"); assert.ok(listTool, "list_claude_code_runs must be registered"); assert.match(listTool.description, /one-shot/i); assert.match(listTool.description, /NOT a wait or polling mechanism/i); assert.match(listTool.promptSnippet, /not a wait\/poll/i); const listGuidelines: string[] = listTool.promptGuidelines; assert.ok(listGuidelines.some(g => /repeatedly/i.test(g)), "must warn against repeated calls while runs are active"); });