/** * Tests for the per-turn payload rewriter. * * Run with: node --test --experimental-strip-types .pi/extensions/plan-stepdown/rewrite.test.ts * * Fixtures mirror real wire payloads from * pi/packages/ai/src/providers/openai-responses.ts (buildParams) * pi/packages/ai/src/providers/openai-completions.ts * pi/packages/ai/src/providers/anthropic.ts (buildParams) * pi/packages/ai/src/providers/google.ts */ import test from "node:test"; import assert from "node:assert/strict"; import { applyOpenAIWebSearchToPayload, applyPromptCacheToPayload, applyRungToPayload, capThinkingLevel, chooseContextSafeRungIndex, chooseRung, contentText, createPromptCacheKey, detectApi, detectReasoningBump, extractLatestAssistantText, formatAcceptedPlanContext, hasCustomMessage, isMaxOutputTokensLikeStop, nextReasoningEffortForMaxOutput, nextStage, normalizeAcceptedPlan, startsWithShellCommand, type ReasoningBumpConfig, type Rung, } from "./rewrite.ts"; // ============================================================================ // detectApi // ============================================================================ test("detectApi: openai-responses payload", () => { const p = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], stream: true, reasoning: { effort: "high", summary: "auto" }, store: false, }; assert.equal(detectApi(p), "openai-responses"); }); test("detectApi: openai-completions payload", () => { const p = { model: "gpt-4o", messages: [{ role: "user", content: "hi" }], stream: true, reasoning_effort: "high", }; assert.equal(detectApi(p), "openai-completions"); }); test("detectApi: anthropic with thinking", () => { const p = { model: "claude-sonnet-4-6", messages: [{ role: "user", content: "hi" }], max_tokens: 8192, stream: true, thinking: { type: "adaptive", display: "summarized" }, }; assert.equal(detectApi(p), "anthropic"); }); test("detectApi: anthropic with system + max_tokens but no thinking", () => { const p = { model: "claude-haiku-4-5", messages: [{ role: "user", content: "hi" }], system: "You are helpful.", max_tokens: 8192, stream: true, }; assert.equal(detectApi(p), "anthropic"); }); test("detectApi: google generative ai payload", () => { const p = { model: "gemini-3-pro", contents: [{ role: "user", parts: [{ text: "hi" }] }], generationConfig: { thinkingConfig: { thinkingBudget: 4096 } }, }; assert.equal(detectApi(p), "google"); }); test("detectApi: garbage", () => { assert.equal(detectApi(null), "unknown"); assert.equal(detectApi(undefined), "unknown"); assert.equal(detectApi("string"), "unknown"); assert.equal(detectApi(42), "unknown"); assert.equal(detectApi({ random: "field" }), "unknown"); }); // (rungAt was inlined into chooseRung — its clamping behaviour is now // covered by the chooseRung tests below.) // ============================================================================ // accepted plan helpers // ============================================================================ test("normalizeAcceptedPlan: trims and nulls empty plans", () => { assert.equal(normalizeAcceptedPlan(" plan "), "plan"); assert.equal(normalizeAcceptedPlan("\n\t \n"), null); assert.equal(normalizeAcceptedPlan(undefined), null); }); test("contentText: keeps only text blocks from array content", () => { assert.equal( contentText([ { type: "text", text: "Line 1" }, { type: "thinking", thinking: "hidden" }, { type: "text", text: "Line 2" }, ]), "Line 1\nLine 2", ); assert.equal(contentText("plain string"), "plain string"); assert.equal(contentText(null), ""); }); test("extractLatestAssistantText: returns the latest assistant text content", () => { assert.equal( extractLatestAssistantText([ { role: "user", content: "hello" }, { role: "assistant", content: [{ type: "text", text: "First plan" }] }, { role: "assistant", content: [{ type: "text", text: " Final accepted plan " }] }, ]), "Final accepted plan", ); }); test("extractLatestAssistantText: skips assistants without visible text", () => { assert.equal( extractLatestAssistantText([ { role: "assistant", content: [{ type: "thinking", thinking: "hidden" }] }, { role: "assistant", content: [{ type: "toolCall", name: "read", arguments: { path: "a.ts" } }] }, ]), null, ); }); test("formatAcceptedPlanContext: wraps the exact accepted plan", () => { assert.equal( formatAcceptedPlanContext("Plan:\n1. Do the thing"), [ "The latest accepted implementation plan is preserved below in exact form.", "Treat it as the approved baseline unless newer user messages explicitly change direction.", "", "", "Plan:\n1. Do the thing", "", ].join("\n"), ); }); test("hasCustomMessage: detects matching custom messages only", () => { assert.equal(hasCustomMessage([{ role: "custom", customType: "accepted-plan" }], "accepted-plan"), true); assert.equal(hasCustomMessage([{ role: "custom", customType: "other" }], "accepted-plan"), false); assert.equal(hasCustomMessage([{ role: "assistant" }], "accepted-plan"), false); }); // ============================================================================ // applyRungToPayload — OpenAI Responses // ============================================================================ test("openai-responses: swaps model and reasoning.effort", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], stream: true, reasoning: { effort: "xhigh", summary: "auto" }, include: ["reasoning.encrypted_content"], }; const out = applyRungToPayload(original, { modelId: "gpt-5.4", thinking: "high", }) as Record; assert.equal(out.model, "gpt-5.4"); assert.deepEqual(out.reasoning, { effort: "high", summary: "auto" }); // preserved fields assert.deepEqual(out.input, original.input); assert.equal(out.stream, true); assert.deepEqual(out.include, original.include); }); test("openai-responses: adds reasoning when payload had none (with auto summary)", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], stream: true, }; const out = applyRungToPayload(original, { modelId: "gpt-5.4", thinking: "medium", }) as Record; assert.equal(out.model, "gpt-5.4"); assert.deepEqual(out.reasoning, { effort: "medium", summary: "auto" }); }); test("openai-responses: does not mutate input", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], reasoning: { effort: "xhigh", summary: "auto" }, }; const snapshot = JSON.parse(JSON.stringify(original)); applyRungToPayload(original, { modelId: "gpt-5.4", thinking: "low" }); assert.deepEqual(original, snapshot); }); // ============================================================================ // applyRungToPayload — OpenAI Completions // ============================================================================ test("openai-completions: swaps model and reasoning_effort", () => { const original = { model: "gpt-4o", messages: [{ role: "user", content: "hi" }], stream: true, reasoning_effort: "high", }; const out = applyRungToPayload(original, { modelId: "gpt-4o-mini", thinking: "low", }) as Record; assert.equal(out.model, "gpt-4o-mini"); assert.equal(out.reasoning_effort, "low"); assert.deepEqual(out.messages, original.messages); }); test("openai-completions: openrouter-style nested reasoning is also rewritten", () => { const original = { model: "openai/gpt-4o", messages: [{ role: "user", content: "hi" }], reasoning: { effort: "high" }, }; const out = applyRungToPayload(original, { modelId: "openai/gpt-4o-mini", thinking: "minimal", }) as Record; assert.equal(out.model, "openai/gpt-4o-mini"); assert.deepEqual(out.reasoning, { effort: "minimal" }); // reasoning_effort top-level is also set assert.equal(out.reasoning_effort, "minimal"); }); test("openai-completions: adds reasoning_effort if payload had none", () => { const original = { model: "gpt-4o", messages: [{ role: "user", content: "hi" }], }; const out = applyRungToPayload(original, { modelId: "gpt-4o-mini", thinking: "high", }) as Record; assert.equal(out.model, "gpt-4o-mini"); assert.equal(out.reasoning_effort, "high"); }); // ============================================================================ // applyRungToPayload — Anthropic adaptive thinking // ============================================================================ test("anthropic adaptive: swaps model and writes output_config.effort", () => { const original = { model: "claude-opus-4-7", messages: [{ role: "user", content: "hi" }], max_tokens: 8192, thinking: { type: "adaptive", display: "summarized" }, output_config: { effort: "xhigh" }, }; const out = applyRungToPayload(original, { modelId: "claude-sonnet-4-6", thinking: "high", }) as Record; assert.equal(out.model, "claude-sonnet-4-6"); assert.deepEqual(out.output_config, { effort: "high" }); // thinking object preserved assert.deepEqual(out.thinking, original.thinking); }); test("anthropic budget-based: model swaps but budget is left alone", () => { const original = { model: "claude-sonnet-4-0", messages: [{ role: "user", content: "hi" }], max_tokens: 8192, thinking: { type: "enabled", budget_tokens: 4096, display: "summarized" }, }; const out = applyRungToPayload(original, { modelId: "claude-haiku-4-5", thinking: "low", }) as Record; assert.equal(out.model, "claude-haiku-4-5"); // budget_tokens is left as-is — see comment in rewrite.ts assert.deepEqual(out.thinking, original.thinking); // no output_config injected assert.equal(out.output_config, undefined); }); // ============================================================================ // applyRungToPayload — graceful degradation // ============================================================================ test("unknown payload: model still rewritten", () => { const original = { model: "weird", whoknows: 1 }; const out = applyRungToPayload(original, { modelId: "weird-2", thinking: "medium", }) as Record; assert.equal(out.model, "weird-2"); assert.equal(out.whoknows, 1); }); test("non-object payload: passes through unchanged", () => { assert.equal(applyRungToPayload(null, { modelId: "x", thinking: "high" }), null); assert.equal(applyRungToPayload(undefined, { modelId: "x", thinking: "high" }), undefined); assert.equal(applyRungToPayload("oops", { modelId: "x", thinking: "high" }), "oops"); }); // ============================================================================ // createPromptCacheKey // ============================================================================ test("createPromptCacheKey: hashes username + cwd with visible prefix", () => { const a = createPromptCacheKey("pi-model-staging:", "alice", "/repo/a"); const b = createPromptCacheKey("pi-model-staging:", "alice", "/repo/a"); const c = createPromptCacheKey("pi-model-staging:", "alice", "/repo/b"); const d = createPromptCacheKey("pi-model-staging:", "bob", "/repo/a"); assert.equal(a, b); assert.match(a, /^pi-model-staging:[0-9a-f]{32}$/); assert.notEqual(a, c); assert.notEqual(a, d); assert.ok(!a.includes("alice")); assert.ok(!a.includes("/repo/a")); }); // ============================================================================ // applyOpenAIWebSearchToPayload — OpenAI Responses hosted web search // ============================================================================ test("web search: openai-responses appends web_search tool and sets tool_choice auto", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], stream: true, tools: [{ type: "function", name: "read", parameters: {} }], }; const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "medium", userLocation: { type: "approximate", country: "SE", timezone: "Europe/Stockholm" }, }) as Record; assert.equal(out.model, "gpt-5.5"); assert.equal(out.tool_choice, "auto"); assert.ok(Array.isArray(out.tools)); const tools = out.tools as Array>; assert.equal(tools.length, 2); assert.equal(tools[0]?.type, "function"); assert.equal(tools[1]?.type, "web_search"); assert.equal(tools[1]?.search_context_size, "medium"); assert.deepEqual(tools[1]?.user_location, { type: "approximate", country: "SE", timezone: "Europe/Stockholm" }); }); test("web search: preserves existing tool_choice", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], tool_choice: "required", }; const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "low" }) as Record< string, unknown >; assert.equal(out.tool_choice, "required"); }); test("web search: disabled option and non-object payloads pass through unchanged", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], }; assert.deepEqual(applyOpenAIWebSearchToPayload(original, { enabled: false, contextSize: "high" }), original); assert.equal(applyOpenAIWebSearchToPayload(null, { enabled: true, contextSize: "high" }), null); assert.equal(applyOpenAIWebSearchToPayload("oops", { enabled: true, contextSize: "high" }), "oops"); }); test("web search: does not duplicate if web_search already present", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], tools: [{ type: "web_search", search_context_size: "low" }], tool_choice: "auto", }; const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "high" }) as Record< string, unknown >; assert.ok(Array.isArray(out.tools)); assert.equal((out.tools as unknown[]).length, 1); }); test("web search: does not duplicate if web_search_preview already present", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], tools: [{ type: "web_search_preview" }], }; const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "high" }) as Record< string, unknown >; assert.ok(Array.isArray(out.tools)); assert.equal((out.tools as unknown[]).length, 1); }); test("web search: openai-completions payload unchanged", () => { const original = { model: "gpt-4o", messages: [{ role: "user", content: "hi" }], }; const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "high" }); assert.deepEqual(out, original); }); test("web search: contextSize off disables injection", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], }; const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "off" }); assert.deepEqual(out, original); }); test("web search: omits user_location when not provided", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], }; const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "low" }) as Record; const tools = (out.tools as Array> | undefined) ?? []; const ws = tools.find((t) => t?.type === "web_search"); assert.ok(ws); assert.equal(ws?.user_location, undefined); }); test("web search: omits user_location when provided but empty", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], }; const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "low", userLocation: { type: "approximate" }, }) as Record; const tools = (out.tools as Array> | undefined) ?? []; const ws = tools.find((t) => t?.type === "web_search"); assert.ok(ws); assert.equal(ws?.user_location, undefined); }); test("web search: does not mutate input", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], tools: [{ type: "function", name: "read", parameters: {} }], }; const snapshot = JSON.parse(JSON.stringify(original)); applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "low" }); assert.deepEqual(original, snapshot); }); // ============================================================================ // applyPromptCacheToPayload — OpenAI prompt caching (augmentation) // ============================================================================ test("prompt cache: openai-responses with pi-set key augments retention (the common path)", () => { // Realistic: pi has set prompt_cache_key from sessionId. Our extension // supplies the longer 24h retention on top. Both fields must end up in // the wire payload. const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], stream: true, prompt_cache_key: "pi-session-id-abc", }; const out = applyPromptCacheToPayload(original, { key: "should-not-override", retention: "24h" }) as Record< string, unknown >; assert.equal(out.prompt_cache_key, "pi-session-id-abc"); // pi's key preserved assert.equal(out.prompt_cache_retention, "24h"); // retention augmented // preserved assert.equal(out.model, "gpt-5.5"); assert.deepEqual(out.input, original.input); }); test("prompt cache: openai-completions adds prompt_cache_key only when retention omitted", () => { const original = { model: "gpt-4o", messages: [{ role: "user", content: "hi" }], stream: true, // pi has set the key but not the retention — partial signal, NOT // "disabled". Augment retention only. prompt_cache_key: "pi-session-existing", }; const out = applyPromptCacheToPayload(original, { key: "session-456" }) as Record; assert.equal(out.prompt_cache_key, "pi-session-existing"); assert.equal(out.prompt_cache_retention, undefined); }); test("prompt cache: respects pi-disabled caching (both fields undefined → pass through)", () => { // When pi has cacheRetention: "none" in settings, it leaves BOTH // prompt_cache_key and prompt_cache_retention undefined. We must not // silently re-enable caching by injecting our key — see the comment // in applyPromptCacheToPayload. const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], stream: true, }; const out = applyPromptCacheToPayload(original, { key: "should-not-be-injected", retention: "24h" }) as Record< string, unknown >; assert.equal(out.prompt_cache_key, undefined); assert.equal(out.prompt_cache_retention, undefined); }); test("prompt cache: pi set retention but not key → augment key only (still treats as enabled)", () => { // Edge: pi set retention but not key. That's not the "disabled" signal // (which is BOTH undefined), so we should still augment the key. const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], prompt_cache_retention: "in_memory", }; const out = applyPromptCacheToPayload(original, { key: "session-abc", retention: "24h" }) as Record< string, unknown >; assert.equal(out.prompt_cache_key, "session-abc"); assert.equal(out.prompt_cache_retention, "in_memory"); }); test("prompt cache: preserves existing provider fields (including null)", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], prompt_cache_key: null, prompt_cache_retention: "24h", }; const out = applyPromptCacheToPayload(original, { key: "session-should-not-override", retention: "24h" }) as Record< string, unknown >; assert.equal(out.prompt_cache_key, null); assert.equal(out.prompt_cache_retention, "24h"); }); test("prompt cache: non-openai payload passes through unchanged", () => { const original = { model: "gemini", contents: [{ role: "user", parts: [{ text: "hi" }] }], }; const out = applyPromptCacheToPayload(original, { key: "session-1", retention: "24h" }); assert.deepEqual(out, original); }); test("prompt cache: does not mutate input", () => { const original = { model: "gpt-5.5", input: [{ role: "user", content: "hi" }], stream: true, }; const snapshot = JSON.parse(JSON.stringify(original)); applyPromptCacheToPayload(original, { key: "session-123", retention: "24h" }); assert.deepEqual(original, snapshot); }); // ============================================================================ // Reasoning bumps — trigger detection // ============================================================================ test("startsWithShellCommand: matches only at start (after whitespace)", () => { assert.equal(startsWithShellCommand("npm test", "npm"), true); assert.equal(startsWithShellCommand(" npm run build", "npm"), true); assert.equal(startsWithShellCommand("npm", "npm"), true); assert.equal(startsWithShellCommand("pnpm test", "npm"), false); assert.equal(startsWithShellCommand("echo npm test", "npm"), false); assert.equal(startsWithShellCommand("cd x && npm test", "npm"), false); }); const BUMP_CFG: ReasoningBumpConfig = { bumpOnFailedBash: true, bumpOnFailedTool: true, bumpOnPackageManagerCommand: true, packageManagerCommands: ["npm", "pnpm", "yarn", "bun"], }; test("detectReasoningBump: bumps on failed bash", () => { assert.equal( detectReasoningBump({ toolName: "bash", input: { command: "npm test" }, isError: true }, BUMP_CFG), "failed bash command", ); }); test("detectReasoningBump: bumps on npm output even when successful", () => { assert.equal( detectReasoningBump({ toolName: "bash", input: { command: "npm test" }, isError: false }, BUMP_CFG), "npm command result", ); }); test("detectReasoningBump: bumps on other package managers", () => { assert.equal( detectReasoningBump({ toolName: "bash", input: { command: "pnpm test" }, isError: false }, BUMP_CFG), "pnpm command result", ); }); test("detectReasoningBump: no bump for non-matching bash", () => { assert.equal( detectReasoningBump({ toolName: "bash", input: { command: "git status" }, isError: false }, BUMP_CFG), null, ); }); test("detectReasoningBump: bumps on failed edit tool", () => { assert.equal( detectReasoningBump({ toolName: "edit", input: { path: "a.txt" }, isError: true }, BUMP_CFG), "failed edit tool", ); }); test("detectReasoningBump: no bump for successful edit tool", () => { assert.equal( detectReasoningBump({ toolName: "edit", input: { path: "a.txt" }, isError: false }, BUMP_CFG), null, ); }); test("detectReasoningBump: can disable non-bash failed-tool bump", () => { const cfg: ReasoningBumpConfig = { ...BUMP_CFG, bumpOnFailedTool: false }; assert.equal(detectReasoningBump({ toolName: "edit", input: { path: "a.txt" }, isError: true }, cfg), null); // Still bumps bash failures. assert.equal(detectReasoningBump({ toolName: "bash", input: { command: "false" }, isError: true }, cfg), "failed bash command"); }); test("isMaxOutputTokensLikeStop: treats length stops as max-output-like", () => { assert.equal(isMaxOutputTokensLikeStop({ stopReason: "length" }), true); }); test("isMaxOutputTokensLikeStop: detects proxy/OpenAI incomplete error text", () => { assert.equal( isMaxOutputTokensLikeStop({ stopReason: "error", errorMessage: "upstream_terminal_failure_preflush type=response.incomplete reason=max_output_tokens" }), true, ); assert.equal( isMaxOutputTokensLikeStop({ stopReason: "error", errorMessage: "Responses API returned response.incomplete before first token" }), true, ); assert.equal( isMaxOutputTokensLikeStop({ stopReason: "error", errorMessage: "incomplete: max_output_tokens" }), true, ); }); test("isMaxOutputTokensLikeStop: ignores non-max-output errors and stops", () => { assert.equal(isMaxOutputTokensLikeStop({ stopReason: "error", errorMessage: "network timeout" }), false); assert.equal(isMaxOutputTokensLikeStop({ stopReason: "stop", errorMessage: "max_output_tokens" }), false); assert.equal(isMaxOutputTokensLikeStop({ stopReason: "aborted", errorMessage: "response.incomplete" }), false); }); test("nextReasoningEffortForMaxOutput: downgrades xhigh to high", () => { assert.equal(nextReasoningEffortForMaxOutput("xhigh"), "high"); }); test("nextReasoningEffortForMaxOutput: downgrades high to medium", () => { assert.equal(nextReasoningEffortForMaxOutput("high"), "medium"); }); test("nextReasoningEffortForMaxOutput: medium and below request rung reset instead of another downgrade", () => { assert.equal(nextReasoningEffortForMaxOutput("medium"), null); assert.equal(nextReasoningEffortForMaxOutput("low"), null); assert.equal(nextReasoningEffortForMaxOutput("minimal"), null); assert.equal(nextReasoningEffortForMaxOutput(null), null); }); test("capThinkingLevel: never raises thinking, only caps it", () => { assert.equal(capThinkingLevel("xhigh", "high"), "high"); assert.equal(capThinkingLevel("high", "high"), "high"); assert.equal(capThinkingLevel("medium", "high"), "medium"); assert.equal(capThinkingLevel("low", "medium"), "low"); assert.equal(capThinkingLevel("minimal", "xhigh"), "minimal"); }); const STAGE_LADDER: Rung[] = [ { modelId: "r0", thinking: "high" }, { modelId: "r1", thinking: "high" }, { modelId: "r2", thinking: "high" }, { modelId: "r3", thinking: "high" }, ]; test("nextStage: advances by one, clamped to last rung", () => { assert.equal(nextStage(0, STAGE_LADDER), 1); assert.equal(nextStage(2, STAGE_LADDER), 3); assert.equal(nextStage(3, STAGE_LADDER), 3); // clamped assert.equal(nextStage(99, STAGE_LADDER), 3); // far past end }); test("nextStage: post-bump caller passes the bump index — advance from there", () => { // Caller-side composition: nextStage(activeBumpIndex ?? stage, ladder) // A bump on [1] should resume at [2] regardless of where stage was. assert.equal(nextStage(1, STAGE_LADDER), 2); }); test("nextStage: ladder size 1 stays at 0", () => { const ladder: Rung[] = [{ modelId: "only", thinking: "high" }]; assert.equal(nextStage(0, ladder), 0); }); test("nextStage: empty ladder returns 0", () => { assert.equal(nextStage(5, []), 0); }); // ============================================================================ // chooseRung — mode/stage dispatch // ============================================================================ const LADDER: Rung[] = [ { modelId: "quick", thinking: "xhigh" }, // [0] plan + user-facing { modelId: "model-a", thinking: "xhigh" }, // [1] first autonomous step { modelId: "model-a", thinking: "high" }, // [2] { modelId: "model-b", thinking: "high" }, // [3] ]; test("chooseRung: idle → null", () => { assert.equal(chooseRung("idle", 0, LADDER), null); assert.equal(chooseRung("idle", 99, LADDER), null); }); test("chooseRung: empty ladder → null", () => { assert.equal(chooseRung("planning", 0, []), null); assert.equal(chooseRung("implementing", 5, []), null); }); test("chooseRung: planning always returns LADDER[0] regardless of stage", () => { assert.deepEqual(chooseRung("planning", 0, LADDER), LADDER[0]); assert.deepEqual(chooseRung("planning", 1, LADDER), LADDER[0]); assert.deepEqual(chooseRung("planning", 99, LADDER), LADDER[0]); }); test("chooseRung: implementing returns LADDER[stage]", () => { assert.deepEqual(chooseRung("implementing", 0, LADDER), LADDER[0]); assert.deepEqual(chooseRung("implementing", 1, LADDER), LADDER[1]); assert.deepEqual(chooseRung("implementing", 2, LADDER), LADDER[2]); assert.deepEqual(chooseRung("implementing", 3, LADDER), LADDER[3]); }); test("chooseRung: implementing past the end clamps to last", () => { assert.deepEqual(chooseRung("implementing", 4, LADDER), LADDER[3]); assert.deepEqual(chooseRung("implementing", 99, LADDER), LADDER[3]); }); test("chooseRung: implementing with negative stage clamps to first", () => { assert.deepEqual(chooseRung("implementing", -1, LADDER), LADDER[0]); }); test("chooseContextSafeRungIndex: keeps base rung when context is unknown", () => { assert.deepEqual( chooseContextSafeRungIndex(2, LADDER, [{ contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 375000 }], null, 16384), { index: 2, fallback: false }, ); }); test("chooseContextSafeRungIndex: keeps base rung when it still fits", () => { assert.deepEqual( chooseContextSafeRungIndex(1, LADDER, [{ contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 375000 }, { contextWindow: 375000 }], 120000, 16384), { index: 1, fallback: false }, ); }); test("chooseContextSafeRungIndex: falls forward to a later larger-context rung", () => { assert.deepEqual( chooseContextSafeRungIndex(1, LADDER, [{ contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 375000 }], 299081, 16384), { index: 3, fallback: true }, ); }); test("chooseContextSafeRungIndex: unknown base window does not guess", () => { assert.deepEqual( chooseContextSafeRungIndex(1, LADDER, [{ contextWindow: 272000 }, {}, { contextWindow: 375000 }, { contextWindow: 375000 }], 299081, 16384), { index: 1, fallback: false }, ); }); test("chooseContextSafeRungIndex: no later rung fits → keep base rung", () => { assert.deepEqual( chooseContextSafeRungIndex(1, LADDER, [{ contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 300000 }, { contextWindow: 310000 }], 299081, 16384), { index: 1, fallback: false }, ); }); // ============================================================================ // End-to-end lifecycle: simulate the full plan→implement→follow-up flow and // confirm each LLM call gets the right rung's model + effort. // // The state machine driving stage transitions lives in index.ts, but the // rules are simple enough to encode inline here: // // /plan mode=planning, stage=0 // turn_end during implementing stage = min(stage+1, len-1) // accept mode=implementing, stage=1 // agent_end during implementing stage=0 // ============================================================================ test("end-to-end: full lifecycle uses correct rung at every LLM call", () => { const ladder: Rung[] = [ { modelId: "gpt-5.5:quick", thinking: "xhigh", webSearchContextSize: "high" }, { modelId: "gpt-5.5", thinking: "xhigh", webSearchContextSize: "high" }, { modelId: "gpt-5.5", thinking: "high", webSearchContextSize: "medium" }, { modelId: "gpt-5.4", thinking: "high", webSearchContextSize: "low" }, ]; const basePayload = () => ({ model: "ignored", input: [{ role: "user", content: "x" }], stream: true, reasoning: { effort: "ignored", summary: "auto" }, }); const seen: string[] = []; const seenSearch: Array = []; function fire(mode: "planning" | "implementing", stage: number) { const rung = chooseRung(mode, stage, ladder); assert.ok(rung); const out = applyRungToPayload(basePayload(), rung) as Record & { model: string; reasoning: { effort: string }; }; seen.push(`${out.model}:${out.reasoning.effort}`); // Web search context size follows the rung's setting. const withSearch = applyOpenAIWebSearchToPayload(out, { enabled: true, contextSize: rung.webSearchContextSize, }) as Record; const tools = (withSearch.tools as Array> | undefined) ?? []; const ws = tools.find((t) => t?.type === "web_search"); seenSearch.push(ws ? String(ws.search_context_size) : undefined); return rung; } // /plan → mode=planning, stage=0 let mode: "planning" | "implementing" = "planning"; let stage = 0; // Plan run with 3 turns (LLM does some reads, asks for clarification). fire(mode, stage); // turn 1 fire(mode, stage); // turn 2 fire(mode, stage); // turn 3 // agent_end during planning fires the dialog (no stage mutation here). // Accept → mode=implementing, stage=1, "Please start implementation." // auto-prompt fires. mode = "implementing"; stage = 1; // Implementing run #1: 4 turns. stage advances at each turn_end. fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 1 fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 2 fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 3 fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 4 (clamped) // agent_end during implementing → reset stage to 0. stage = 0; // User follow-up "also do X". Run #2: 3 turns. fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 1 fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 2 fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 3 // agent_end → reset. stage = 0; // One more follow-up, single turn. fire(mode, stage); assert.deepEqual(seen, [ // 3 plan turns: all LADDER[0] "gpt-5.5:quick:xhigh", "gpt-5.5:quick:xhigh", "gpt-5.5:quick:xhigh", // Start implementation run: starts at LADDER[1], steps to [3], then clamps. "gpt-5.5:xhigh", "gpt-5.5:high", "gpt-5.4:high", "gpt-5.4:high", // clamped // Follow-up #1: starts at LADDER[0] (user-facing), steps down. "gpt-5.5:quick:xhigh", "gpt-5.5:xhigh", "gpt-5.5:high", // Follow-up #2 single turn: LADDER[0]. "gpt-5.5:quick:xhigh", ]); assert.deepEqual(seenSearch, [ // Plan run: LADDER[0] repeated. "high", "high", "high", // Start implementation run: [1] → [2] → [3] → [3] clamped. "high", "medium", "low", "low", // Follow-up #1: [0] → [1] → [2]. "high", "high", "medium", // Follow-up #2: [0]. "high", ]); }); // ============================================================================ // End-to-end with reasoning bump: simulate an implementing run where a bash // failure mid-stream triggers a one-shot bump, then confirm the bumped turn // uses LADDER[1] AND the next normal turn resumes at LADDER[2] (not back at // the pre-bump stage cursor). // // State machine (subset relevant here): // tool_result trigger pendingBump = { rungIndex: BUMP_RUNG_INDEX } // turn_start activeBump = pendingBump; pendingBump = null // before_provider_request rung = activeBump ? LADDER[bump] : LADDER[stage] // turn_end stage = nextStage(activeBump?.rungIndex ?? stage, ladder) // activeBump = null // ============================================================================ test("end-to-end with bump: bumped turn uses LADDER[1], next turn resumes at LADDER[2]", () => { const ladder: Rung[] = [ { modelId: "gpt-5.5:quick", thinking: "xhigh" }, // [0] never reached here { modelId: "gpt-5.5", thinking: "xhigh" }, // [1] post-accept + bump target { modelId: "gpt-5.5", thinking: "high" }, // [2] post-bump resume { modelId: "gpt-5.4", thinking: "high" }, // [3] ]; const BUMP_INDEX = Math.min(1, ladder.length - 1); const basePayload = () => ({ model: "ignored", input: [{ role: "user", content: "x" }], stream: true, reasoning: { effort: "ignored", summary: "auto" }, }); const seen: string[] = []; let stage = 1; // accept just happened let pendingBump: { rungIndex: number } | null = null; let activeBump: { rungIndex: number } | null = null; function fireTurn() { // turn_start: arm the bump if queued. if (pendingBump) { activeBump = pendingBump; pendingBump = null; } // before_provider_request: bump wins over normal stage. const rung = activeBump ? ladder[activeBump.rungIndex] : chooseRung("implementing", stage, ladder); assert.ok(rung); const out = applyRungToPayload(basePayload(), rung) as { model: string; reasoning: { effort: string }; }; seen.push(`${out.model}:${out.reasoning.effort}`); // turn_end: advance stage from bump target if there was one. const activeBumpIndex = activeBump?.rungIndex; activeBump = null; stage = nextStage(activeBumpIndex ?? stage, ladder); } // Turn 1: normal post-accept turn. Should use LADDER[1]. fireTurn(); // Between turns 2 and 3 the agent ran `npm test` and the result fired a // bump trigger. Queue it before turn 2 starts. pendingBump = { rungIndex: BUMP_INDEX }; // Turn 2: bump active. Should use LADDER[1] (the bump target). fireTurn(); // Turn 3: no new bump. Stage was advanced from bump index (1) → 2. // Should use LADDER[2]. fireTurn(); // Turn 4: continues stepping down to LADDER[3]. fireTurn(); assert.deepEqual(seen, [ "gpt-5.5:xhigh", // turn 1: LADDER[1] (post-accept default) "gpt-5.5:xhigh", // turn 2: LADDER[1] (bump target — same value here, but bump path used) "gpt-5.5:high", // turn 3: LADDER[2] (resumed AFTER the bump rung, not from pre-bump stage) "gpt-5.4:high", // turn 4: LADDER[3] ]); }); // ============================================================================ // End-to-end with max-output rescue: simulate repeated implementation runs that // terminate with max-output-like stops, confirming the extension retries the // same rung at lower effort (xhigh → high → medium) before falling back to a // bump/reset at LADDER[1]. Non-max-output errors should not enter this path. // ============================================================================ function createMaxOutputRescueHarness(ladder: Rung[], initialStage: number) { const MAX_CONSECUTIVE_MAX_OUTPUT_RESCUES = 3; const BUMP_INDEX = Math.min(1, ladder.length - 1); const basePayload = () => ({ model: "ignored", input: [{ role: "user", content: "x" }], stream: true, reasoning: { effort: "ignored", summary: "auto" }, }); const seen: string[] = []; let stage = initialStage; let pendingBump: { rungIndex: number; reason: string } | null = null; let activeBump: { rungIndex: number; reason: string } | null = null; let pendingMaxOutputRescue: { rungIndex: number; from: Rung["thinking"]; to: Rung["thinking"] } | null = null; let activeMaxOutputRescue: { rungIndex: number; from: Rung["thinking"]; to: Rung["thinking"] } | null = null; let lastProviderRequest: { rungIndex: number; thinking: Rung["thinking"] } | null = null; let consecutiveMaxOutputRescues = 0; function fireAttempt(stopInfo: { stopReason?: string; errorMessage?: string | null }) { // turn_start lastProviderRequest = null; if (!pendingBump) { if (pendingMaxOutputRescue) { activeMaxOutputRescue = pendingMaxOutputRescue; pendingMaxOutputRescue = null; } } else { activeBump = pendingBump; pendingBump = null; pendingMaxOutputRescue = null; activeMaxOutputRescue = null; } // before_provider_request const preferredIndex = activeBump?.rungIndex ?? activeMaxOutputRescue?.rungIndex ?? stage; const rung = ladder[preferredIndex]; assert.ok(rung); const rescue = pendingMaxOutputRescue ?? activeMaxOutputRescue; const appliedThinking = rescue ? capThinkingLevel(rung.thinking, rescue.to) : rung.thinking; const out = applyRungToPayload(basePayload(), { ...rung, thinking: appliedThinking }) as { model: string; reasoning: { effort: string }; }; seen.push(`${out.model}:${out.reasoning.effort}`); lastProviderRequest = { rungIndex: preferredIndex, thinking: appliedThinking }; // turn_end stage = nextStage(lastProviderRequest.rungIndex, ladder); activeBump = null; activeMaxOutputRescue = null; // agent_end const maxOutputLike = isMaxOutputTokensLikeStop(stopInfo); pendingBump = null; activeBump = null; if ( maxOutputLike && consecutiveMaxOutputRescues < MAX_CONSECUTIVE_MAX_OUTPUT_RESCUES && lastProviderRequest ) { const nextEffort = nextReasoningEffortForMaxOutput(lastProviderRequest.thinking); if (nextEffort) { pendingMaxOutputRescue = { rungIndex: lastProviderRequest.rungIndex, from: lastProviderRequest.thinking, to: nextEffort, }; consecutiveMaxOutputRescues++; return { kind: "rescue" as const, to: nextEffort }; } pendingMaxOutputRescue = null; activeMaxOutputRescue = null; pendingBump = { rungIndex: BUMP_INDEX, reason: "max output tokens exhausted" }; consecutiveMaxOutputRescues++; return { kind: "bump" as const, rungIndex: BUMP_INDEX }; } pendingMaxOutputRescue = null; activeMaxOutputRescue = null; lastProviderRequest = null; consecutiveMaxOutputRescues = 0; stage = 0; return { kind: "reset" as const }; } return { seen, fireAttempt, getStage: () => stage, getPendingBump: () => pendingBump, getPendingMaxOutputRescue: () => pendingMaxOutputRescue, }; } test("end-to-end with max-output rescue: same rung steps xhigh → high → medium, then bumps to LADDER[1]", () => { const ladder: Rung[] = [ { modelId: "gpt-5.5:quick", thinking: "xhigh" }, { modelId: "gpt-5.4", thinking: "xhigh" }, { modelId: "gpt-5.4", thinking: "high" }, { modelId: "gpt-5.4", thinking: "medium" }, { modelId: "gpt-5.4-mini", thinking: "xhigh" }, ]; const harness = createMaxOutputRescueHarness(ladder, 4); assert.deepEqual(harness.fireAttempt({ stopReason: "length" }), { kind: "rescue", to: "high" }); assert.deepEqual(harness.getPendingMaxOutputRescue(), { rungIndex: 4, from: "xhigh", to: "high" }); assert.deepEqual(harness.fireAttempt({ stopReason: "length" }), { kind: "rescue", to: "medium" }); assert.deepEqual(harness.getPendingMaxOutputRescue(), { rungIndex: 4, from: "high", to: "medium" }); assert.deepEqual(harness.fireAttempt({ stopReason: "length" }), { kind: "bump", rungIndex: 1 }); assert.deepEqual(harness.getPendingBump(), { rungIndex: 1, reason: "max output tokens exhausted" }); // Next autonomous retry uses the queued bump/reset rung. assert.deepEqual(harness.fireAttempt({ stopReason: "stop" }), { kind: "reset" }); assert.deepEqual(harness.seen, [ "gpt-5.4-mini:xhigh", "gpt-5.4-mini:high", "gpt-5.4-mini:medium", "gpt-5.4:xhigh", ]); assert.equal(harness.getStage(), 0); }); test("end-to-end with max-output rescue: non-max-output errors do not trigger the rescue path", () => { const ladder: Rung[] = [ { modelId: "gpt-5.5:quick", thinking: "xhigh" }, { modelId: "gpt-5.4", thinking: "xhigh" }, { modelId: "gpt-5.4", thinking: "high" }, { modelId: "gpt-5.4", thinking: "medium" }, { modelId: "gpt-5.4-mini", thinking: "xhigh" }, ]; const harness = createMaxOutputRescueHarness(ladder, 4); assert.deepEqual(harness.fireAttempt({ stopReason: "error", errorMessage: "network timeout" }), { kind: "reset" }); assert.equal(harness.getPendingMaxOutputRescue(), null); assert.equal(harness.getPendingBump(), null); assert.deepEqual(harness.seen, ["gpt-5.4-mini:xhigh"]); assert.equal(harness.getStage(), 0); });