/**
* Tests for the per-turn payload rewriter.
*
* Run with: node --test --experimental-strip-types .pi/extensions/plan-stepdown/rewrite.test.ts
*
* Fixtures mirror real wire payloads from
* pi/packages/ai/src/providers/openai-responses.ts (buildParams)
* pi/packages/ai/src/providers/openai-completions.ts
* pi/packages/ai/src/providers/anthropic.ts (buildParams)
* pi/packages/ai/src/providers/google.ts
*/
import test from "node:test";
import assert from "node:assert/strict";
import {
applyOpenAIWebSearchToPayload,
applyPromptCacheToPayload,
applyRungToPayload,
capThinkingLevel,
chooseContextSafeRungIndex,
chooseRung,
contentText,
createPromptCacheKey,
detectApi,
detectReasoningBump,
extractLatestAssistantText,
formatAcceptedPlanContext,
hasCustomMessage,
isMaxOutputTokensLikeStop,
nextReasoningEffortForMaxOutput,
nextStage,
normalizeAcceptedPlan,
startsWithShellCommand,
type ReasoningBumpConfig,
type Rung,
} from "./rewrite.ts";
// ============================================================================
// detectApi
// ============================================================================
test("detectApi: openai-responses payload", () => {
const p = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
stream: true,
reasoning: { effort: "high", summary: "auto" },
store: false,
};
assert.equal(detectApi(p), "openai-responses");
});
test("detectApi: openai-completions payload", () => {
const p = {
model: "gpt-4o",
messages: [{ role: "user", content: "hi" }],
stream: true,
reasoning_effort: "high",
};
assert.equal(detectApi(p), "openai-completions");
});
test("detectApi: anthropic with thinking", () => {
const p = {
model: "claude-sonnet-4-6",
messages: [{ role: "user", content: "hi" }],
max_tokens: 8192,
stream: true,
thinking: { type: "adaptive", display: "summarized" },
};
assert.equal(detectApi(p), "anthropic");
});
test("detectApi: anthropic with system + max_tokens but no thinking", () => {
const p = {
model: "claude-haiku-4-5",
messages: [{ role: "user", content: "hi" }],
system: "You are helpful.",
max_tokens: 8192,
stream: true,
};
assert.equal(detectApi(p), "anthropic");
});
test("detectApi: google generative ai payload", () => {
const p = {
model: "gemini-3-pro",
contents: [{ role: "user", parts: [{ text: "hi" }] }],
generationConfig: { thinkingConfig: { thinkingBudget: 4096 } },
};
assert.equal(detectApi(p), "google");
});
test("detectApi: garbage", () => {
assert.equal(detectApi(null), "unknown");
assert.equal(detectApi(undefined), "unknown");
assert.equal(detectApi("string"), "unknown");
assert.equal(detectApi(42), "unknown");
assert.equal(detectApi({ random: "field" }), "unknown");
});
// (rungAt was inlined into chooseRung — its clamping behaviour is now
// covered by the chooseRung tests below.)
// ============================================================================
// accepted plan helpers
// ============================================================================
test("normalizeAcceptedPlan: trims and nulls empty plans", () => {
assert.equal(normalizeAcceptedPlan(" plan "), "plan");
assert.equal(normalizeAcceptedPlan("\n\t \n"), null);
assert.equal(normalizeAcceptedPlan(undefined), null);
});
test("contentText: keeps only text blocks from array content", () => {
assert.equal(
contentText([
{ type: "text", text: "Line 1" },
{ type: "thinking", thinking: "hidden" },
{ type: "text", text: "Line 2" },
]),
"Line 1\nLine 2",
);
assert.equal(contentText("plain string"), "plain string");
assert.equal(contentText(null), "");
});
test("extractLatestAssistantText: returns the latest assistant text content", () => {
assert.equal(
extractLatestAssistantText([
{ role: "user", content: "hello" },
{ role: "assistant", content: [{ type: "text", text: "First plan" }] },
{ role: "assistant", content: [{ type: "text", text: " Final accepted plan " }] },
]),
"Final accepted plan",
);
});
test("extractLatestAssistantText: skips assistants without visible text", () => {
assert.equal(
extractLatestAssistantText([
{ role: "assistant", content: [{ type: "thinking", thinking: "hidden" }] },
{ role: "assistant", content: [{ type: "toolCall", name: "read", arguments: { path: "a.ts" } }] },
]),
null,
);
});
test("formatAcceptedPlanContext: wraps the exact accepted plan", () => {
assert.equal(
formatAcceptedPlanContext("Plan:\n1. Do the thing"),
[
"The latest accepted implementation plan is preserved below in exact form.",
"Treat it as the approved baseline unless newer user messages explicitly change direction.",
"",
"",
"Plan:\n1. Do the thing",
"",
].join("\n"),
);
});
test("hasCustomMessage: detects matching custom messages only", () => {
assert.equal(hasCustomMessage([{ role: "custom", customType: "accepted-plan" }], "accepted-plan"), true);
assert.equal(hasCustomMessage([{ role: "custom", customType: "other" }], "accepted-plan"), false);
assert.equal(hasCustomMessage([{ role: "assistant" }], "accepted-plan"), false);
});
// ============================================================================
// applyRungToPayload — OpenAI Responses
// ============================================================================
test("openai-responses: swaps model and reasoning.effort", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
stream: true,
reasoning: { effort: "xhigh", summary: "auto" },
include: ["reasoning.encrypted_content"],
};
const out = applyRungToPayload(original, {
modelId: "gpt-5.4",
thinking: "high",
}) as Record;
assert.equal(out.model, "gpt-5.4");
assert.deepEqual(out.reasoning, { effort: "high", summary: "auto" });
// preserved fields
assert.deepEqual(out.input, original.input);
assert.equal(out.stream, true);
assert.deepEqual(out.include, original.include);
});
test("openai-responses: adds reasoning when payload had none (with auto summary)", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
stream: true,
};
const out = applyRungToPayload(original, {
modelId: "gpt-5.4",
thinking: "medium",
}) as Record;
assert.equal(out.model, "gpt-5.4");
assert.deepEqual(out.reasoning, { effort: "medium", summary: "auto" });
});
test("openai-responses: does not mutate input", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
reasoning: { effort: "xhigh", summary: "auto" },
};
const snapshot = JSON.parse(JSON.stringify(original));
applyRungToPayload(original, { modelId: "gpt-5.4", thinking: "low" });
assert.deepEqual(original, snapshot);
});
// ============================================================================
// applyRungToPayload — OpenAI Completions
// ============================================================================
test("openai-completions: swaps model and reasoning_effort", () => {
const original = {
model: "gpt-4o",
messages: [{ role: "user", content: "hi" }],
stream: true,
reasoning_effort: "high",
};
const out = applyRungToPayload(original, {
modelId: "gpt-4o-mini",
thinking: "low",
}) as Record;
assert.equal(out.model, "gpt-4o-mini");
assert.equal(out.reasoning_effort, "low");
assert.deepEqual(out.messages, original.messages);
});
test("openai-completions: openrouter-style nested reasoning is also rewritten", () => {
const original = {
model: "openai/gpt-4o",
messages: [{ role: "user", content: "hi" }],
reasoning: { effort: "high" },
};
const out = applyRungToPayload(original, {
modelId: "openai/gpt-4o-mini",
thinking: "minimal",
}) as Record;
assert.equal(out.model, "openai/gpt-4o-mini");
assert.deepEqual(out.reasoning, { effort: "minimal" });
// reasoning_effort top-level is also set
assert.equal(out.reasoning_effort, "minimal");
});
test("openai-completions: adds reasoning_effort if payload had none", () => {
const original = {
model: "gpt-4o",
messages: [{ role: "user", content: "hi" }],
};
const out = applyRungToPayload(original, {
modelId: "gpt-4o-mini",
thinking: "high",
}) as Record;
assert.equal(out.model, "gpt-4o-mini");
assert.equal(out.reasoning_effort, "high");
});
// ============================================================================
// applyRungToPayload — Anthropic adaptive thinking
// ============================================================================
test("anthropic adaptive: swaps model and writes output_config.effort", () => {
const original = {
model: "claude-opus-4-7",
messages: [{ role: "user", content: "hi" }],
max_tokens: 8192,
thinking: { type: "adaptive", display: "summarized" },
output_config: { effort: "xhigh" },
};
const out = applyRungToPayload(original, {
modelId: "claude-sonnet-4-6",
thinking: "high",
}) as Record;
assert.equal(out.model, "claude-sonnet-4-6");
assert.deepEqual(out.output_config, { effort: "high" });
// thinking object preserved
assert.deepEqual(out.thinking, original.thinking);
});
test("anthropic budget-based: model swaps but budget is left alone", () => {
const original = {
model: "claude-sonnet-4-0",
messages: [{ role: "user", content: "hi" }],
max_tokens: 8192,
thinking: { type: "enabled", budget_tokens: 4096, display: "summarized" },
};
const out = applyRungToPayload(original, {
modelId: "claude-haiku-4-5",
thinking: "low",
}) as Record;
assert.equal(out.model, "claude-haiku-4-5");
// budget_tokens is left as-is — see comment in rewrite.ts
assert.deepEqual(out.thinking, original.thinking);
// no output_config injected
assert.equal(out.output_config, undefined);
});
// ============================================================================
// applyRungToPayload — graceful degradation
// ============================================================================
test("unknown payload: model still rewritten", () => {
const original = { model: "weird", whoknows: 1 };
const out = applyRungToPayload(original, {
modelId: "weird-2",
thinking: "medium",
}) as Record;
assert.equal(out.model, "weird-2");
assert.equal(out.whoknows, 1);
});
test("non-object payload: passes through unchanged", () => {
assert.equal(applyRungToPayload(null, { modelId: "x", thinking: "high" }), null);
assert.equal(applyRungToPayload(undefined, { modelId: "x", thinking: "high" }), undefined);
assert.equal(applyRungToPayload("oops", { modelId: "x", thinking: "high" }), "oops");
});
// ============================================================================
// createPromptCacheKey
// ============================================================================
test("createPromptCacheKey: hashes username + cwd with visible prefix", () => {
const a = createPromptCacheKey("pi-model-staging:", "alice", "/repo/a");
const b = createPromptCacheKey("pi-model-staging:", "alice", "/repo/a");
const c = createPromptCacheKey("pi-model-staging:", "alice", "/repo/b");
const d = createPromptCacheKey("pi-model-staging:", "bob", "/repo/a");
assert.equal(a, b);
assert.match(a, /^pi-model-staging:[0-9a-f]{32}$/);
assert.notEqual(a, c);
assert.notEqual(a, d);
assert.ok(!a.includes("alice"));
assert.ok(!a.includes("/repo/a"));
});
// ============================================================================
// applyOpenAIWebSearchToPayload — OpenAI Responses hosted web search
// ============================================================================
test("web search: openai-responses appends web_search tool and sets tool_choice auto", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
stream: true,
tools: [{ type: "function", name: "read", parameters: {} }],
};
const out = applyOpenAIWebSearchToPayload(original, {
enabled: true,
contextSize: "medium",
userLocation: { type: "approximate", country: "SE", timezone: "Europe/Stockholm" },
}) as Record;
assert.equal(out.model, "gpt-5.5");
assert.equal(out.tool_choice, "auto");
assert.ok(Array.isArray(out.tools));
const tools = out.tools as Array>;
assert.equal(tools.length, 2);
assert.equal(tools[0]?.type, "function");
assert.equal(tools[1]?.type, "web_search");
assert.equal(tools[1]?.search_context_size, "medium");
assert.deepEqual(tools[1]?.user_location, { type: "approximate", country: "SE", timezone: "Europe/Stockholm" });
});
test("web search: preserves existing tool_choice", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
tool_choice: "required",
};
const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "low" }) as Record<
string,
unknown
>;
assert.equal(out.tool_choice, "required");
});
test("web search: disabled option and non-object payloads pass through unchanged", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
};
assert.deepEqual(applyOpenAIWebSearchToPayload(original, { enabled: false, contextSize: "high" }), original);
assert.equal(applyOpenAIWebSearchToPayload(null, { enabled: true, contextSize: "high" }), null);
assert.equal(applyOpenAIWebSearchToPayload("oops", { enabled: true, contextSize: "high" }), "oops");
});
test("web search: does not duplicate if web_search already present", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
tools: [{ type: "web_search", search_context_size: "low" }],
tool_choice: "auto",
};
const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "high" }) as Record<
string,
unknown
>;
assert.ok(Array.isArray(out.tools));
assert.equal((out.tools as unknown[]).length, 1);
});
test("web search: does not duplicate if web_search_preview already present", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
tools: [{ type: "web_search_preview" }],
};
const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "high" }) as Record<
string,
unknown
>;
assert.ok(Array.isArray(out.tools));
assert.equal((out.tools as unknown[]).length, 1);
});
test("web search: openai-completions payload unchanged", () => {
const original = {
model: "gpt-4o",
messages: [{ role: "user", content: "hi" }],
};
const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "high" });
assert.deepEqual(out, original);
});
test("web search: contextSize off disables injection", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
};
const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "off" });
assert.deepEqual(out, original);
});
test("web search: omits user_location when not provided", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
};
const out = applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "low" }) as Record;
const tools = (out.tools as Array> | undefined) ?? [];
const ws = tools.find((t) => t?.type === "web_search");
assert.ok(ws);
assert.equal(ws?.user_location, undefined);
});
test("web search: omits user_location when provided but empty", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
};
const out = applyOpenAIWebSearchToPayload(original, {
enabled: true,
contextSize: "low",
userLocation: { type: "approximate" },
}) as Record;
const tools = (out.tools as Array> | undefined) ?? [];
const ws = tools.find((t) => t?.type === "web_search");
assert.ok(ws);
assert.equal(ws?.user_location, undefined);
});
test("web search: does not mutate input", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
tools: [{ type: "function", name: "read", parameters: {} }],
};
const snapshot = JSON.parse(JSON.stringify(original));
applyOpenAIWebSearchToPayload(original, { enabled: true, contextSize: "low" });
assert.deepEqual(original, snapshot);
});
// ============================================================================
// applyPromptCacheToPayload — OpenAI prompt caching (augmentation)
// ============================================================================
test("prompt cache: openai-responses with pi-set key augments retention (the common path)", () => {
// Realistic: pi has set prompt_cache_key from sessionId. Our extension
// supplies the longer 24h retention on top. Both fields must end up in
// the wire payload.
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
stream: true,
prompt_cache_key: "pi-session-id-abc",
};
const out = applyPromptCacheToPayload(original, { key: "should-not-override", retention: "24h" }) as Record<
string,
unknown
>;
assert.equal(out.prompt_cache_key, "pi-session-id-abc"); // pi's key preserved
assert.equal(out.prompt_cache_retention, "24h"); // retention augmented
// preserved
assert.equal(out.model, "gpt-5.5");
assert.deepEqual(out.input, original.input);
});
test("prompt cache: openai-completions adds prompt_cache_key only when retention omitted", () => {
const original = {
model: "gpt-4o",
messages: [{ role: "user", content: "hi" }],
stream: true,
// pi has set the key but not the retention — partial signal, NOT
// "disabled". Augment retention only.
prompt_cache_key: "pi-session-existing",
};
const out = applyPromptCacheToPayload(original, { key: "session-456" }) as Record;
assert.equal(out.prompt_cache_key, "pi-session-existing");
assert.equal(out.prompt_cache_retention, undefined);
});
test("prompt cache: respects pi-disabled caching (both fields undefined → pass through)", () => {
// When pi has cacheRetention: "none" in settings, it leaves BOTH
// prompt_cache_key and prompt_cache_retention undefined. We must not
// silently re-enable caching by injecting our key — see the comment
// in applyPromptCacheToPayload.
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
stream: true,
};
const out = applyPromptCacheToPayload(original, { key: "should-not-be-injected", retention: "24h" }) as Record<
string,
unknown
>;
assert.equal(out.prompt_cache_key, undefined);
assert.equal(out.prompt_cache_retention, undefined);
});
test("prompt cache: pi set retention but not key → augment key only (still treats as enabled)", () => {
// Edge: pi set retention but not key. That's not the "disabled" signal
// (which is BOTH undefined), so we should still augment the key.
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
prompt_cache_retention: "in_memory",
};
const out = applyPromptCacheToPayload(original, { key: "session-abc", retention: "24h" }) as Record<
string,
unknown
>;
assert.equal(out.prompt_cache_key, "session-abc");
assert.equal(out.prompt_cache_retention, "in_memory");
});
test("prompt cache: preserves existing provider fields (including null)", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
prompt_cache_key: null,
prompt_cache_retention: "24h",
};
const out = applyPromptCacheToPayload(original, { key: "session-should-not-override", retention: "24h" }) as Record<
string,
unknown
>;
assert.equal(out.prompt_cache_key, null);
assert.equal(out.prompt_cache_retention, "24h");
});
test("prompt cache: non-openai payload passes through unchanged", () => {
const original = {
model: "gemini",
contents: [{ role: "user", parts: [{ text: "hi" }] }],
};
const out = applyPromptCacheToPayload(original, { key: "session-1", retention: "24h" });
assert.deepEqual(out, original);
});
test("prompt cache: does not mutate input", () => {
const original = {
model: "gpt-5.5",
input: [{ role: "user", content: "hi" }],
stream: true,
};
const snapshot = JSON.parse(JSON.stringify(original));
applyPromptCacheToPayload(original, { key: "session-123", retention: "24h" });
assert.deepEqual(original, snapshot);
});
// ============================================================================
// Reasoning bumps — trigger detection
// ============================================================================
test("startsWithShellCommand: matches only at start (after whitespace)", () => {
assert.equal(startsWithShellCommand("npm test", "npm"), true);
assert.equal(startsWithShellCommand(" npm run build", "npm"), true);
assert.equal(startsWithShellCommand("npm", "npm"), true);
assert.equal(startsWithShellCommand("pnpm test", "npm"), false);
assert.equal(startsWithShellCommand("echo npm test", "npm"), false);
assert.equal(startsWithShellCommand("cd x && npm test", "npm"), false);
});
const BUMP_CFG: ReasoningBumpConfig = {
bumpOnFailedBash: true,
bumpOnFailedTool: true,
bumpOnPackageManagerCommand: true,
packageManagerCommands: ["npm", "pnpm", "yarn", "bun"],
};
test("detectReasoningBump: bumps on failed bash", () => {
assert.equal(
detectReasoningBump({ toolName: "bash", input: { command: "npm test" }, isError: true }, BUMP_CFG),
"failed bash command",
);
});
test("detectReasoningBump: bumps on npm output even when successful", () => {
assert.equal(
detectReasoningBump({ toolName: "bash", input: { command: "npm test" }, isError: false }, BUMP_CFG),
"npm command result",
);
});
test("detectReasoningBump: bumps on other package managers", () => {
assert.equal(
detectReasoningBump({ toolName: "bash", input: { command: "pnpm test" }, isError: false }, BUMP_CFG),
"pnpm command result",
);
});
test("detectReasoningBump: no bump for non-matching bash", () => {
assert.equal(
detectReasoningBump({ toolName: "bash", input: { command: "git status" }, isError: false }, BUMP_CFG),
null,
);
});
test("detectReasoningBump: bumps on failed edit tool", () => {
assert.equal(
detectReasoningBump({ toolName: "edit", input: { path: "a.txt" }, isError: true }, BUMP_CFG),
"failed edit tool",
);
});
test("detectReasoningBump: no bump for successful edit tool", () => {
assert.equal(
detectReasoningBump({ toolName: "edit", input: { path: "a.txt" }, isError: false }, BUMP_CFG),
null,
);
});
test("detectReasoningBump: can disable non-bash failed-tool bump", () => {
const cfg: ReasoningBumpConfig = { ...BUMP_CFG, bumpOnFailedTool: false };
assert.equal(detectReasoningBump({ toolName: "edit", input: { path: "a.txt" }, isError: true }, cfg), null);
// Still bumps bash failures.
assert.equal(detectReasoningBump({ toolName: "bash", input: { command: "false" }, isError: true }, cfg), "failed bash command");
});
test("isMaxOutputTokensLikeStop: treats length stops as max-output-like", () => {
assert.equal(isMaxOutputTokensLikeStop({ stopReason: "length" }), true);
});
test("isMaxOutputTokensLikeStop: detects proxy/OpenAI incomplete error text", () => {
assert.equal(
isMaxOutputTokensLikeStop({ stopReason: "error", errorMessage: "upstream_terminal_failure_preflush type=response.incomplete reason=max_output_tokens" }),
true,
);
assert.equal(
isMaxOutputTokensLikeStop({ stopReason: "error", errorMessage: "Responses API returned response.incomplete before first token" }),
true,
);
assert.equal(
isMaxOutputTokensLikeStop({ stopReason: "error", errorMessage: "incomplete: max_output_tokens" }),
true,
);
});
test("isMaxOutputTokensLikeStop: ignores non-max-output errors and stops", () => {
assert.equal(isMaxOutputTokensLikeStop({ stopReason: "error", errorMessage: "network timeout" }), false);
assert.equal(isMaxOutputTokensLikeStop({ stopReason: "stop", errorMessage: "max_output_tokens" }), false);
assert.equal(isMaxOutputTokensLikeStop({ stopReason: "aborted", errorMessage: "response.incomplete" }), false);
});
test("nextReasoningEffortForMaxOutput: downgrades xhigh to high", () => {
assert.equal(nextReasoningEffortForMaxOutput("xhigh"), "high");
});
test("nextReasoningEffortForMaxOutput: downgrades high to medium", () => {
assert.equal(nextReasoningEffortForMaxOutput("high"), "medium");
});
test("nextReasoningEffortForMaxOutput: medium and below request rung reset instead of another downgrade", () => {
assert.equal(nextReasoningEffortForMaxOutput("medium"), null);
assert.equal(nextReasoningEffortForMaxOutput("low"), null);
assert.equal(nextReasoningEffortForMaxOutput("minimal"), null);
assert.equal(nextReasoningEffortForMaxOutput(null), null);
});
test("capThinkingLevel: never raises thinking, only caps it", () => {
assert.equal(capThinkingLevel("xhigh", "high"), "high");
assert.equal(capThinkingLevel("high", "high"), "high");
assert.equal(capThinkingLevel("medium", "high"), "medium");
assert.equal(capThinkingLevel("low", "medium"), "low");
assert.equal(capThinkingLevel("minimal", "xhigh"), "minimal");
});
const STAGE_LADDER: Rung[] = [
{ modelId: "r0", thinking: "high" },
{ modelId: "r1", thinking: "high" },
{ modelId: "r2", thinking: "high" },
{ modelId: "r3", thinking: "high" },
];
test("nextStage: advances by one, clamped to last rung", () => {
assert.equal(nextStage(0, STAGE_LADDER), 1);
assert.equal(nextStage(2, STAGE_LADDER), 3);
assert.equal(nextStage(3, STAGE_LADDER), 3); // clamped
assert.equal(nextStage(99, STAGE_LADDER), 3); // far past end
});
test("nextStage: post-bump caller passes the bump index — advance from there", () => {
// Caller-side composition: nextStage(activeBumpIndex ?? stage, ladder)
// A bump on [1] should resume at [2] regardless of where stage was.
assert.equal(nextStage(1, STAGE_LADDER), 2);
});
test("nextStage: ladder size 1 stays at 0", () => {
const ladder: Rung[] = [{ modelId: "only", thinking: "high" }];
assert.equal(nextStage(0, ladder), 0);
});
test("nextStage: empty ladder returns 0", () => {
assert.equal(nextStage(5, []), 0);
});
// ============================================================================
// chooseRung — mode/stage dispatch
// ============================================================================
const LADDER: Rung[] = [
{ modelId: "quick", thinking: "xhigh" }, // [0] plan + user-facing
{ modelId: "model-a", thinking: "xhigh" }, // [1] first autonomous step
{ modelId: "model-a", thinking: "high" }, // [2]
{ modelId: "model-b", thinking: "high" }, // [3]
];
test("chooseRung: idle → null", () => {
assert.equal(chooseRung("idle", 0, LADDER), null);
assert.equal(chooseRung("idle", 99, LADDER), null);
});
test("chooseRung: empty ladder → null", () => {
assert.equal(chooseRung("planning", 0, []), null);
assert.equal(chooseRung("implementing", 5, []), null);
});
test("chooseRung: planning always returns LADDER[0] regardless of stage", () => {
assert.deepEqual(chooseRung("planning", 0, LADDER), LADDER[0]);
assert.deepEqual(chooseRung("planning", 1, LADDER), LADDER[0]);
assert.deepEqual(chooseRung("planning", 99, LADDER), LADDER[0]);
});
test("chooseRung: implementing returns LADDER[stage]", () => {
assert.deepEqual(chooseRung("implementing", 0, LADDER), LADDER[0]);
assert.deepEqual(chooseRung("implementing", 1, LADDER), LADDER[1]);
assert.deepEqual(chooseRung("implementing", 2, LADDER), LADDER[2]);
assert.deepEqual(chooseRung("implementing", 3, LADDER), LADDER[3]);
});
test("chooseRung: implementing past the end clamps to last", () => {
assert.deepEqual(chooseRung("implementing", 4, LADDER), LADDER[3]);
assert.deepEqual(chooseRung("implementing", 99, LADDER), LADDER[3]);
});
test("chooseRung: implementing with negative stage clamps to first", () => {
assert.deepEqual(chooseRung("implementing", -1, LADDER), LADDER[0]);
});
test("chooseContextSafeRungIndex: keeps base rung when context is unknown", () => {
assert.deepEqual(
chooseContextSafeRungIndex(2, LADDER, [{ contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 375000 }], null, 16384),
{ index: 2, fallback: false },
);
});
test("chooseContextSafeRungIndex: keeps base rung when it still fits", () => {
assert.deepEqual(
chooseContextSafeRungIndex(1, LADDER, [{ contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 375000 }, { contextWindow: 375000 }], 120000, 16384),
{ index: 1, fallback: false },
);
});
test("chooseContextSafeRungIndex: falls forward to a later larger-context rung", () => {
assert.deepEqual(
chooseContextSafeRungIndex(1, LADDER, [{ contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 375000 }], 299081, 16384),
{ index: 3, fallback: true },
);
});
test("chooseContextSafeRungIndex: unknown base window does not guess", () => {
assert.deepEqual(
chooseContextSafeRungIndex(1, LADDER, [{ contextWindow: 272000 }, {}, { contextWindow: 375000 }, { contextWindow: 375000 }], 299081, 16384),
{ index: 1, fallback: false },
);
});
test("chooseContextSafeRungIndex: no later rung fits → keep base rung", () => {
assert.deepEqual(
chooseContextSafeRungIndex(1, LADDER, [{ contextWindow: 272000 }, { contextWindow: 272000 }, { contextWindow: 300000 }, { contextWindow: 310000 }], 299081, 16384),
{ index: 1, fallback: false },
);
});
// ============================================================================
// End-to-end lifecycle: simulate the full plan→implement→follow-up flow and
// confirm each LLM call gets the right rung's model + effort.
//
// The state machine driving stage transitions lives in index.ts, but the
// rules are simple enough to encode inline here:
//
// /plan mode=planning, stage=0
// turn_end during implementing stage = min(stage+1, len-1)
// accept mode=implementing, stage=1
// agent_end during implementing stage=0
// ============================================================================
test("end-to-end: full lifecycle uses correct rung at every LLM call", () => {
const ladder: Rung[] = [
{ modelId: "gpt-5.5:quick", thinking: "xhigh", webSearchContextSize: "high" },
{ modelId: "gpt-5.5", thinking: "xhigh", webSearchContextSize: "high" },
{ modelId: "gpt-5.5", thinking: "high", webSearchContextSize: "medium" },
{ modelId: "gpt-5.4", thinking: "high", webSearchContextSize: "low" },
];
const basePayload = () => ({
model: "ignored",
input: [{ role: "user", content: "x" }],
stream: true,
reasoning: { effort: "ignored", summary: "auto" },
});
const seen: string[] = [];
const seenSearch: Array = [];
function fire(mode: "planning" | "implementing", stage: number) {
const rung = chooseRung(mode, stage, ladder);
assert.ok(rung);
const out = applyRungToPayload(basePayload(), rung) as Record & {
model: string;
reasoning: { effort: string };
};
seen.push(`${out.model}:${out.reasoning.effort}`);
// Web search context size follows the rung's setting.
const withSearch = applyOpenAIWebSearchToPayload(out, {
enabled: true,
contextSize: rung.webSearchContextSize,
}) as Record;
const tools = (withSearch.tools as Array> | undefined) ?? [];
const ws = tools.find((t) => t?.type === "web_search");
seenSearch.push(ws ? String(ws.search_context_size) : undefined);
return rung;
}
// /plan → mode=planning, stage=0
let mode: "planning" | "implementing" = "planning";
let stage = 0;
// Plan run with 3 turns (LLM does some reads, asks for clarification).
fire(mode, stage); // turn 1
fire(mode, stage); // turn 2
fire(mode, stage); // turn 3
// agent_end during planning fires the dialog (no stage mutation here).
// Accept → mode=implementing, stage=1, "Please start implementation."
// auto-prompt fires.
mode = "implementing";
stage = 1;
// Implementing run #1: 4 turns. stage advances at each turn_end.
fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 1
fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 2
fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 3
fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 4 (clamped)
// agent_end during implementing → reset stage to 0.
stage = 0;
// User follow-up "also do X". Run #2: 3 turns.
fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 1
fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 2
fire(mode, stage); stage = Math.min(stage + 1, ladder.length - 1); // turn 3
// agent_end → reset.
stage = 0;
// One more follow-up, single turn.
fire(mode, stage);
assert.deepEqual(seen, [
// 3 plan turns: all LADDER[0]
"gpt-5.5:quick:xhigh",
"gpt-5.5:quick:xhigh",
"gpt-5.5:quick:xhigh",
// Start implementation run: starts at LADDER[1], steps to [3], then clamps.
"gpt-5.5:xhigh",
"gpt-5.5:high",
"gpt-5.4:high",
"gpt-5.4:high", // clamped
// Follow-up #1: starts at LADDER[0] (user-facing), steps down.
"gpt-5.5:quick:xhigh",
"gpt-5.5:xhigh",
"gpt-5.5:high",
// Follow-up #2 single turn: LADDER[0].
"gpt-5.5:quick:xhigh",
]);
assert.deepEqual(seenSearch, [
// Plan run: LADDER[0] repeated.
"high",
"high",
"high",
// Start implementation run: [1] → [2] → [3] → [3] clamped.
"high",
"medium",
"low",
"low",
// Follow-up #1: [0] → [1] → [2].
"high",
"high",
"medium",
// Follow-up #2: [0].
"high",
]);
});
// ============================================================================
// End-to-end with reasoning bump: simulate an implementing run where a bash
// failure mid-stream triggers a one-shot bump, then confirm the bumped turn
// uses LADDER[1] AND the next normal turn resumes at LADDER[2] (not back at
// the pre-bump stage cursor).
//
// State machine (subset relevant here):
// tool_result trigger pendingBump = { rungIndex: BUMP_RUNG_INDEX }
// turn_start activeBump = pendingBump; pendingBump = null
// before_provider_request rung = activeBump ? LADDER[bump] : LADDER[stage]
// turn_end stage = nextStage(activeBump?.rungIndex ?? stage, ladder)
// activeBump = null
// ============================================================================
test("end-to-end with bump: bumped turn uses LADDER[1], next turn resumes at LADDER[2]", () => {
const ladder: Rung[] = [
{ modelId: "gpt-5.5:quick", thinking: "xhigh" }, // [0] never reached here
{ modelId: "gpt-5.5", thinking: "xhigh" }, // [1] post-accept + bump target
{ modelId: "gpt-5.5", thinking: "high" }, // [2] post-bump resume
{ modelId: "gpt-5.4", thinking: "high" }, // [3]
];
const BUMP_INDEX = Math.min(1, ladder.length - 1);
const basePayload = () => ({
model: "ignored",
input: [{ role: "user", content: "x" }],
stream: true,
reasoning: { effort: "ignored", summary: "auto" },
});
const seen: string[] = [];
let stage = 1; // accept just happened
let pendingBump: { rungIndex: number } | null = null;
let activeBump: { rungIndex: number } | null = null;
function fireTurn() {
// turn_start: arm the bump if queued.
if (pendingBump) {
activeBump = pendingBump;
pendingBump = null;
}
// before_provider_request: bump wins over normal stage.
const rung = activeBump
? ladder[activeBump.rungIndex]
: chooseRung("implementing", stage, ladder);
assert.ok(rung);
const out = applyRungToPayload(basePayload(), rung) as {
model: string;
reasoning: { effort: string };
};
seen.push(`${out.model}:${out.reasoning.effort}`);
// turn_end: advance stage from bump target if there was one.
const activeBumpIndex = activeBump?.rungIndex;
activeBump = null;
stage = nextStage(activeBumpIndex ?? stage, ladder);
}
// Turn 1: normal post-accept turn. Should use LADDER[1].
fireTurn();
// Between turns 2 and 3 the agent ran `npm test` and the result fired a
// bump trigger. Queue it before turn 2 starts.
pendingBump = { rungIndex: BUMP_INDEX };
// Turn 2: bump active. Should use LADDER[1] (the bump target).
fireTurn();
// Turn 3: no new bump. Stage was advanced from bump index (1) → 2.
// Should use LADDER[2].
fireTurn();
// Turn 4: continues stepping down to LADDER[3].
fireTurn();
assert.deepEqual(seen, [
"gpt-5.5:xhigh", // turn 1: LADDER[1] (post-accept default)
"gpt-5.5:xhigh", // turn 2: LADDER[1] (bump target — same value here, but bump path used)
"gpt-5.5:high", // turn 3: LADDER[2] (resumed AFTER the bump rung, not from pre-bump stage)
"gpt-5.4:high", // turn 4: LADDER[3]
]);
});
// ============================================================================
// End-to-end with max-output rescue: simulate repeated implementation runs that
// terminate with max-output-like stops, confirming the extension retries the
// same rung at lower effort (xhigh → high → medium) before falling back to a
// bump/reset at LADDER[1]. Non-max-output errors should not enter this path.
// ============================================================================
function createMaxOutputRescueHarness(ladder: Rung[], initialStage: number) {
const MAX_CONSECUTIVE_MAX_OUTPUT_RESCUES = 3;
const BUMP_INDEX = Math.min(1, ladder.length - 1);
const basePayload = () => ({
model: "ignored",
input: [{ role: "user", content: "x" }],
stream: true,
reasoning: { effort: "ignored", summary: "auto" },
});
const seen: string[] = [];
let stage = initialStage;
let pendingBump: { rungIndex: number; reason: string } | null = null;
let activeBump: { rungIndex: number; reason: string } | null = null;
let pendingMaxOutputRescue: { rungIndex: number; from: Rung["thinking"]; to: Rung["thinking"] } | null = null;
let activeMaxOutputRescue: { rungIndex: number; from: Rung["thinking"]; to: Rung["thinking"] } | null = null;
let lastProviderRequest: { rungIndex: number; thinking: Rung["thinking"] } | null = null;
let consecutiveMaxOutputRescues = 0;
function fireAttempt(stopInfo: { stopReason?: string; errorMessage?: string | null }) {
// turn_start
lastProviderRequest = null;
if (!pendingBump) {
if (pendingMaxOutputRescue) {
activeMaxOutputRescue = pendingMaxOutputRescue;
pendingMaxOutputRescue = null;
}
} else {
activeBump = pendingBump;
pendingBump = null;
pendingMaxOutputRescue = null;
activeMaxOutputRescue = null;
}
// before_provider_request
const preferredIndex = activeBump?.rungIndex ?? activeMaxOutputRescue?.rungIndex ?? stage;
const rung = ladder[preferredIndex];
assert.ok(rung);
const rescue = pendingMaxOutputRescue ?? activeMaxOutputRescue;
const appliedThinking = rescue ? capThinkingLevel(rung.thinking, rescue.to) : rung.thinking;
const out = applyRungToPayload(basePayload(), { ...rung, thinking: appliedThinking }) as {
model: string;
reasoning: { effort: string };
};
seen.push(`${out.model}:${out.reasoning.effort}`);
lastProviderRequest = { rungIndex: preferredIndex, thinking: appliedThinking };
// turn_end
stage = nextStage(lastProviderRequest.rungIndex, ladder);
activeBump = null;
activeMaxOutputRescue = null;
// agent_end
const maxOutputLike = isMaxOutputTokensLikeStop(stopInfo);
pendingBump = null;
activeBump = null;
if (
maxOutputLike &&
consecutiveMaxOutputRescues < MAX_CONSECUTIVE_MAX_OUTPUT_RESCUES &&
lastProviderRequest
) {
const nextEffort = nextReasoningEffortForMaxOutput(lastProviderRequest.thinking);
if (nextEffort) {
pendingMaxOutputRescue = {
rungIndex: lastProviderRequest.rungIndex,
from: lastProviderRequest.thinking,
to: nextEffort,
};
consecutiveMaxOutputRescues++;
return { kind: "rescue" as const, to: nextEffort };
}
pendingMaxOutputRescue = null;
activeMaxOutputRescue = null;
pendingBump = { rungIndex: BUMP_INDEX, reason: "max output tokens exhausted" };
consecutiveMaxOutputRescues++;
return { kind: "bump" as const, rungIndex: BUMP_INDEX };
}
pendingMaxOutputRescue = null;
activeMaxOutputRescue = null;
lastProviderRequest = null;
consecutiveMaxOutputRescues = 0;
stage = 0;
return { kind: "reset" as const };
}
return {
seen,
fireAttempt,
getStage: () => stage,
getPendingBump: () => pendingBump,
getPendingMaxOutputRescue: () => pendingMaxOutputRescue,
};
}
test("end-to-end with max-output rescue: same rung steps xhigh → high → medium, then bumps to LADDER[1]", () => {
const ladder: Rung[] = [
{ modelId: "gpt-5.5:quick", thinking: "xhigh" },
{ modelId: "gpt-5.4", thinking: "xhigh" },
{ modelId: "gpt-5.4", thinking: "high" },
{ modelId: "gpt-5.4", thinking: "medium" },
{ modelId: "gpt-5.4-mini", thinking: "xhigh" },
];
const harness = createMaxOutputRescueHarness(ladder, 4);
assert.deepEqual(harness.fireAttempt({ stopReason: "length" }), { kind: "rescue", to: "high" });
assert.deepEqual(harness.getPendingMaxOutputRescue(), { rungIndex: 4, from: "xhigh", to: "high" });
assert.deepEqual(harness.fireAttempt({ stopReason: "length" }), { kind: "rescue", to: "medium" });
assert.deepEqual(harness.getPendingMaxOutputRescue(), { rungIndex: 4, from: "high", to: "medium" });
assert.deepEqual(harness.fireAttempt({ stopReason: "length" }), { kind: "bump", rungIndex: 1 });
assert.deepEqual(harness.getPendingBump(), { rungIndex: 1, reason: "max output tokens exhausted" });
// Next autonomous retry uses the queued bump/reset rung.
assert.deepEqual(harness.fireAttempt({ stopReason: "stop" }), { kind: "reset" });
assert.deepEqual(harness.seen, [
"gpt-5.4-mini:xhigh",
"gpt-5.4-mini:high",
"gpt-5.4-mini:medium",
"gpt-5.4:xhigh",
]);
assert.equal(harness.getStage(), 0);
});
test("end-to-end with max-output rescue: non-max-output errors do not trigger the rescue path", () => {
const ladder: Rung[] = [
{ modelId: "gpt-5.5:quick", thinking: "xhigh" },
{ modelId: "gpt-5.4", thinking: "xhigh" },
{ modelId: "gpt-5.4", thinking: "high" },
{ modelId: "gpt-5.4", thinking: "medium" },
{ modelId: "gpt-5.4-mini", thinking: "xhigh" },
];
const harness = createMaxOutputRescueHarness(ladder, 4);
assert.deepEqual(harness.fireAttempt({ stopReason: "error", errorMessage: "network timeout" }), { kind: "reset" });
assert.equal(harness.getPendingMaxOutputRescue(), null);
assert.equal(harness.getPendingBump(), null);
assert.deepEqual(harness.seen, ["gpt-5.4-mini:xhigh"]);
assert.equal(harness.getStage(), 0);
});