/**
* pi-prefix-cache-compaction
*
* Pi's default compaction serializes the history into ONE new user message under a
* different system prompt, so a server-side prefix cache (vLLM, SGLang, llama.cpp)
* misses completely and the whole history is prefilled again. On local GPUs that is
* minutes per compaction.
*
* This extension instead:
* 1. captures each real provider request (before_provider_request),
* 2. on compaction re-sends that exact request plus one summarize instruction,
* thinking settings untouched -> the prefix is served from cache (Qwen-style
* templates render reasoning effort into the system prompt, so changing thinking
* would change the whole prefix),
* 3. after compaction, optionally sends a 1-token warm-up with the new context so the
* next turn does not prefill the summary + kept messages cold.
*
* Works for the Anthropic Messages and OpenAI Chat Completions wire formats.
* Anything unexpected returns nothing, and Pi runs its default compaction.
* Technique credit: pisceslailai/deepseek-kvcache (MIT), adapted to Anthropic Messages.
*/
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
import { buildSessionContext, convertToLlm } from "@earendil-works/pi-coding-agent";
import { completeSimple } from "@earendil-works/pi-ai/compat";
import {
appliesTo,
backgroundFits,
backgroundStartTokens,
buildSummaryBody,
buildWarmupBody,
type Config,
createCollector,
endpointUrl,
fileListSuffix,
isCapturable,
mergeConfig,
captureFileName,
modelKey,
parseStoredCapture,
requestHeaders,
serializeCapture,
type ResolvedAuth,
type SseResult,
type StreamCollector,
streamMessages,
summaryTokenBudget,
toPiUsage,
wireApi,
} from "./core.ts";
const CONFIG_NAME = "pi-prefix-cache-compaction.json";
const CAPTURE_TTL_MS = 14 * 24 * 3600_000;
/** Debug: PI_PREFIX_CACHE_DUMP=
writes every real request and warm-up body there. */
let dumpSeq = 0;
function dump(kind: string, body: unknown) {
const dir = process.env.PI_PREFIX_CACHE_DUMP;
if (!dir) return;
try {
mkdirSync(dir, { recursive: true });
writeFileSync(join(dir, `${String(++dumpSeq).padStart(3, "0")}-${kind}.json`), JSON.stringify(body, null, 1));
} catch {
// debug only
}
}
const agentDir = () => process.env.PI_CODING_AGENT_DIR ?? join(homedir(), ".pi", "agent");
const captureDir = () => join(agentDir(), "cache", "pi-prefix-cache-compaction");
function readJsonFile(path: string, problems: string[]): Partial | undefined {
if (!existsSync(path)) return undefined;
try {
return JSON.parse(readFileSync(path, "utf8"));
} catch (err) {
problems.push(`${path}: ${(err as Error).message}`);
return undefined;
}
}
function loadConfig(cwd: string): { config: Config; problems: string[] } {
const problems: string[] = [];
const config = mergeConfig(readJsonFile(join(agentDir(), CONFIG_NAME), problems), readJsonFile(join(cwd, ".pi", CONFIG_NAME), problems));
return { config, problems };
}
type Capture = {
payload?: Record;
headers?: Record;
/** modelKey() of the model the payload was captured for (undefined = legacy capture). */
modelKey?: string;
/** True after a Pi-default compaction invalidated the prefix; a new real turn clears it. */
stale?: boolean;
/** Session leaf when the payload was sent: the last entry the payload contains. */
leafId?: string | null;
/** Pi's context estimate when the payload was sent. */
tokens?: number | null;
};
/** A summary of one captured request, computed while the session keeps working. */
type BackgroundJob = {
/** The assistant reply to the summarized request: the kept tail starts here. */
firstKeptEntryId: string;
tokensAt: number;
modelKey?: string;
abort: AbortController;
collector: StreamCollector;
done: Promise;
result?: SseResult;
error?: string;
t0: number;
/** A pre-warm request for the post-compaction context is in flight. */
prewarming?: boolean;
/** Settles when the in-flight pre-warm finishes (success or not). */
prewarmDone?: Promise;
/** Number of completed pre-warms (the first one prefills the whole future context). */
prewarms: number;
};
const SPINNER = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
const STATUS_KEY = "prefix-cache-compaction";
type PillTone = "busy" | "bg" | "ok" | "warn";
const PILL_STYLE: Record = {
busy: { fg: "accent", bg: "toolPendingBg" },
bg: { fg: "muted", bg: "toolPendingBg" },
ok: { fg: "success", bg: "toolSuccessBg" },
warn: { fg: "warning", bg: "toolErrorBg" },
};
/**
* Footer pill (ctx.ui.setStatus): an animated spinner with elapsed time while a phase runs,
* then a short-lived colored result. Pure decoration: every UI call is guarded so a Pi build
* without themes or without a TUI (print / RPC mode) never affects the compaction itself.
*/
/**
* Background work (summary, pre-warm, warm-up) can finish after `/reload` or a session switch,
* when the ctx it captured is stale: Pi then throws on any ctx access, and an exception escaping
* a fire-and-forget promise kills the whole process (seen live). Every UI touch goes through this.
*/
function uiAlive(ctx: ExtensionContext): boolean {
try {
return ctx.hasUI;
} catch {
return false;
}
}
function createStatusPill() {
let timer: ReturnType | undefined;
let clearTimer: ReturnType | undefined;
// The background summary only paints while nothing in the foreground owns the pill.
let owner: "fg" | "bg" | undefined;
const stop = () => {
clearInterval(timer);
clearTimeout(clearTimer);
timer = clearTimer = undefined;
};
const paint = (ctx: ExtensionContext, tone: PillTone, text: string) => {
try {
const theme = ctx.ui.theme;
const label = ` ${text} `;
const pill = theme ? theme.bg(PILL_STYLE[tone].bg, theme.bold(theme.fg(PILL_STYLE[tone].fg, label))) : label;
ctx.ui.setStatus(STATUS_KEY, pill);
} catch {
// decoration only
}
};
return {
/** Animate `label()` (re-evaluated every tick, so it can show live progress). */
start(ctx: ExtensionContext, label: (secs: number) => string, who: "fg" | "bg" = "fg") {
if (!uiAlive(ctx) || (who === "bg" && owner === "fg" && timer)) return;
stop();
owner = who;
const t0 = Date.now();
let i = 0;
const tone: PillTone = who === "bg" ? "bg" : "busy";
const tick = () => paint(ctx, tone, `${SPINNER[i++ % SPINNER.length]} ${label(Math.floor((Date.now() - t0) / 1000))}`);
tick();
timer = setInterval(tick, 120);
timer.unref?.();
},
/** Show a final state for a few seconds, then remove the pill. */
finish(ctx: ExtensionContext, tone: PillTone, text: string, holdMs = 8_000, who: "fg" | "bg" = "fg") {
if (!uiAlive(ctx) || (who === "bg" && owner !== "bg")) return;
stop();
owner = who;
paint(ctx, tone, text);
if (holdMs <= 0) return; // stays until the next start/finish/clear
clearTimer = setTimeout(() => this.clear(ctx, who), holdMs);
clearTimer.unref?.();
},
/** Remove the pill; a background caller only removes its own. */
clear(ctx: ExtensionContext, who: "fg" | "bg" = "fg") {
if (who === "bg" && owner !== "bg") return;
stop();
owner = undefined;
try {
ctx.ui.setStatus(STATUS_KEY, undefined);
} catch {
// decoration only
}
},
};
}
const fmtTokens = (n: number) => (n >= 1000 ? `${(n / 1000).toFixed(n >= 10_000 ? 0 : 1)}k` : String(n));
const stats = { compactions: 0, fallbacks: 0, warmups: 0, lastSeconds: 0, lastReason: "", lastWarmupError: "", background: 0, backgroundUsed: 0, lastBackgroundError: "", prewarms: 0, lastPrewarmError: "" };
/**
* Request-time auth, the same way Pi resolves it for a real turn. `before_provider_headers`
* sees Pi's header map BEFORE the API key is attached (the key goes to the SDK as `apiKey`),
* so the captured headers alone are keyless against any authenticated endpoint (issue #2).
* Older Pi builds without `getApiKeyAndHeaders` fall back to the captured map only.
*/
async function resolveAuth(ctx: ExtensionContext): Promise<{ auth?: ResolvedAuth; error?: string }> {
const registry = ctx.modelRegistry as { getApiKeyAndHeaders?: (m: any) => Promise } | undefined;
if (!ctx.model || typeof registry?.getApiKeyAndHeaders !== "function") return {};
try {
const r = await registry.getApiKeyAndHeaders(ctx.model);
if (!r?.ok) return { error: r?.error ?? "auth resolution failed" };
return { auth: { apiKey: r.apiKey, headers: r.headers, baseUrl: r.baseUrl } };
} catch (err) {
return { error: (err as Error).message };
}
}
/**
* One 1-token request with the context Pi would build from `entries` at `leafId`, through
* Pi's model registry (request-time auth, Pi's own converter) with the captured system
* prompt, tools and thinking settings, so that context lands in the server's prefix cache.
*/
async function sendWarm(ctx: ExtensionContext, cap: Capture, id: string, entries: any[], leafId: string | null): Promise {
const model = ctx.model;
if (!model || !cap.payload) throw new Error("no model or capture");
const session = buildSessionContext(entries, leafId);
const messages = convertToLlm(session.messages);
// The captured map only contributes routing / attribution headers; the credential
// is resolved at request time, never replayed from the capture (issue #2).
const headers = requestHeaders(wireApi(model)!, cap.headers, undefined, { "x-session-id": id });
delete headers["content-type"];
const context = { systemPrompt: "", messages, tools: [] };
const payload = cap.payload;
const onPayload = (built: unknown) => {
const body = buildWarmupBody(payload, built as Record);
dump(leafId === "pi-prefix-cache-compaction-prewarm" ? "prewarm" : "warmup", body);
return body;
};
const registry = ctx.modelRegistry as unknown as { streamSimple?: (m: any, c: any, o: any) => { result(): Promise } };
let result: { stopReason?: string; errorMessage?: string };
if (typeof registry.streamSimple === "function") {
// Pi >= 0.86: the registry streams with request-time auth exactly like a real turn.
result = await registry.streamSimple(model, context, { maxTokens: 1, headers, onPayload }).result();
} else {
// Pi 0.85: the registry facade cannot stream. Resolve auth through it and call the
// same pi-ai entrypoint Pi 0.85 uses for its own compaction.
const { auth, error } = await resolveAuth(ctx);
if (error) throw new Error(`auth: ${error}`);
const routed = auth?.baseUrl ? { ...model, baseUrl: auth.baseUrl } : model;
result = await completeSimple(routed, context, {
maxTokens: 1,
headers: { ...headers, ...(auth?.headers ?? {}) },
apiKey: auth?.apiKey,
onPayload,
});
}
if (result.stopReason === "error" || result.stopReason === "aborted") {
throw new Error(result.errorMessage ?? `stop reason ${result.stopReason}`);
}
}
export default function prefixCacheCompaction(pi: ExtensionAPI) {
const captures = new Map(); // Pi session id -> last real request
let config: Config | undefined;
const cfg = (ctx: ExtensionContext) => (config ??= loadConfig(ctx.cwd).config);
const notify = (ctx: ExtensionContext, msg: string, kind: "info" | "warning" = "info") => {
try {
if (cfg(ctx).notify) ctx.ui.notify(msg, kind);
} catch {
// stale ctx after /reload or a session switch: the message has nowhere to go
}
};
const pill = createStatusPill();
const jobs = new Map(); // Pi session id -> background summary
const dropJob = (id: string) => {
jobs.get(id)?.abort.abort();
jobs.delete(id);
};
const prewarmed = new Set(); // sessions whose last compaction used a pre-warmed summary
/**
* Warm the context Pi will build after it compacts with this job's summary: system,
* summary message, then every entry from firstKeptEntryId to now. That context only grows
* at the end, so each later pre-warm prefills just the turns added since the last one, and
* the first turn after the compaction finds it cached instead of prefilling ~100k cold.
*/
const prewarm = (ctx: ExtensionContext, id: string, job: BackgroundJob) => {
try {
startPrewarm(ctx, id, job);
} catch {
job.prewarming = false; // stale ctx: no pre-warm, the first turn after compaction prefills normally
}
};
const startPrewarm = (ctx: ExtensionContext, id: string, job: BackgroundJob) => {
const c = cfg(ctx);
const cap = captures.get(id);
if (!c.warmup || !job.result || job.prewarming || !cap?.payload || jobs.get(id) !== job) return;
job.prewarming = true;
const t0 = Date.now();
const virtual = {
type: "compaction",
id: "pi-prefix-cache-compaction-prewarm",
parentId: ctx.sessionManager.getLeafId(),
timestamp: new Date().toISOString(),
summary: job.result.text,
firstKeptEntryId: job.firstKeptEntryId,
tokensBefore: 0,
// Pi stores the current system message on the compaction entry (Pi >= 1.0) and puts it
// before the summary; mirror that so the warmed prefix is the one Pi will send.
systemMessage: [...(ctx.sessionManager.buildSessionProjection?.().messages ?? [])].reverse().find((m: any) => m.role === "system"),
};
job.prewarmDone = sendWarm(ctx, cap, id, [...ctx.sessionManager.getEntries(), virtual], virtual.id)
.then(() => {
job.prewarms++;
stats.prewarms++;
stats.lastPrewarmError = "";
if (job.prewarms === 1) notify(ctx, `Background compaction: post-compaction context pre-warmed in ${Math.round((Date.now() - t0) / 1000)}s; the first turn after compaction starts from cache`);
})
.catch((err) => {
stats.lastPrewarmError = (err as Error).message ?? String(err);
})
.finally(() => {
job.prewarming = false;
});
};
const fallback = (ctx: ExtensionContext, reason: string) => {
stats.fallbacks++;
stats.lastReason = reason;
if (cfg(ctx).statusBar) pill.finish(ctx, "warn", `⚠ prefix-cache skipped: ${reason.length > 48 ? `${reason.slice(0, 47)}…` : reason}`);
notify(ctx, `Prefix-cache compaction skipped (${reason}); using Pi default`, "warning");
};
// Persist mode: the last real request per session, written off the hot path (debounced)
// and flushed on shutdown, so a resumed session can compact from cache before its first turn.
const pendingWrites = new Map>();
const writeCapture = (id: string) => {
pendingWrites.delete(id);
const cap = captures.get(id);
if (!cap?.payload || cap.stale) return;
try {
mkdirSync(captureDir(), { recursive: true });
const file = join(captureDir(), captureFileName(id));
writeFileSync(`${file}.tmp`, serializeCapture(id, { payload: cap.payload, headers: cap.headers, modelKey: cap.modelKey }));
renameSync(`${file}.tmp`, file);
} catch {
// best effort: no file just means the old "no captured request" fallback
}
};
const scheduleWrite = (id: string) => {
clearTimeout(pendingWrites.get(id));
const t = setTimeout(() => writeCapture(id), 2_000);
t.unref?.();
pendingWrites.set(id, t);
};
const dropStored = (id: string) => {
clearTimeout(pendingWrites.get(id));
pendingWrites.delete(id);
try {
unlinkSync(join(captureDir(), captureFileName(id)));
} catch {
// absent is fine
}
};
const loadStored = (id: string): Capture | undefined => {
try {
const stored = parseStoredCapture(readFileSync(join(captureDir(), captureFileName(id)), "utf8"), id);
return stored && { payload: stored.payload, headers: stored.headers, modelKey: stored.modelKey, stale: false };
} catch {
return undefined;
}
};
pi.on("session_start", async (_event, ctx) => {
const { config: c, problems } = loadConfig(ctx.cwd);
config = c;
if (problems.length && c.notify) ctx.ui.notify(`pi-prefix-cache-compaction: ignoring malformed config (${problems.join("; ")})`, "warning");
try {
for (const f of readdirSync(captureDir())) {
const p = join(captureDir(), f);
if (Date.now() - statSync(p).mtimeMs > CAPTURE_TTL_MS) unlinkSync(p);
}
} catch {
// no cache dir yet
}
});
// Release the (potentially megabyte-scale) captured payload when a session goes away;
// in persist mode flush it first so `pi -c` can pick it up.
pi.on("session_shutdown", (_event, ctx) => {
const id = ctx.sessionManager.getSessionId();
if (pendingWrites.has(id)) {
clearTimeout(pendingWrites.get(id));
writeCapture(id);
}
captures.delete(id);
dropJob(id);
pill.clear(ctx);
});
pi.on("before_provider_headers", (event, ctx) => {
if (!appliesTo(ctx.model, cfg(ctx))) return;
const id = ctx.sessionManager.getSessionId();
const cap = captures.get(id) ?? {};
cap.headers = { ...event.headers }; // copy: later extensions may mutate event.headers in place
captures.set(id, cap);
});
pi.on("before_provider_request", (event, ctx) => {
if (!appliesTo(ctx.model, cfg(ctx)) || !isCapturable(event.payload)) return;
const id = ctx.sessionManager.getSessionId();
const cap = captures.get(id) ?? {};
cap.payload = structuredClone(event.payload as Record);
dump("request", cap.payload);
cap.modelKey = modelKey(ctx.model);
cap.stale = false; // this real turn re-anchors the prefix
cap.leafId = ctx.sessionManager.getLeafId();
cap.tokens = ctx.getContextUsage?.()?.tokens;
captures.set(id, cap);
if (cfg(ctx).persist) scheduleWrite(id);
});
// Background mode: after a turn whose request crossed `backgroundAt`, summarize that request
// (its prefix is now in the server cache) while the session keeps going. Runs at most one
// job per session; a job whose kept tail grew too large is replaced by a fresher one.
pi.on("turn_end", (event, ctx) => {
const c = cfg(ctx);
const api = wireApi(ctx.model);
if (!c.background || !api || !appliesTo(ctx.model, c)) return;
const id = ctx.sessionManager.getSessionId();
const cap = captures.get(id);
const window = ctx.model?.contextWindow ?? 0;
if (!cap?.payload || cap.stale || cap.modelKey !== modelKey(ctx.model) || cap.tokens == null) return;
if (cap.tokens < backgroundStartTokens(window, c)) return;
// The reply must directly follow the captured request, or the kept tail would miss entries.
if (!cap.leafId || ctx.sessionManager.getEntry(event.messageEntryId)?.parentId !== cap.leafId) return;
const job = jobs.get(id);
if (job && job.modelKey === cap.modelKey && !job.error && backgroundFits(window, cap.tokens, job.tokensAt, job.collector.text.length)) {
prewarm(ctx, id, job); // extend the warmed post-compaction prefix by this turn
return;
}
if (job?.error && cap.tokens - job.tokensAt < window * 0.1) return; // failed recently: no retry storm
const budget = summaryTokenBudget(window, cap.tokens, c);
if (!budget) return;
const refreshing = Boolean(job?.result);
dropJob(id);
const abort = new AbortController();
const collector = createCollector(api);
const next: BackgroundJob = { firstKeptEntryId: event.messageEntryId, tokensAt: cap.tokens, modelKey: cap.modelKey, abort, collector, done: Promise.resolve(), t0: Date.now(), prewarms: 0 };
const payload = cap.payload;
const headers = cap.headers;
stats.background++;
notify(
ctx,
refreshing
? `Background compaction: refreshing the summary (too much happened since the last one), ${cap.tokens.toLocaleString()} tokens`
: `Background compaction: started at ${cap.tokens.toLocaleString()} / ${window.toLocaleString()} tokens (${Math.round((cap.tokens / window) * 100)}%); you can keep working`,
);
if (c.statusBar) {
pill.start(ctx, (secs) => {
const out = collector.text.length + collector.thinkingChars;
return `pre-summarizing in background${out ? ` · ~${fmtTokens(Math.round(out / 4))} out` : ""} · ${secs}s`;
}, "bg");
}
next.done = (async () => {
try {
const { auth, error } = await resolveAuth(ctx);
if (error) throw new Error(`auth: ${error}`);
next.result = await streamMessages(
endpointUrl(auth?.baseUrl ?? ctx.model?.baseUrl, api),
requestHeaders(api, headers, auth, { "x-session-id": id }, ctx.model?.headers),
JSON.stringify(buildSummaryBody(payload, budget, api)),
collector,
abort.signal,
);
if (!next.result.text.trim()) throw new Error("empty summary");
const secs = Math.round((Date.now() - next.t0) / 1000);
const u = next.result.usage;
const promptIn = (u.input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0);
notify(
ctx,
`Background compaction: summary ready in ${secs}s (${promptIn ? `${promptIn.toLocaleString()} tokens in` : `${cap.tokens!.toLocaleString()} tokens`}${u.cache_read_input_tokens ? `, ${u.cache_read_input_tokens.toLocaleString()} from prefix cache` : ""}, ${u.output_tokens ?? "?"} out); ready for the next compaction`,
);
// Stays visible until the compaction uses it, so it is clear one is on hand.
if (c.statusBar) pill.finish(ctx, "ok", `● compaction ready · summary made in ${secs}s`, 0, "bg");
prewarm(ctx, id, next);
} catch (err) {
next.result = undefined;
next.error = abort.signal.aborted ? "aborted" : (err as Error).message;
if (!abort.signal.aborted) {
stats.lastBackgroundError = next.error;
notify(ctx, `Background compaction failed (${next.error}); the next compaction runs normally`, "warning");
if (c.statusBar) pill.finish(ctx, "warn", "⚠ background summary failed; will compact normally", 6_000, "bg");
} else pill.clear(ctx, "bg");
}
})().catch(() => {});
jobs.set(id, next);
});
pi.on("session_before_compact", async (event, ctx) => {
const c = cfg(ctx);
const api = wireApi(ctx.model);
if (!api || !appliesTo(ctx.model, c)) return;
const id = ctx.sessionManager.getSessionId();
const job = jobs.get(id);
if (c.background && job && !event.customInstructions && job.modelKey === modelKey(ctx.model) && event.branchEntries.some((e) => e.id === job.firstKeptEntryId)) {
const { preparation, signal } = event;
const window = ctx.model?.contextWindow ?? 0;
const tCompact = Date.now();
if (!job.result && !job.error) {
// Still streaming: waiting for the rest beats starting a summary from scratch.
notify(ctx, `Compaction: background summary still running (${Math.round((Date.now() - job.t0) / 1000)}s so far); waiting for the rest instead of starting over`);
if (c.statusBar) pill.start(ctx, (secs) => `finishing background summary · ~${fmtTokens(Math.round((job.collector.text.length + job.collector.thinkingChars) / 4))} out · ${secs}s`);
const onAbort = () => job.abort.abort();
signal.addEventListener("abort", onAbort, { once: true });
await job.done;
signal.removeEventListener("abort", onAbort);
if (signal.aborted) {
jobs.delete(id);
pill.clear(ctx);
return { cancel: true };
}
}
if (job.result && backgroundFits(window, preparation.tokensBefore, job.tokensAt, job.result.text.length)) {
if (job.prewarming && job.prewarmDone) {
// The next request goes out right after this returns. Racing an in-flight pre-warm
// of the same prefix makes it prefill everything itself (measured: 32.6 s vs 3.3 s
// after the pre-warm finished), so let the pre-warm finish first. It only covers
// the turns since the previous pre-warm, so this is normally a few seconds.
if (c.statusBar) pill.start(ctx, (secs) => `finishing pre-warm · ${secs}s`);
let onAbort = () => {};
await Promise.race([
job.prewarmDone,
new Promise((r) => {
setTimeout(r, 90_000).unref?.();
}),
new Promise((r) => {
onAbort = () => r();
signal.addEventListener("abort", onAbort, { once: true });
}),
]);
signal.removeEventListener("abort", onAbort);
if (signal.aborted) {
pill.clear(ctx);
return { cancel: true };
}
}
jobs.delete(id);
job.abort.abort(); // nothing in flight should outlive the job (a pre-warm finishes on its own)
if (job.prewarms > 0) prewarmed.add(id);
stats.compactions++;
stats.backgroundUsed++;
stats.lastSeconds = 0;
notify(ctx, `Compaction: used background summary (${preparation.tokensBefore.toLocaleString()} tokens; kept everything after the summarized request)`);
const took = ((Date.now() - tCompact) / 1000).toFixed(1);
if (c.statusBar) pill.finish(ctx, "ok", `✓ compacted in ${took}s · ${job.prewarms ? "context pre-warmed" : "summary from background"}`);
return {
compaction: {
// Exactly the pre-warmed text: a file-list suffix (only known now) would change
// the summary message and lose the cache hit on everything after it.
summary: job.result.text,
firstKeptEntryId: job.firstKeptEntryId,
tokensBefore: preparation.tokensBefore,
usage: toPiUsage(job.result.usage),
},
};
}
if (job.result) notify(ctx, "Compaction: background summary is out of date (too much happened since it was made); compacting normally", "warning");
dropJob(id); // failed or outgrown: fall through to the normal path
}
if (event.reason === "overflow") return; // context already over the window; nothing to reuse
let cap = captures.get(id);
if (!cap?.payload && c.persist) {
// Resumed session compacting before its first turn: replay the request the previous
// process sent. That prefix is what the server cached, whatever this process would
// render now (the warm-up afterwards uses this process's converter).
const stored = loadStored(id);
if (stored) captures.set(id, (cap = { ...stored, headers: cap?.headers ?? stored.headers }));
}
if (!cap?.payload) return fallback(ctx, "no captured request yet in this process");
if (cap.modelKey !== modelKey(ctx.model)) return fallback(ctx, "last captured request was from a different model");
if (cap.stale) return fallback(ctx, "capture is stale after an earlier default compaction");
const { preparation, signal } = event;
const budget = summaryTokenBudget(ctx.model?.contextWindow ?? 0, preparation.tokensBefore, c);
if (!budget) return fallback(ctx, "not enough room left in the context window");
const { auth, error } = await resolveAuth(ctx);
if (error) return fallback(ctx, `auth: ${error}`);
const t0 = Date.now();
const incremental = c.incremental && event.reason !== "overflow" ? preparation.previousSummary : undefined;
notify(ctx, `Compaction: reusing cached prefix (${preparation.tokensBefore.toLocaleString()} tokens${incremental ? `, incremental (delta since last summary)` : ""})`);
const collector = createCollector(api);
if (c.statusBar) {
const size = fmtTokens(preparation.tokensBefore);
pill.start(ctx, (secs) => {
// ~4 chars per token is only for a sense of motion, not a count.
const out = collector.text.length + collector.thinkingChars;
return out ? `summarizing ${size} · ~${fmtTokens(Math.round(out / 4))} out · ${secs}s` : `prefilling ${size} from cache · ${secs}s`;
});
}
try {
const { text, usage } = await streamMessages(
endpointUrl(auth?.baseUrl ?? ctx.model?.baseUrl, api),
requestHeaders(api, cap.headers, auth, { "x-session-id": id }, ctx.model?.headers),
JSON.stringify(buildSummaryBody(cap.payload, budget, api, incremental)),
collector,
signal,
);
const secs = Math.round((Date.now() - t0) / 1000);
stats.compactions++;
stats.lastSeconds = secs;
const cached = usage.cache_read_input_tokens;
notify(ctx, `Compaction done in ${secs}s (${usage.output_tokens ?? "?"} tokens out${cached ? `, ${cached.toLocaleString()} prompt tokens served from prefix cache` : ""})`);
// Warm-up (if any) takes over the pill in session_compact.
if (c.statusBar) pill.finish(ctx, "ok", `✓ compacted in ${secs}s${cached ? ` · ${fmtTokens(cached)} cached` : ""}`);
return {
compaction: {
summary: text + fileListSuffix(preparation.fileOps),
firstKeptEntryId: preparation.firstKeptEntryId,
tokensBefore: preparation.tokensBefore,
usage: toPiUsage(usage),
// Incremental: carry the previous summary through so the NEXT cycle can
// merge its delta into this one. Pi 0.99's prepareCompaction reads the prior
// compaction entry's `summary` for preparation.previousSummary, so storing the
// merged delta summary here is what makes the chain work. (Pi's own entries use
// a details field for extension data; we keep the previous summary in it.)
details: incremental ? { previousSummary: incremental } : undefined,
},
};
} catch (err) {
if (signal.aborted) {
pill.clear(ctx);
return { cancel: true };
}
return fallback(ctx, (err as Error).message);
}
});
// Warm the new prefix (summary + kept messages) so the next turn is not a cold prefill.
pi.on("session_compact", async (event, ctx) => {
const c = cfg(ctx);
const id = ctx.sessionManager.getSessionId();
const cap = captures.get(id);
// A Pi-default compaction means the captured prefix no longer describes the session.
// Keep it only to feed the warm-up below; never for a summary again until a real turn
// re-anchors it (before_provider_request clears `stale`).
if (cap && !event.fromExtension) cap.stale = true;
// The stored request now describes pre-compaction history; never replay it on resume.
dropStored(id);
dropJob(id); // any background summary describes pre-compaction history too
pill.clear(ctx, "bg"); // a "compaction ready" pill would now be wrong
if (!c.warmup || !appliesTo(ctx.model, c) || event.willRetry) return;
if (!cap?.payload || !ctx.model) return;
if (cap.modelKey !== modelKey(ctx.model)) return; // warmed prefix would not match the new model
const warmed = prewarmed.delete(id);
const warmup = async () => {
try {
const t0 = Date.now();
if (c.statusBar) pill.start(ctx, (secs) => `warming next turn · ${secs}s`);
await sendWarm(ctx, cap, id, ctx.sessionManager.getEntries(), ctx.sessionManager.getLeafId());
stats.warmups++;
stats.lastWarmupError = "";
const warmSecs = Math.round((Date.now() - t0) / 1000);
notify(ctx, `Context re-warmed in ${warmSecs}s; next turn starts from cache`);
if (c.statusBar) pill.finish(ctx, "ok", `✓ compacted · warm in ${warmSecs}s`);
} catch (err) {
// Warm-up is best effort; the next turn simply prefills normally. The reason is
// kept for /prefix-compaction so a silently failing warm-up is diagnosable.
stats.lastWarmupError = (err as Error).message ?? String(err);
if (c.statusBar) pill.finish(ctx, "warn", "⚠ warm-up failed; next turn prefills cold");
if (process.env.PI_PREFIX_CACHE_DEBUG) console.error(`[pi-prefix-cache-compaction] warm-up failed: ${stats.lastWarmupError}`);
}
};
if (!c.background) return void (await warmup().catch(() => {}));
// Background mode never holds the session for a warm-up.
if (warmed) return; // the post-compaction context was pre-warmed before Pi compacted
// A request that is about to go out (auto-compaction mid-run) prefills this context
// itself; a parallel warm-up would prefill the same tokens twice and slow it down.
if (typeof ctx.isIdle === "function" && !ctx.isIdle()) return;
void warmup().catch(() => {});
});
pi.registerCommand("prefix-compaction", {
description: "pi-prefix-cache-compaction: status and config",
handler: async (_args, ctx) => {
const { config: c, problems } = loadConfig(ctx.cwd); // fresh read so edits are visible
config = c;
ctx.ui.notify(
[
`applies to current model: ${appliesTo(ctx.model, c)} (api: ${ctx.model?.api ?? "none"})`,
`cached compactions: ${stats.compactions} (last ${stats.lastSeconds}s), fallbacks: ${stats.fallbacks}${stats.lastReason ? ` (last: ${stats.lastReason})` : ""}, warm-ups: ${stats.warmups}${stats.lastWarmupError ? ` (last warm-up error: ${stats.lastWarmupError})` : ""}`,
`background summaries: ${stats.background} started, ${stats.backgroundUsed} used${stats.lastBackgroundError ? ` (last error: ${stats.lastBackgroundError})` : ""}; pre-warms: ${stats.prewarms}${stats.lastPrewarmError ? ` (last error: ${stats.lastPrewarmError})` : ""}`,
`config: ${JSON.stringify(c)}`,
...problems.map((p) => `config problem: ${p}`),
].join("\n"),
"info",
);
},
});
}