/** * What actually went to the provider — and whether its HEAD changed since last time. * * `/context` (see `context-measure.ts`) measures the messages we hand Pi. It cannot see * the other half of a prompt: the system prompt Pi builds and the tool schemas it sends. * On a local backend those are not a footnote — the chat template renders the tool list * INTO the system message, so the tools sit at the very front of the prompt, ahead of * every word anyone has said. * * Which makes the head the one thing that must never change. Every serving backend * (llama.cpp, vLLM, SGLang, Anthropic's cache) reuses exactly one thing: a prefix of bytes * it has already read. Change byte zero and the whole prompt is re-read — the tokens are * the same, the seconds are not. * * We know this because it happened. From the owner's live session, two consecutive LLM * calls INSIDE ONE TURN: * * [6] assistant prefill= 97 cached=24 348 → called bash ×2 (prompt 24 445) * [9] assistant prefill=11 653 cached= 0 → called bash (prompt 11 653) * * The cache went to zero, and the prompt HALVED while the conversation grew by two tool * results. ~12 800 tokens vanished from the head between one call and the next — the size * of a tool list, not of a message. Nobody could say which tools, because nothing was * looking. * * So this looks. It is not a benchmark to run once and delete: it is a standing alarm for * the class of bug the last branch closed — a prompt we rewrite while asking a backend to * remember it. `/context` reports it, and a head that changes mid-run is reported as the * defect it is. * * Structural typing, no SDK import: a payload is whatever the provider was handed. */ /** The parts of a provider payload we can compare. Provider-agnostic by shape. */ interface PayloadLike { model?: unknown; messages?: unknown; tools?: unknown; system?: unknown; } /** One request, reduced to what decides whether a backend can reuse it. */ export interface PayloadShape { /** Characters in the head: the system prompt plus the serialized tool schemas. */ headChars: number; /** The tools the model was offered, in the order it was offered them. */ toolNames: string[]; /** Characters in the tool schemas alone — usually most of the head. */ toolChars: number; /** Characters in the conversation. */ messageChars: number; messages: number; /** The whole payload, serialized — what a prefix cache actually sees. */ totalChars: number; /** A cheap fingerprint of the head. Equal heads → equal fingerprints. */ headKey: string; } /** How this request compares with the one before it. */ export interface PayloadDelta { /** The head is byte-identical to the previous request's. This is the good case. */ headStable: boolean; /** Tools the model gained since the previous request. */ toolsAdded: string[]; /** Tools it lost. */ toolsRemoved: string[]; /** Characters the head grew (negative: shrank). */ headCharsDelta: number; } function textOf(value: unknown): string { if (typeof value === "string") return value; if (value === undefined || value === null) return ""; return JSON.stringify(value); } /** * The name of a tool as the provider was given it. OpenAI-style wraps it * (`{type:"function", function:{name}}`); Anthropic-style does not (`{name}`). */ function toolName(tool: unknown): string { if (typeof tool !== "object" || tool === null) return "?"; const record = tool as { name?: unknown; function?: { name?: unknown } }; const name = record.function?.name ?? record.name; return typeof name === "string" ? name : "?"; } /** * Whether a message is the system prompt. OpenAI-completions carries it as the first * message (`role: "system"`); Anthropic carries it in a `system` field of its own, which * {@link describePayload} reads separately. */ function isSystemMessage(message: unknown): boolean { if (typeof message !== "object" || message === null) return false; const role = (message as { role?: unknown }).role; return role === "system" || role === "developer"; } /** * A stable fingerprint of a string. Not cryptographic and does not need to be: it answers * one question — "are these the same bytes?" — for a value we already hold in full. */ export function fingerprint(text: string): string { let h1 = 0x811c9dc5; let h2 = 0x01000193; for (let i = 0; i < text.length; i += 1) { const c = text.charCodeAt(i); h1 = Math.imul(h1 ^ c, 0x01000193); h2 = Math.imul(h2 + c, 0x85ebca6b) ^ (h2 >>> 13); } const a = (h1 >>> 0).toString(16).padStart(8, "0"); const b = (h2 >>> 0).toString(16).padStart(8, "0"); return `${a}${b}:${text.length}`; } /** Reduce a provider payload to the shape a prefix cache cares about. */ export function describePayload(payload: unknown): PayloadShape { const body = ( typeof payload === "object" && payload !== null ? payload : {} ) as PayloadLike; const messages = Array.isArray(body.messages) ? body.messages : []; const tools = Array.isArray(body.tools) ? body.tools : []; // The system prompt: a `system` field (Anthropic) or the leading system message // (OpenAI-completions, which is what a llama.cpp server speaks). const leading = messages.filter(isSystemMessage); const systemText = textOf(body.system) + leading.map(textOf).join(""); // No tools → no tool bytes. Faithful to the payload, which omits the key entirely // (`params.tools` is only set when the list is non-empty), and to the prompt, where an // empty list renders to nothing. const toolsText = tools.length > 0 ? JSON.stringify(tools) : ""; const conversation = messages.filter((message) => !isSystemMessage(message)); const messageChars = conversation.reduce( (sum, message) => sum + textOf(message).length, 0, ); return { headChars: systemText.length + toolsText.length, toolNames: tools.map(toolName), toolChars: toolsText.length, messageChars, messages: conversation.length, totalChars: systemText.length + toolsText.length + messageChars, headKey: fingerprint(`${systemText}${toolsText}`), }; } /** Compare a request with the one before it. */ export function comparePayloads( previous: PayloadShape, next: PayloadShape, ): PayloadDelta { const before = new Set(previous.toolNames); const after = new Set(next.toolNames); return { headStable: previous.headKey === next.headKey, toolsAdded: next.toolNames.filter((name) => !before.has(name)), toolsRemoved: previous.toolNames.filter((name) => !after.has(name)), headCharsDelta: next.headChars - previous.headChars, }; } /** A head that changed, and everything needed to say who changed it. */ export interface HeadChurn { at: number; /** True when the head changed BETWEEN STEPS OF ONE RUN. */ midRun: boolean; /** * The tools the model was offered are NOT the ones we last set: the list was rewritten * by someone outside this extension. See {@link PrefixWatch.record}. */ foreign: boolean; delta: PayloadDelta; shape: PayloadShape; } /** Whether two tool lists hold the same names (order is not the question here). */ function sameTools(a: readonly string[], b: readonly string[]): boolean { if (a.length !== b.length) return false; const seen = new Set(a); return b.every((name) => seen.has(name)); } /** * Watches every provider request and remembers when the head changed. * * Two questions decide whether a head change is worth waking anybody for, and they are * different questions: * * - **When?** Between RUNS is expected and often correct — a mode switch really does * change which tools exist; the manager and the owner are not having the same * conversation. Between two calls of ONE run, nothing about the model's situation * changed, and the backend threw away everything it had read. * - **Who?** `setActiveTools` is a global setter with no notion of whose tools are whose, * and we are not the only extension that writes it (`pi-planner` rebuilds the whole * list from `getAllTools()` on every provider request). So a mid-run change may be a * stranger's — the bug this watchdog was built to name — or it may be ours, which is a * cost we chose with our eyes open, not news. * * Only a change we did not make is a defect. Told nothing about who writes what, the first * version of this watchdog reported our own deliberate move — a finished memory pass * withdrawing its probes — as an intrusion, once per pass, in the owner's feed. An alarm * that cannot recognise its owner's footsteps is an alarm nobody keeps armed. Hence * {@link record} takes the list WE set, and {@link defects} counts only what is both * mid-run and foreign. */ export class PrefixWatch { private last: PayloadShape | null = null; private churns: HeadChurn[] = []; /** Requests seen since the current run began — 0 means the next one opens a run. */ private requestsThisRun = 0; constructor( private readonly now: () => number = Date.now, /** How many churn events to keep. The newest are the ones anybody reads. */ private readonly keep = 8, ) {} /** A run started: the next request is its first, so a head change there is not mid-run. */ runStarted(): void { this.requestsThisRun = 0; } /** * Record one provider request. Returns the churn it caused, if any. * * `ours` is the tool list we last wrote (`ToolVisibility.lastSet()`), or `null` when we * have not written one — the only way to tell a head we rewrote from a head somebody * else did. A payload whose tools are not ours is `foreign` whether it churned or not; * a churn is only reported when the head actually changed. */ record(payload: unknown, ours?: readonly string[] | null): HeadChurn | null { const shape = describePayload(payload); const previous = this.last; this.last = shape; const midRun = this.requestsThisRun > 0; this.requestsThisRun += 1; if (!previous) return null; const delta = comparePayloads(previous, shape); if (delta.headStable) return null; // Unknown ownership (no list of ours to compare with) is not an accusation: we // cannot claim it and we will not blame anyone for it. const foreign = ours != null && !sameTools(shape.toolNames, ours); const churn: HeadChurn = { at: this.now(), midRun, foreign, delta, shape }; this.churns = [churn, ...this.churns].slice(0, this.keep); return churn; } /** The last request's shape, or null before the first one. */ current(): PayloadShape | null { return this.last; } /** Head changes seen this session, newest first — ours and everyone else's. */ history(): readonly HeadChurn[] { return this.churns; } /** * The defects: a head rewritten mid-run by something outside this extension. Our own * mid-run changes are not here — they are deliberate, and we pay for them knowingly. */ defects(): readonly HeadChurn[] { return this.churns.filter((churn) => churn.midRun && churn.foreign); } }