// keep-warm: holding an idle conversation's prompt cache open by re-reading it shortly before it expires. Only the // clock lives here; how many refreshes a hold may spend is priced by the provider that serves it (runtimes/claude). import { z } from "zod"; import type { SandboxNotice } from "../events/sandbox-notice.js"; // A cache clock as the daemon publishes it: the last request that touched the entry, and the entry's lifetime. export interface CacheClock { readonly at: number; readonly ttlMs: number; } /** How long before expiry a refresh starts: a fifth of the entry's life, never under a minute or over ten. */ export const keepWarmLeadMs = (ttlMs: number): number => Math.min(10 * 60_000, Math.max(60_000, Math.round(ttlMs / 5))); /** How many refreshes keep an entry alive through `until`. */ export const keepWarmRefreshes = (cache: CacheClock, until: number): number => { const beyond = until - (cache.at + cache.ttlMs); return beyond <= 0 ? 0 : Math.ceil(beyond / (cache.ttlMs - keepWarmLeadMs(cache.ttlMs))); }; /** The latest instant `left` more refreshes can carry an entry to: past it, refreshing costs more than the cold resume it saves. */ export const keepWarmHorizon = (cache: CacheClock, left: number): number => cache.at + cache.ttlMs + Math.max(0, left) * (cache.ttlMs - keepWarmLeadMs(cache.ttlMs)); /** When the next refresh is due for an entry last touched at `cache.at`. */ export const keepWarmDueAt = (cache: CacheClock): number => cache.at + cache.ttlMs - keepWarmLeadMs(cache.ttlMs); /** The furthest `until` a hold can honestly promise: the economic horizon, or the date change, whichever comes first. */ export const keepWarmCap = (cache: CacheClock, rollsAt: number | undefined, left: number): number => rollsAt === undefined ? keepWarmHorizon(cache, left) : Math.min(keepWarmHorizon(cache, left), rollsAt); // Why a hold ended before anyone picked the conversation up, one per thing a person would do about it; `detail` carries // the specifics. `elapsed` is the one ending that is not a problem. export const KeepWarmEndSchema = z.enum(["elapsed", "allowance", "changed", "cold", "failed"]); export type KeepWarmEnd = z.infer; export const KeepWarmSchema = z.object({ since: z.number().describe("When keeping it warm started, in milliseconds."), until: z .number() .describe( "When it stops by itself, in milliseconds: the time asked for, shortened to what the sandbox can honestly keep, which is never past the point where refreshing costs more than re-reading, nor past the date change that rewrites the prompt.", ), refreshes: z.number().int().min(0).describe("Refreshes sent so far."), readTokens: z .number() .optional() .describe("How much the last refresh read back from the provider's cache, in tokens: the proof the cache was still there."), ended: z .object({ at: z.number().describe("When it stopped, in milliseconds."), reason: KeepWarmEndSchema.describe( "Why: `elapsed` the time asked for ran out; `allowance` the account reached the reserve kept for real work, or the provider refused a refresh for its limit; `changed` what the next turn would send no longer matches the cache (a part of the prompt, the date in it, or the conversation's session or account); `cold` the cache was gone, expired before a refresh could run or found missing by one; `failed` a refresh failed.", ), detail: z .string() .optional() .describe("The specifics, when there are any: which parts of the prompt changed, how full the account was, or the failure's own words."), }) .optional() .describe("Why keeping it warm stopped before anyone picked the conversation up. Absent while it is still being kept."), }); export type KeepWarm = z.infer; export const AgentKeepWarmSchema = z.object({ id: z.string().min(1).describe("Which conversation."), until: z .number() .nullable() .describe( "Keep its prompt cache warm until this instant, in milliseconds; shortened to what the sandbox can honestly keep. Null stops keeping it warm.", ), }); export type AgentKeepWarm = z.infer; // What a turn's first request found in the cache, and how long it had been kept warm for it when it was. export const PromptCacheOpeningSchema = z.object({ readTokens: z.number().describe("Tokens the turn's first request read from the provider's cache."), writtenTokens: z.number().describe("Tokens it wrote to the cache, which is what it paid full price for."), kept: z .object({ forMs: z.number().describe("How long the cache had been kept warm for this turn, in milliseconds."), refreshes: z.number().int().min(0).describe("How many refreshes that took."), }) .optional() .describe("Present when this turn picked up a conversation the sandbox had been keeping warm."), }); export type PromptCacheOpening = z.infer; /** A token count as the transcript words it: "305k", or the bare count under a thousand. */ export const tokensLabel = (tokens: number): string => (tokens >= 1_000 ? `${Math.round(tokens / 1_000)}k` : String(tokens)); const spanLabel = (ms: number): string => { const minutes = Math.max(1, Math.round(ms / 60_000)); const hours = Math.floor(minutes / 60); return hours === 0 ? `${minutes}m` : `${hours}h ${minutes % 60}m`; }; /** The notice for a turn that picked up a conversation kept warm for it, by code and facts; undefined for any other turn. */ export const keptWarmNotice = (opening: PromptCacheOpening): Extract | undefined => { const { kept } = opening; if (kept === undefined) { return undefined; } const warm = opening.readTokens >= opening.writtenTokens; const params = { tokens: tokensLabel(warm ? opening.readTokens : opening.writtenTokens), span: spanLabel(kept.forMs), refreshes: kept.refreshes }; return warm ? { code: "keptWarm", params } : { code: "keptCold", params }; }; /** The transcript's line for a turn that picked up a conversation kept warm for it; undefined for any other turn. */ export const keptWarmLine = (opening: PromptCacheOpening): string | undefined => { const notice = keptWarmNotice(opening); if (notice === undefined) { return undefined; } const { tokens, span, refreshes } = notice.params; const held = `kept warm for ${span} with ${refreshes} ${refreshes === 1 ? "refresh" : "refreshes"}`; return notice.code === "keptWarm" ? `Picked up warm: ${tokens} tokens read from the cache, ${held}.` : `Picked up cold despite being ${held}: ${tokens} tokens sent again, because this turn's prompt no longer matched the cache.`; }; // A turn's prefix, part by part, so two turns that should share a cache can say which part stopped matching. export const PromptFingerprintSchema = z.object({ hash: z.string().describe("One short hash over every part."), parts: z.record(z.string(), z.string()).describe("Each part's own short hash or value, by name."), }); export type PromptFingerprint = z.infer; /** Which parts differ between two fingerprints, by name; empty when they match. */ export const changedParts = (before: PromptFingerprint, after: PromptFingerprint): string[] => [...new Set([...Object.keys(before.parts), ...Object.keys(after.parts)])].filter((name) => before.parts[name] !== after.parts[name]).toSorted(); // keep-warm's knobs, one object since each only means anything beside the others (cache-keepwarm.ts). export const KeepWarmSettingsSchema = z.object({ auto: z .boolean() .default(false) .describe( "Keep a Claude conversation's prompt cache warm after each turn a person asked for. Each refresh re-reads the cached context at the cache price and adds nothing to the conversation.", ), hours: z .number() .min(1) .max(8) .default(4) .describe( "How long an idle conversation is kept warm after its last turn, in hours, and what a press on one offers first. Shortened where refreshing would cost more than the cold resume it saves, and at midnight where the agent runs, when the date in its prompt changes.", ), minTokens: z .number() .int() .min(0) .default(100_000) .describe("Only conversations at least this large, in tokens, are kept warm by `auto`: a small one is cheap to re-read anyway."), reserve: z .number() .int() .min(0) .max(90) .default(15) .describe( "How much of an account's usage limit, in percent, keeping conversations warm must leave untouched for real work. Refreshing stops once any limit that account's model spends is fuller than that.", ), }); export type KeepWarmSettings = z.infer;