/** * KVFlowCacheManager — workflow-aware KV cache management with steps-to-execution * eviction and overlapped prefetch. * * The core KVFlow insight (arXiv:2507.07400): LRU eviction fails in multi-agent * workflows because it doesn't anticipate future agent usage. This manager uses * the AgentStepGraph to compute "steps-to-execution" (STE) for each cache entry, * then: * * 1. **Eviction**: When GPU memory pressure exceeds threshold, evict entries * with the highest STE first (they'll be needed farthest in the future). * Shared-prefix entries (team-board context) are protected — they're never * evicted until ALL dependent role-overlays are gone. * * 2. **Prefetch**: When an agent is scheduled to execute soon, proactively * load its KV tensors from CPU to GPU in a background thread, overlapping * the PCIe transfer with the current agent's generation. * * 3. **Hit-rate tracking**: Emit telemetry events for cache hits/misses so * the early-warning system and /pipeline-audit can surface metrics. * * Sovereign re-implementation per NMoS (F.046 + F.057): algorithm harvested * from the KVFlow paper, implemented on HoloScript's substrate, not adopting * SGLang or any external binary. * * @module @holoscript/llm-provider/kvflow * @version 0.1.0 */ import type { KVFlowConfig, KVCacheEntry, KVFlowScope, KVFlowTelemetry, EvictionResult, PrefetchResult, StepNodeId } from './types'; import { InMemoryAgentStepGraph } from './AgentStepGraph'; /** * The KVFlow cache manager. Maintains an in-memory model of KV cache entries, * an AgentStepGraph for workflow-aware scheduling, and drives eviction/prefetch * decisions based on steps-to-execution values. * * This is a *coordination layer* — it doesn't manage actual GPU memory or * tensor transfers. Downstream adapters (PagedKVCache, Anthropic prompt cache, * etc.) implement the residency transitions. This manager tells them *what* * to evict, prefetch, or retain. * * Wire to the @caching declaration via `scopeFromBrainCaching()` which maps * BrainCachingScope → KVFlowScope. */ export declare class KVFlowCacheManager { private readonly config; private readonly graph; private readonly entries; private readonly telemetry; private readonly bytesPerToken; private activeStepIds; constructor(config?: Partial); /** * Register an agent step into the workflow graph. * Call when an agent activates, claims a task, or changes role. */ addStep(step: import('./types').AgentStep): void; /** * Remove an agent step (and its edges) from the graph. * Call when an agent deactivates or its session ends. */ removeStep(stepId: StepNodeId): void; /** * Get the underlying step graph for direct inspection. */ getGraph(): InMemoryAgentStepGraph; /** * Set the currently active (executing) steps. These get STE=0 in eviction * calculations and are never evicted. */ setActiveSteps(stepIds: StepNodeId[]): void; /** * Register a KV cache entry for an agent step. Call when an agent's * KV tensors are first loaded (either freshly computed or prefetched). */ addEntry(entry: KVCacheEntry): void; /** * Mark a cache entry as used (updates lastUsedAt timestamp and * recomputes STE from the graph). Call on every cache hit. */ touchEntry(stepId: StepNodeId, now?: string): void; /** * Get a cache entry by step ID. */ getEntry(stepId: StepNodeId): KVCacheEntry | undefined; /** * Get all cache entries. */ getAllEntries(): KVCacheEntry[]; /** * Run an eviction pass when GPU memory pressure exceeds threshold. * * Strategy: * 1. Compute STE for all entries using the AgentStepGraph. * 2. Protect entries with STE <= minRetentionSte (they'll be needed soon). * 3. Protect shared-prefix entries until ALL their dependent overlays are evicted. * 4. Evict scene-turn entries with highest STE first. * 5. Demote role-overlay entries to CPU (host) before evicting entirely. * 6. Evict role-overlay entries with highest STE if still over pressure. * * Returns the eviction result with entries categorized by action. */ evict(targetFreedBytes: number): EvictionResult; /** * Run a prefetch pass for agents scheduled to execute soon. * * Uses the AgentStepGraph to identify the next N agents in the schedule, * checks if their KV entries are on CPU (host) or evicted, and initiates * background transfer to GPU. * * This is the "overlapped prefetch" from KVFlow: while the current agent * generates tokens, the next agent's KV tensors are being loaded in * parallel, hiding PCIe transfer latency. * * In a real implementation, this would dispatch GPU memory copy operations * on a background thread/stream. Here, we model the scheduling decision * and return the prefetch plan for the adapter layer to execute. */ prefetch(currentStepId: StepNodeId): PrefetchResult; /** * Record a cache hit for an agent step. Updates STE and emits telemetry. * Call when a cached KV entry is reused without recomputation. */ recordHit(stepId: StepNodeId): void; /** * Record a cache miss for an agent step. Emits telemetry. * Call when an agent's KV tensors need to be recomputed from scratch. */ recordMiss(stepId: StepNodeId, scope: KVFlowScope): void; /** * Get recent telemetry events. Used by /pipeline-audit and /reflect * to surface KVFlow hit rate and prefetch metrics. */ getTelemetry(limit?: number): KVFlowTelemetry[]; /** * Compute cache hit rate over recent telemetry. */ hitRate(sampleSize?: number): { hits: number; misses: number; rate: number; }; /** * Current GPU memory pressure (0.0 = empty, 1.0 = full). */ pressure(): number; private currentGpuUsage; private emitTelemetry; } /** * Map Brittney's BrainCachingScope to KVFlowScope. * This is the bridge between the @caching declaration in brain compositions * and the KVFlow cache manager's eviction policy. * * BrainCachingScope → KVFlowScope: * - 'team-board' → 'shared-prefix' (high reuse, protected in eviction) * - 'agent-role' → 'role-overlay' (medium reuse, demoted before eviction) * - 'scene-local' → 'scene-turn' (low reuse, evicted first) */ export declare function scopeFromBrainCaching(brainScope: 'team-board' | 'agent-role' | 'scene-local'): KVFlowScope; /** * Map KVFlowScope back to BrainCacheUsage for telemetry and diagnostics. */ export declare function scopeToCacheUsage(scope: KVFlowScope): 'shared-prefix' | 'role-overlay' | 'scene-turn'; /** * Estimate the byte size of a KV cache entry from its token count. * Uses a configurable bytes-per-token estimate (default 512 bytes/token). */ export declare function estimateKVBytes(tokenCount: number, bytesPerToken?: number): number; /** * Create a KVCacheEntry from an AgentStep and computed STE value. * Convenience factory for wiring step graph → cache manager. */ export declare function entryFromStep(step: import('./types').AgentStep, stepsToExecution: number, residency?: KVResidency, dependentOverlayIds?: StepNodeId[], bytesPerToken?: number): KVCacheEntry; type KVResidency = import('./types').KVResidency; export {};