import type { DrainStatus } from "./fs-daemon-flush"; const CIRCUIT_BREAKER_LIMIT = 5; const CIRCUIT_BREAKER_WINDOW_MS = 10_000; const CIRCUIT_BREAKER_PAUSE_MS = 60_000; export type BreakerState = { path: string; trippedAt: number; resumesAt: number; syncsInWindow: number; }; // `DrainStatus` lives in cli/fs-daemon-flush.ts so the flush subcommand's // polling helper can import it without dragging in the daemon's main // entrypoint (which would crash test runners that import this file). export class HashTracker { private hashes = new Map(); private timers = new Map(); private recentSyncs = new Map(); private paused = new Map(); private onBreakerTrip?: (state: BreakerState) => void; // Bumped right before each scheduled fn() actually starts; decremented // in the fn()'s `.finally()`. The flush subcommand reads this through // `drainStatus()` (mirrored into the status file once per reconcile). private inFlightUploads = 0; private inFlightPaths = new Set(); private failedPaths = new Set(); private generations = new Map(); needsRetry(path: string): boolean { return this.failedPaths.has(path); } retryPaths(): string[] { return [...this.failedPaths]; } get(absPath: string): string | undefined { return this.hashes.get(absPath); } set(absPath: string, hash: string): void { this.hashes.set(absPath, hash); } forget(absPath: string): void { this.generations.set(absPath, (this.generations.get(absPath) ?? 0) + 1); this.hashes.delete(absPath); this.failedPaths.delete(absPath); const timer = this.timers.get(absPath); if (timer) { clearTimeout(timer); this.timers.delete(absPath); } this.recentSyncs.delete(absPath); this.paused.delete(absPath); } knownPaths(): string[] { return [...this.hashes.keys()]; } activeBreakers(): BreakerState[] { return [...this.paused.values()]; } setBreakerListener(listener: (state: BreakerState) => void): void { this.onBreakerTrip = listener; } /** * Returns true if the path is currently throttled by the circuit * breaker (>5 syncs in the last 10s). */ recordAndCheckBreaker(absPath: string): boolean { if (this.paused.has(absPath)) return true; const now = Date.now(); const recent = (this.recentSyncs.get(absPath) ?? []).filter( (ts) => now - ts < CIRCUIT_BREAKER_WINDOW_MS, ); recent.push(now); this.recentSyncs.set(absPath, recent); if (recent.length > CIRCUIT_BREAKER_LIMIT) { const state: BreakerState = { path: absPath, trippedAt: now, resumesAt: now + CIRCUIT_BREAKER_PAUSE_MS, syncsInWindow: recent.length, }; this.paused.set(absPath, state); // Loud enough to land in the supervisor's tail of fs-daemon.log // (sandbox-bootstrap.ts surfaces the last 50 lines on failure). console.error( `[dench-fs-daemon] CIRCUIT_BREAKER tripped path=${absPath} syncs=${recent.length} pause=${CIRCUIT_BREAKER_PAUSE_MS}ms`, ); this.onBreakerTrip?.(state); setTimeout(() => { this.paused.delete(absPath); this.recentSyncs.delete(absPath); console.warn( `[dench-fs-daemon] circuit breaker released for ${absPath}`, ); }, CIRCUIT_BREAKER_PAUSE_MS); return true; } return false; } schedule(absPath: string, ms: number, fn: () => Promise): void { const existing = this.timers.get(absPath); if (existing) clearTimeout(existing); const timer = setTimeout(() => { this.timers.delete(absPath); // Bound mounted reads and network uploads together; one path is serial. if (this.inFlightUploads >= 4 || this.inFlightPaths.has(absPath)) { this.schedule(absPath, 50, fn); return; } this.inFlightPaths.add(absPath); const generation = this.generations.get(absPath) ?? 0; // Track this work in-flight so `flush` knows to wait for the // upload to actually finish, not just for the next reconcile // tick to fire. The decrement runs in `finally` so even a // throwing upload doesn't strand the counter at non-zero. this.inFlightUploads += 1; Promise.resolve() .then(fn) .then(() => { if ((this.generations.get(absPath) ?? 0) === generation) this.failedPaths.delete(absPath); }) .catch((error) => { if ((this.generations.get(absPath) ?? 0) === generation) this.failedPaths.add(absPath); console.error( `[dench-fs-daemon] sync failed for ${absPath}: ${ error instanceof Error ? error.message : String(error) }`, ); }) .finally(() => { this.inFlightPaths.delete(absPath); this.inFlightUploads = Math.max(0, this.inFlightUploads - 1); }); }, ms); this.timers.set(absPath, timer); } /** * Snapshot of pending sync work. Mirrored into the daemon status * file once per reconcile tick so `dench-fs-daemon flush` can wait * on it without an IPC channel. */ drainStatus(): DrainStatus { return { pendingPaths: new Set([...this.timers.keys(), ...this.failedPaths]).size, inFlightUploads: this.inFlightUploads, }; } }