import type { ZeroEventsEmitter, ZeroRecoveryReasonKey } from '../types' import type { Context, LogLevel, LogSink } from '@rocicorp/logger' import type { UpdateNeededReason } from '@rocicorp/zero' // per-reason guard window: the SAME reason re-failing right after its reload is // surfaced as fatal instead of reload-storming on something the reload can't fix. const RECOVER_GUARD_MS = 60_000 // if the scheduled reload never actually takes the page down (a consumer's // scheduleReload deferred it, a native reload no-oped, or reload() threw) the // module latch would stay set and kill EVERY later recovery for the page's // life. time it out so recovery can try again — a real reload tears down the // context long before this fires, so it only matters when the reload didn't // land. const RELOAD_LATCH_TIMEOUT_MS = 15_000 // the fatal local-store-loss signature Zero logs when the IndexedDB it expects // is gone (evicted, or deleted by another tab). const LOCAL_STORE_LOST = 'Expected IndexedDB not found' const SQLITE_ERROR_NAME = 'SqliteError' const SQLITE_STATEMENT_FINALIZED = 'This statement has been finalized' const STORE_CLOSED = 'Store is closed' const STORE_CLOSED_REPEAT_MIN_MS = 2_000 const STORE_CLOSED_REPEAT_MAX_MS = 60_000 export type ZeroRecoveryLogClassification = { reasonKey: Exclude< ZeroRecoveryReasonKey, | 'NewClientGroup' | 'VersionNotSupported' | 'SchemaVersionNotSupported' | 'client-state-not-found' > message: string dropLocalState: boolean } export type ZeroLogPattern = string | RegExp // a minimal synchronous key/value store the cross-reload guard persists into. // web defaults to sessionStorage; native (Hermes) has none, so a consumer can // inject an MMKV/sqlite-backed store to get cross-reload loop protection — the // in-memory guard already covers within-a-page-load. export type RecoveryGuardStorage = { getItem: (key: string) => string | null setItem: (key: string, value: string) => void } // what a scheduleReload consumer receives so it can gate/annotate the reload // (soot: IDE-active gate + countdown toast + background-mutation fence; mobile: // expo-updates reload) while still driving the SAME deletes-then-reload work. export type ScheduleReloadContext = { reason: string reasonKey: ZeroRecoveryReasonKey dropLocalState: boolean // deletes every affected instance's local store, awaits beforeReload, then // reloads. idempotent — safe to call once the consumer decides to proceed. performReload: () => Promise } export type ZeroRecoveryDeps = { // delete THIS client's local persistent state (its own scoped IDB). pass the // instance's `.delete`. recovery is per-instance, so a combined client drops // EVERY underlying store; the latch below keeps it to one reload. deleteLocalState: () => Promise // recovery lifecycle for consumers (preview postMessage, toast, …). zeroEvents: ZeroEventsEmitter // awaited before the reload — e.g. wait for the dev origin to come back so the // reload doesn't land on a restarting server. optional. beforeReload?: () => Promise // take over WHEN/HOW the recovery reload happens. default: immediate guarded // reload. the consumer decides (defer until safe, show a countdown, reload // natively) but still calls ctx.performReload to run the real work. scheduleReload?: (ctx: ScheduleReloadContext) => void // cross-reload guard backing store (defaults to sessionStorage on web). inject // a native KV so Hermes gets real cross-reload loop protection. guardStorage?: RecoveryGuardStorage // classified transport/app messages that are expected and must not recover. benignLogPatterns?: readonly ZeroLogPattern[] // internal lifecycle fence. recovery invalidates queued and in-flight work // before the consumer can defer the reload. onRecovery?: (reasonKey: ZeroRecoveryReasonKey) => void // non-DOM hosts cannot reload a page. reconstruct the mounted client in // place after the same delete/beforeReload work completes. recoverInPlace?: () => Promise // injectable for tests; defaults to a real page reload. reload?: () => void | boolean } // one reload per page-load. the local-state deletes that must precede it are // collected here so EVERY affected instance (a combined client has several) drops // its own store before we reload — not just whichever instance fired first. they // are thunks (not started promises) so a deferred reload can't leave the app // running on an already-deleted store: the deletes run only when the reload // actually proceeds. let reloadScheduled = false let reloadInProgress = false let reloadLatchTimer: ReturnType | undefined const pendingDeletes: Array<() => Promise> = [] const pendingInPlaceRecoveries: Array<() => Promise> = [] // within-a-page-load per-reason guard: real loop protection everywhere, // including Hermes (no storage needed). a reload wipes it, which is why the // injectable storage below survives across reloads. const inMemoryGuard = new Map() function defaultGuardStorage(): RecoveryGuardStorage | undefined { try { if (typeof window === 'undefined' || !window.sessionStorage) return undefined return window.sessionStorage } catch { // sessionStorage access throws in some sandboxes / Hermes — no web store. return undefined } } // cross-page-load, per-reason guard: distinct keys mean a NewClientGroup recovery // never suppresses a later SchemaVersionNotSupported one. the in-memory tier // catches re-fires within a page-load (works on Hermes); the storage tier // survives the reload, so a reason that reloaded then immediately re-fires is // caught as a genuine fatal instead of reload-storming across loads. function recoveryGuardOpen( reasonKey: ZeroRecoveryReasonKey, guardStorage: RecoveryGuardStorage | undefined ): boolean { // sessionStorage is shared by same-origin frames in one top-level tab. scope // its guard to the current document so one preview frame recovering does not // falsely fatal a sibling frame. the URL survives a reload, preserving the // cross-reload loop guard for the document that actually recovered. let documentScope = '' try { documentScope = globalThis.location?.href ?? '' } catch {} const key = `on-zero-recover-${documentScope}-${reasonKey}` const now = Date.now() const memLast = inMemoryGuard.get(key) ?? 0 if (memLast > 0 && now - memLast < RECOVER_GUARD_MS) return false const storage = guardStorage ?? defaultGuardStorage() if (storage) { try { const rawLast = storage.getItem(key) const last = rawLast ? Number(rawLast) : 0 if (last > 0 && now - last < RECOVER_GUARD_MS) return false storage.setItem(key, String(now)) } catch { // storage unavailable mid-flight — the in-memory tier is the floor. } } inMemoryGuard.set(key, now) return true } // "Store is closed" is noisy during intentional instance replacement. One log is // teardown noise; repeated logs seconds apart mean a live client is stuck using a // closed local store and needs fresh local state. let lastStoreClosedAtMs = 0 function armReloadLatchTimeout() { if (typeof setTimeout !== 'function') return if (reloadLatchTimer) clearTimeout(reloadLatchTimer) reloadLatchTimer = setTimeout(() => { reloadLatchTimer = undefined // re-open SCHEDULING only. a reload that never landed (still deferred behind // a consumer's IDE gate / countdown, which can hold for minutes) must not // kill future recovery — but the deferred performReload still owns the // pending delete thunks and, if it is already mid-flight (a slow // beforeReload can exceed this timeout), reloadInProgress must stay set so a // second recovery can't double-drive the reload. so touch neither // pendingDeletes nor reloadInProgress here. reloadScheduled = false }, RELOAD_LATCH_TIMEOUT_MS) // don't hold the event loop open (node/native) waiting on this defense timer. if ( reloadLatchTimer && typeof reloadLatchTimer === 'object' && 'unref' in reloadLatchTimer ) { reloadLatchTimer.unref() } } function disarmReloadLatchTimeout() { if (reloadLatchTimer) { clearTimeout(reloadLatchTimer) reloadLatchTimer = undefined } } // deletes every collected instance's local store, awaits beforeReload, then // reloads. deferred a microtask so sibling instances failing in the same tick // enqueue their deletes first. idempotent: a consumer that calls performReload // twice (or after the latch timeout) only runs one reload. function performReload(deps: ZeroRecoveryDeps): Promise { if (reloadInProgress) return Promise.resolve() reloadInProgress = true // the consumer has committed to reloading, so the un-latch timeout has no job // now — disarm it before the (possibly slow) delete/beforeReload chain so it // can't fire mid-flight and re-open scheduling under an in-progress reload. disarmReloadLatchTimeout() // resolve the reload exactly as @rocicorp/zero does internally // (`getBrowserGlobal('location')?.reload()`, zero.js): read location off the // global and optional-chain through it. a non-DOM host with a `window` shim // but no real `location` (the sootsim tenant render-worker hides `location` // for isolation) reconstructs the client in place after the same cleanup. const doReload = deps.reload ?? (() => { const reload = globalThis.location?.reload if (!reload) return false reload.call(globalThis.location) return true }) let inPlaceRecoveries: Array<() => Promise> = [] return Promise.resolve() .then(() => { inPlaceRecoveries = pendingInPlaceRecoveries.splice(0) return Promise.allSettled(pendingDeletes.splice(0).map((run) => run())) }) .then(() => deps.beforeReload?.()) .catch(() => {}) .then(async () => { const reloadStarted = doReload() if (reloadStarted !== false) return try { const results = await Promise.all(inPlaceRecoveries.map((recover) => recover())) if (inPlaceRecoveries.length > 0 && results.some((recovered) => !recovered)) { console.error('[on-zero] recovery could not reload or reconstruct the client') } } catch (error) { console.error('[on-zero] in-place recovery failed', error) } finally { // a page reload tears down this module, but an in-place recovery keeps // it alive. reopen the latch so a later, distinct failure can recover. reloadInProgress = false reloadScheduled = false } }) } function recover( deps: ZeroRecoveryDeps, reasonKey: ZeroRecoveryReasonKey, message: string, dropLocalState: boolean ): void { if (typeof window === 'undefined') return deps.onRecovery?.(reasonKey) // each affected instance drops its OWN stale store, even when a reload is // already queued — otherwise a sibling's store survives the reload and // fatal-loops on the next boot. the single scheduled reload awaits these. if (dropLocalState) { pendingDeletes.push(() => Promise.resolve().then(deps.deleteLocalState)) } if (deps.recoverInPlace && !pendingInPlaceRecoveries.includes(deps.recoverInPlace)) { pendingInPlaceRecoveries.push(deps.recoverInPlace) } // only ONE reload per page-load; a later trigger just contributes its delete. if (reloadScheduled) return if (!recoveryGuardOpen(reasonKey, deps.guardStorage)) { console.error(`[on-zero] ${message} — already recovered once, not reloading`) deps.zeroEvents.emit({ type: 'fatal', reasonKey, reason: message }) // hosts embedding this app (preview shells, test harnesses) observe the // terminal state without sharing the module instance: a plain event on the // realm's global scope. try { globalThis.dispatchEvent?.( new CustomEvent('on-zero-fatal', { detail: { reasonKey, reason: message } }) ) } catch {} return } reloadScheduled = true armReloadLatchTimeout() console.warn(`[on-zero] ${message} — recovering`) deps.zeroEvents.emit({ type: 'recovering', reasonKey, reason: message }) const runReload = () => performReload(deps) if (deps.scheduleReload) { deps.scheduleReload({ reason: message, reasonKey, dropLocalState, performReload: runReload, }) } else { void runReload() } } // passing our own onUpdateNeeded/onClientStateNotFound DISABLES Zero's built-in // reloadWithReason, so these handlers are the ONLY recovery — they must cover // every reason or the app fatal-blanks forever. export function makeZeroRecovery(deps: ZeroRecoveryDeps) { return { onUpdateNeeded(reason: UpdateNeededReason) { // every update-needed reason needs new client code → reload. but only // SchemaVersionNotSupported means the local rows are now incompatible: // mirror Zero's own decision (it disables the client group only for that // reason). NewClientGroup / VersionNotSupported are code-version mismatches // whose data is still valid — and delete() there would wipe the IndexedDB // OTHER live tabs of this user share (the store is keyed by user+storageKey, // not per-tab), e.g. the newer tab that triggered NewClientGroup. const dropLocalState = reason.type === 'SchemaVersionNotSupported' recover( deps, reason.type, `update needed (${reason.message || reason.type})`, dropLocalState ) }, onClientStateNotFound() { // local/server sync state is gone or rejected — the store is unusable, so // drop it and reload into a fresh client. recover(deps, 'client-state-not-found', 'client state not found', true) }, } } // preserve Zero's default console output: we MUST install a logSink to watch for // the corruption signature, but doing so replaces Zero's built-in console sink — // so when the consumer passes none, mirror Zero's key=value context prefix here // instead of silently swallowing every log. function logToConsole( level: LogLevel, context: Context | undefined, ...args: unknown[] ): void { const prefix = context ? Object.entries(context) .map(([key, value]) => (value === undefined ? key : `${key}=${value}`)) .join(' ') : '' const method = level === 'debug' ? 'debug' : level console[method](...(prefix ? [prefix] : []), ...args) } function logArgText(arg: unknown): string { if (typeof arg === 'string') return arg if (arg instanceof Error) return `${arg.name} ${arg.message} ${arg.stack || ''}` if (arg && typeof arg === 'object') { const message = 'message' in arg ? (arg as { message?: unknown }).message : undefined const name = 'name' in arg ? (arg as { name?: unknown }).name : undefined if (typeof message === 'string') return `${typeof name === 'string' ? name : ''} ${message}` } return '' } function isBenignStoreClosedLog(text: string): boolean { if (/Mutator\s+".*"\s+error on server/i.test(text)) return true if (/Mutator\s+".*"\s+app error on client/i.test(text)) return true return false } // the mutation/connection desync class: the local client group is out of sync // with the server's last-mutation-id / cookie / client record, so the store is // unusable and must be dropped + recovered. these surface only through the error // log, never the structured onClientStateNotFound callback. server acknowledgement // timeouts enter through the mutation lifecycle instead of string classification; // expected transport startup messages are declared through benignLogPatterns. function classifyMutationDesync( text: string ): Pick | undefined { if (text.includes('sent mutation ID') && text.includes('but expected')) { return { reasonKey: 'mutation-desync', message: 'mutation id desync', } } if ( text.includes('oooMutation') || text.includes('Server reported an out-of-order mutation') ) { return { reasonKey: 'mutation-desync', message: 'out-of-order mutation', } } if (text.includes('already processed')) { return { reasonKey: 'mutation-desync', message: 'mutation already processed', } } if (text.includes('InvalidConnectionRequestLastMutationID')) { return { reasonKey: 'mutation-desync', message: 'invalid connection last mutation id', } } if (text.includes('InvalidConnectionRequestBaseCookie')) { return { reasonKey: 'connection-cookie-invalid', message: 'invalid connection base cookie', } } if (text.includes('ClientNotFound') || text.includes('Client not found')) { return { reasonKey: 'client-not-found', message: 'client not found', } } if (text.includes('connection userID mismatch')) { return { reasonKey: 'connection-userid-mismatch', message: 'connection user id mismatch', } } return undefined } export function classifyZeroRecoveryLog( level: LogLevel | string, args: readonly unknown[], nowMs = Date.now() ): ZeroRecoveryLogClassification | undefined { if (level !== 'error') return undefined const text = args.map(logArgText).join(' ') if (text.includes(LOCAL_STORE_LOST)) { return { reasonKey: 'indexeddb-not-found', message: 'local store lost', dropLocalState: true, } } if (text.includes(SQLITE_ERROR_NAME) && text.includes(SQLITE_STATEMENT_FINALIZED)) { return { reasonKey: 'sqlite-statement-finalized', message: 'sqlite statement finalized', dropLocalState: true, } } const desync = classifyMutationDesync(text) if (desync) { return { ...desync, dropLocalState: true } } if (text.includes(STORE_CLOSED) && !isBenignStoreClosedLog(text)) { const prevMs = lastStoreClosedAtMs lastStoreClosedAtMs = nowMs if ( prevMs > 0 && nowMs - prevMs >= STORE_CLOSED_REPEAT_MIN_MS && nowMs - prevMs <= STORE_CLOSED_REPEAT_MAX_MS ) { return { reasonKey: 'store-closed-repeat', message: 'local store closed repeatedly', dropLocalState: true, } } } return undefined } // watch error-level logs for the local-store-lost / desync signatures (the ones // Zero surfaces only through the log, not the structured callbacks) and trigger // the same recovery. forwards to the consumer sink, or to the console when there // is none. wire this ONLY when the consumer didn't pass their own logSink, so a // consumer that owns log-based recovery isn't double-fired. export function composeRecoveryLogSink( deps: ZeroRecoveryDeps, consumerLogSink?: LogSink ): LogSink { const consumerFlush = consumerLogSink?.flush return { log(level: LogLevel, context: Context | undefined, ...args: unknown[]): void { if (consumerLogSink) consumerLogSink.log(level, context, ...args) else logToConsole(level, context, ...args) const recovery = classifyZeroRecoveryLog(level, args) if (!recovery) return const text = args.map(logArgText).join(' ') if (deps.benignLogPatterns?.some((pattern) => matchesLogPattern(text, pattern))) return recover(deps, recovery.reasonKey, recovery.message, recovery.dropLocalState) }, // call through the consumer sink so a class-based sink keeps its `this`, // rather than handing Zero a detached method reference. flush: consumerFlush ? () => consumerFlush.call(consumerLogSink) : undefined, } } function matchesLogPattern(message: string, pattern: ZeroLogPattern): boolean { if (typeof pattern === 'string') return message.includes(pattern) pattern.lastIndex = 0 return pattern.test(message) } // generic Zero stale-poke / stale-cookie signatures: the client's view is behind // the server's snapshot cookie, which a plain reconnect resolves — this is not a // fatal store-loss, so ConnectionMonitor reconnects instead of recovering. export function isRecoverableZeroStalePokeMessage(message: string): boolean { return ( message.includes('Server returned unexpected base cookie during sync') || (message.includes('Received cookie') && message.includes('is < than last snapshot cookie') && message.includes('ignoring client view')) ) } // test-only: the reload latch + pending work + in-memory guard are in-memory // (a real page reload clears them); tests simulate that reset between successive // "page loads". the injectable/sessionStorage guard is NOT cleared here — like a // real reload, it survives, which is what catches an immediate re-fire. export function resetRecoveryStateForTests() { reloadScheduled = false reloadInProgress = false if (reloadLatchTimer) { clearTimeout(reloadLatchTimer) reloadLatchTimer = undefined } pendingDeletes.length = 0 pendingInPlaceRecoveries.length = 0 lastStoreClosedAtMs = 0 inMemoryGuard.clear() }