/** * Chunked snapshot storage for RoomDO. * * Cloudflare Durable Object storage rejects any single `put()` value * over 128 KiB. Most EtherCalc rooms have snapshots well under that * ceiling, but the legacy dump contains ~17 k rooms whose SocialCalc * save is 150 KiB–2 MB — large enough that the naive * `storage.put("snapshot", )` path would fail with the opaque * `500 Internal Server Error` we hit during the 2026-04-21 production * migration (see AGENTS.md §14). * * Two layouts on disk: * * - Legacy / small: one key `snapshot` → save string. * - Chunked / large: one key `snapshot:meta` → `{chunks: N}` plus * N keys `snapshot:chunk:` → split UTF-8 pieces. * * Readers call {@link readSnapshot}, which tries the fast single-key * path first and falls back to reassembly. Writers call * {@link snapshotEntries} to get a plain-object shard of keys to merge * into whatever batched `storage.put(entries)` they're building. * * Chunk size is 100 KiB — leaves 28 KiB of headroom under 128 KiB for * JSON framing / encoding overhead when the same batch also carries * other large values (the margin is conservative; D1's limit-walking * behavior is documented sparsely). */ import { STORAGE_KEYS, snapshotChunkKey, } from '@ethercalc/shared/storage-keys'; import { getStorageValuesBatched } from './storage-batch.ts'; /** Max bytes per chunk — well under the 128 KiB DO-storage ceiling. */ export const SNAPSHOT_CHUNK_BYTES = 100 * 1024; /** Defense-in-depth cap for chunk-upload metadata (about 200 MiB). */ export const MAX_SNAPSHOT_CHUNKS = 2_048; /** Shape of the `snapshot:meta` value when chunking is in use. */ export interface SnapshotMeta { /** Number of `snapshot:chunk:` entries, 1-based count. */ readonly chunks: number; } function isValidSnapshotMeta(value: unknown): value is SnapshotMeta { return ( value !== null && typeof value === 'object' && 'chunks' in value && Number.isSafeInteger(value.chunks) && (value.chunks as number) >= 1 && (value.chunks as number) <= MAX_SNAPSHOT_CHUNKS ); } /** * Check whether the room has any snapshot at all, under either * layout, without paying to reassemble the chunked form. Used by * `/_exists/:room` where the answer is just a boolean. */ export async function hasSnapshot( storage: DurableObjectStorage, ): Promise { const keys = [STORAGE_KEYS.snapshot, STORAGE_KEYS.snapshotMeta]; const got = await storage.get(keys); return ( typeof got.get(STORAGE_KEYS.snapshot) === 'string' || got.get(STORAGE_KEYS.snapshotMeta) !== undefined ); } /** * Read the room's snapshot, reassembling from chunks when present. * Returns `null` when no snapshot exists under either layout — the * caller distinguishes "empty room" from "never seeded". */ export async function readSnapshot( storage: DurableObjectStorage, ): Promise { // Fast path: single-key snapshot (small room or legacy DO). const single = await storage.get(STORAGE_KEYS.snapshot); if (typeof single === 'string') return single; // Chunked path: meta tells us how many pieces to fetch. const meta = await storage.get(STORAGE_KEYS.snapshotMeta); if (meta === undefined || meta === null) return null; if (!isValidSnapshotMeta(meta)) { throw new Error('invalid snapshot chunk metadata'); } const keys: string[] = []; for (let i = 0; i < meta.chunks; i++) keys.push(snapshotChunkKey(i)); const got = await getStorageValuesBatched(storage, keys); const parts: string[] = []; for (let i = 0; i < meta.chunks; i++) { const k = snapshotChunkKey(i); const part = got.get(k); // A missing chunk would silently corrupt the save — fail loud. if (typeof part !== 'string') { throw new Error(`snapshot chunk ${i} missing under key ${k}`); } parts.push(part); } return parts.join(''); } /** * Produce snapshot-related key-value entries for callers to store with the * platform-sized batch helpers. This function does not perform I/O. * * For snapshots ≤ {@link SNAPSHOT_CHUNK_BYTES}, returns `{snapshot: …}`. * For larger, returns `{snapshot:meta: {chunks}, snapshot:chunk:: …}`. * * Empty-snapshot contract: the caller decides whether to call this at * all. Passing `""` produces an entries map with the single-key form * (storing an empty string), which mirrors the behavior of the * pre-chunking code path. */ export function snapshotEntries(snapshot: string): Record { const bytes = utf8Bytes(snapshot); if (bytes <= SNAPSHOT_CHUNK_BYTES) { return { [STORAGE_KEYS.snapshot]: snapshot }; } const chunks = chunkString(snapshot, SNAPSHOT_CHUNK_BYTES); // Defense in depth only: `#postSnapshotChunk` already refuses uploads // declaring more than MAX_SNAPSHOT_CHUNKS chunks, and every other caller // is bounded by the 25 MiB request-body cap, so no reachable input // reaches 200 MiB. The chunker is O(n) over code units; this guard is a // hard ceiling, not a performance hot path. /* istanbul ignore next -- @preserve: unreachable below the upload caps */ if (chunks.length > MAX_SNAPSHOT_CHUNKS) { throw new RangeError('snapshot exceeds chunk limit'); } const out: Record = { [STORAGE_KEYS.snapshotMeta]: { chunks: chunks.length } satisfies SnapshotMeta, }; for (let i = 0; i < chunks.length; i++) { out[snapshotChunkKey(i)] = chunks[i]; } return out; } /** * Return the full list of storage keys that belong to a snapshot * under EITHER layout, so callers that don't do a full `deleteAll()` * can selectively remove the old shape before writing a new one. * Accepts the prior meta object (from `readMeta`) so the chunk count * is known without another round-trip. */ export async function readSnapshotMeta( storage: DurableObjectStorage, ): Promise { const meta = await storage.get(STORAGE_KEYS.snapshotMeta); if (meta === undefined || meta === null) return null; if (!isValidSnapshotMeta(meta)) { throw new Error('invalid snapshot chunk metadata'); } return meta; } /** * Split a string into UTF-8 byte-bounded chunks without splitting * multi-byte code points. `String.prototype.slice` can leave a high * surrogate at the end of one chunk and its pair at the start of the * next — that's the behavior we want for JS strings (which are UTF-16) * because concat reassembles byte-for-byte. To bound UTF-8 bytes without * allocating one `TextEncoder` result per scalar, scan code units, account for * each scalar's encoded width, and slice only at chunk boundaries. */ function chunkString(s: string, maxBytes: number): string[] { const out: string[] = []; let start = 0; let index = 0; let currentBytes = 0; while (index < s.length) { const code = s.charCodeAt(index); let scalarBytes: number; let codeUnits = 1; if (code <= 0x7f) { scalarBytes = 1; } else if (code <= 0x7ff) { scalarBytes = 2; } else if (code >= 0xd800 && code <= 0xdbff && index + 1 < s.length) { const next = s.charCodeAt(index + 1); if (next >= 0xdc00 && next <= 0xdfff) { scalarBytes = 4; codeUnits = 2; } else { // TextEncoder replaces an unpaired surrogate with U+FFFD (3 bytes). scalarBytes = 3; } } else { // Three-byte BMP scalar or an unpaired low/final high surrogate. scalarBytes = 3; } if (currentBytes + scalarBytes > maxBytes && index > start) { out.push(s.slice(start, index)); start = index; currentBytes = 0; } currentBytes += scalarBytes; index += codeUnits; } // Empty input yields one empty chunk (`slice(0)` of `''`). Non-empty input // always leaves `start` on the final incomplete chunk, including when the // whole string fits in one piece (`start === 0`). out.push(s.slice(start)); return out; } /** * UTF-8 byte length of a string. Uses `TextEncoder` (available in * workerd, Bun, and Node ≥ 16) rather than `Buffer.byteLength` so the * same code path runs inside the DO, where `Buffer` is only available * via the `nodejs_compat` flag. */ function utf8Bytes(s: string): number { return new TextEncoder().encode(s).byteLength; }