/** * Binary-content detection for `dench-fs-daemon`. * * Background: the daemon's `commitFileUpload` historically inlined every * file ≤ `MAX_INLINE_TEXT_BYTES` (100 KB) into `fileContents.text` via * `TextDecoder("utf-8", { fatal: false }).decode(bytes)`. The * `fatal: false` mode silently substitutes U+FFFD (replacement char) * for any invalid byte sequence — perfect for ad-hoc text rendering, * disastrous when a small PNG / PDF / .zip lands in the cache: * • The storage blob is fine (bytes preserved verbatim in `_storage`). * • The inline text cache is corrupt (replacement chars everywhere). * • The viewer that pulls from `fileContents.text` for fast paths * renders garbled mojibake until something forces a re-write. * * The agent's `commitConvexFileBytes` in chat-turn.ts has had a sibling * `detectBinaryBuffer` for a while — this file ports it verbatim so * the daemon can route binary blobs to the no-inline path. Keeping it * in its own module (no chokidar / Convex deps) means the unit tests * can pin behavior without booting the daemon. */ const BINARY_PROBE_BYTES = 4096; /** * Sniff a buffer for binary content. PNG / PDF / Office / .zip all * contain NUL bytes within the first KB; valid UTF-8 text never does. * We deliberately limit the scan to the first 4 KiB so the check stays * cheap on multi-MB blobs, and treat invalid UTF-8 in that prefix as a * binary signal too — silently round-tripping garbage Unicode through * the inline-text cache is what corrupts the rendered text for * downstream viewers. * * Returns `true` when the buffer looks binary (skip inline text cache), * `false` when it looks like clean UTF-8 text (safe to inline). */ export function detectBinaryBuffer(buffer: Uint8Array): boolean { const slice = buffer.byteLength > BINARY_PROBE_BYTES ? buffer.subarray(0, BINARY_PROBE_BYTES) : buffer; for (const byte of slice) { if (byte === 0) return true; } try { new TextDecoder("utf-8", { fatal: true }).decode(slice); return false; } catch { return true; } } /** * Convenience: decode bytes to a UTF-8 string ONLY when the buffer * looks like clean text. Returns `undefined` for binary content so * callers can pass it straight into `commitFileUpload({ text })` — * Convex schema marks `text` optional and skips the inline cache when * it's missing. */ export function decodeInlineTextOrUndefined( buffer: Uint8Array, ): string | undefined { if (detectBinaryBuffer(buffer)) return undefined; return new TextDecoder("utf-8", { fatal: false }).decode(buffer); }