/** * Main-thread direct-to-S3 multipart uploader for recording streams (the * `s3` storage driver). The drop-in counterpart to {@link ChunkUploader}: * it exposes the SAME `upload(seq, blob)` / `drain()` / `stop()` surface, so * the observer wiring doesn't change — only which uploader the client * constructs, chosen by the server's advertised direct-upload support. * * Why this is NOT just "POST each chunk to S3": * * S3 multipart has hard rules the naive mapping violates. Every part * except the last must be >= 5 MB, and CompleteMultipartUpload only * concatenates the parts it's given. A MediaRecorder timeslice is way * under 5 MB, so we CANNOT make one part per timeslice. Instead we BUFFER * incoming blobs into stable parts (≥ the S3 5 MB floor) and PUT those. The server owns * the manifest + reconciles with S3 `ListParts` on finalize, so: * - parts upload strictly IN ORDER (contiguous part numbers — the server * completes only the longest gap-free prefix from part 1), * - our `ack` is best-effort (a lost ack is recovered by the server's * ListParts reconcile), * - we do NOT complete on stop()/abort — the server's finalize-on- * abandon path recovers whatever reached S3. We only `complete` on a * graceful `drain()`. * * Lifecycle: * upload(seq, blob) → append to a byte buffer. When buffered >= the part * size, cut a part + enqueue it. One part PUT in flight at a time. * first part → lazy `start` (gets mediaUploadId + uploadId). * each part → `sign` → PUT to S3 (capture ETag) → `ack`. Retry 5xx / * network with backoff; a 4xx fails the stream (no point retrying). * drain() → flush the final (possibly < 5 MB) part, await the queue, * then `complete` (server assembles the contiguous prefix → derived). * stop() → abort in-flight + clear; never completes. * * Durability (optional {@link MultipartUploaderConfig.durability}): * The in-memory buffer is volatile — a page refresh / crash discards the * un-flushed sub-part-size tail (the reported data-loss bug). When a * {@link DurableBlobStore} is injected, every incoming blob is also * persisted; on a fresh instance for the same (session, kind) — i.e. after * a refresh — we `restore()` the un-committed tail (sliced at the server's * authoritative resume point, `(nextPartNumber - 1) * partSize`) into the * front of the buffer BEFORE any live blob, and PUT it into the SAME * resumed multipart. Acked parts `commit()` (GC); a graceful `complete()` * `clear()`s the store. */ import type { DurableBlobStore } from "./durable-blob-store.js"; import { CONTROL_REQUEST_TIMEOUT_MS, RequestDeadlineError, mediaRequestTimeoutMs, withRequestDeadline, } from "../internal/request-deadline.js"; export interface MultipartUploaderConfig { sessionId: string; /** Recording stream this uploads — picks the asset + object key server-side. */ kind: "screen" | "webcam"; /** * The events ingest URL (eg. `http://localhost:3001/ingest`). The uploader * derives the origin from this and mounts the `/uploads/multipart/*` * endpoints on it — same origin as the events + legacy recording paths. */ ingestUrl: string; /** Public app key (`pk_live_…`) for org routing (x-app-id). */ appId?: string; /** Per-device fingerprint; threaded to the server so it can attribute the stream. */ fingerprintId?: string; /** MIME type of the recording (`video/webm` / `video/mp4`); used at start. */ mimeType?: string; /** * Target part size in bytes. Buffered blobs are cut into parts at this * threshold. Must be >= the S3 5 MB minimum, which is also the DEFAULT: * we deliberately sit at the floor so the most a hard crash can lose is the * un-flushed sub-5 MB buffer (~1–2 min at typical recording bitrates), * rather than a larger window. The final part (flushed on drain) may be * smaller — that's allowed for the last part. */ partSizeBytes?: number; /** * Max bytes held in memory (buffer + the enqueued, not-yet-PUT parts). * When exceeded the OLDEST enqueued part is dropped — which breaks * contiguity, so the server will complete only up to the gap. This is the * overflow backstop for a stalled network, mirroring ChunkUploader's cap. * Default 64 MB. */ maxBufferBytes?: number; /** Per-part retry budget for 5xx / network errors. Default 5. */ maxRetries?: number; /** * Optional durable buffer. When present, incoming blobs are persisted and * the un-committed tail is restored on a resumed (post-refresh) instance so * the pre-refresh recording isn't lost. Injected already-opened by the * caller; the uploader `commit`s on ack and `clear`s on a graceful complete * but never opens/closes it (the client owns its lifecycle + orphan sweep). */ durability?: DurableBlobStore; /** Called when a part is accepted by S3 (after a successful PUT + ack). */ onPartUploaded?: (partNumber: number, byteSize: number) => void; /** Called when the whole stream finalizes (`complete` returned). */ onCompleted?: (status: string) => void; /** Called when a part (and thus the stream's contiguity) is dropped. */ onDropped?: ( partNumber: number, reason: MultipartDropReason, detail?: MultipartDropDetail, ) => void; } export type MultipartDropReason = "max-retries" | "rejected" | "overflow" | "aborted"; export interface MultipartDropDetail { httpStatus?: number; serverError?: string; statusText?: string; retryCount?: number; message?: string; timedOut?: boolean; timeoutMs?: number; } interface PendingPart { partNumber: number; blob: Blob; attemptCount: number; } interface ResolvedConfig { sessionId: string; kind: "screen" | "webcam"; baseUrl: string; appId: string | undefined; fingerprintId: string | undefined; mimeType: string | undefined; partSizeBytes: number; maxBufferBytes: number; maxRetries: number; onPartUploaded: ((partNumber: number, byteSize: number) => void) | undefined; onCompleted: ((status: string) => void) | undefined; onDropped: | ((partNumber: number, reason: MultipartDropReason, detail?: MultipartDropDetail) => void) | undefined; durability: DurableBlobStore | undefined; } // S3's hard floor for a non-final part. We never cut a part below this. const S3_MIN_PART_BYTES = 5 * 1024 * 1024; export class MultipartUploader { private readonly config: ResolvedConfig; // Incoming-blob buffer, accumulated until it reaches a part's worth. private buffer: Blob[] = []; private bufferBytes = 0; // Parts cut from the buffer, awaiting (in-order) PUT. private readonly queue: PendingPart[] = []; private queuedBytes = 0; private nextPartNumber = 1; // Server manifest handle, resolved lazily on the first upload(). Parts are // only cut after the start handshake settles, so the resume part number // (for a refresh/reconnect) is known before any part is numbered. private mediaUploadId: string | null = null; private startPromise: Promise | null = null; private startResolved = false; private inFlight = false; private currentAbort: AbortController | null = null; private readonly lifecycleAbort = new AbortController(); private stopped = false; private failed = false; private recordingStartedAt: number | null = null; // Durability: the resume rehydration (started in the constructor when a // durable store is present), and the highest contiguously-acked part — its // byte end (`part * partSize`) is the GC watermark handed to the store. private restorePromise: Promise | null = null; private restored = false; private highestAckedPart = 0; constructor(config: MultipartUploaderConfig) { this.config = resolveConfig(config); if (this.config.durability) { this.restorePromise = this.restoreFromDurable(); } } /** * Resume path: settle the multipart start (to learn the server's part * number), then pull the un-committed tail from the durable store and * prepend it to the buffer so it uploads BEFORE any live blob. Best-effort: * on any failure we proceed with whatever's live. Draining stays gated on * this promise so live blobs never jump ahead of the restored tail. */ private async restoreFromDurable(): Promise { try { await this.ensureStarted(); this.startResolved = true; const committedBytes = (this.nextPartNumber - 1) * this.config.partSizeBytes; const tail = await this.config.durability!.restore(committedBytes); for (let i = tail.length - 1; i >= 0; i -= 1) { const b = tail[i]!; if (b.size === 0) continue; this.buffer.unshift(b); this.bufferBytes += b.size; } } catch { // Degrade to live-only; the store's own failures are already contained. } finally { this.restored = true; this.drainBufferIntoParts(); } } /** * Append a recorder blob. Returns immediately. Cuts and enqueues a part * whenever the buffer reaches the target part size; the actual S3 PUT * happens on the internal worker loop. */ upload( _seq: number, blob: Blob, timingHeaders?: Record, ): void { if (this.stopped || this.failed) return; if (blob.size === 0) return; if (this.recordingStartedAt === null) { const raw = timingHeaders?.["x-recording-started-at"]; const parsed = raw === undefined ? Number.NaN : Number(raw); if (Number.isFinite(parsed) && parsed >= 0) { this.recordingStartedAt = Math.trunc(parsed); } } // Persist the LIVE blob first so a refresh in the next instant still has // it. Restored blobs are already in the store, so they're never re-added. this.config.durability?.append(blob); this.buffer.push(blob); this.bufferBytes += blob.size; // While the durable tail is still being restored, hold off cutting parts // so live blobs can't be numbered ahead of the restored (earlier) bytes. if (this.restorePromise && !this.restored) { void this.restorePromise.then(() => this.drainBufferIntoParts()); return; } // Resolve the multipart start (and thus the resume part number) BEFORE // cutting any part — otherwise a part could be numbered from 1 while the // server is about to tell us to resume from N, overwriting prior parts. // The handshake settles in a few hundred ms, far below the seconds it // takes video to accumulate one 5 MB part, so this never stalls uploads. if (!this.startResolved) { void this.ensureStarted() .then(() => { this.startResolved = true; this.drainBufferIntoParts(); }) .catch(() => undefined); return; } this.drainBufferIntoParts(); } /** Cut as many full parts as the buffer allows, then pump the queue. */ private drainBufferIntoParts(): void { if (this.stopped || this.failed) return; while (this.bufferBytes >= this.config.partSizeBytes) { this.cutPart(this.config.partSizeBytes); } this.enforceBufferCap(); void this.processNext(); } /** * Graceful end: flush whatever's buffered as a final part (allowed to be * < 5 MB), wait for every enqueued part to reach S3, then `complete` so * the server assembles the contiguous prefix into the derived recording. * * Resolves once complete returns (or there was nothing to upload). * Rejects on timeout — the caller can then `stop()` and accept that the * server's abandon-finalize path recovers whatever reached S3. */ async drain(timeoutMs = 60_000): Promise { if (this.stopped || this.failed) return; try { await withRequestDeadline("multipart-drain", timeoutMs, (signal) => this.drainWithin(signal)); } catch (error) { if (error instanceof RequestDeadlineError) { this.lifecycleAbort.abort(); this.currentAbort?.abort(); } throw error; } } private async drainWithin(signal: AbortSignal): Promise { // Let the durable-tail restore settle first, so a graceful end still // flushes recovered pre-refresh bytes (a resume that ends immediately). if (this.restorePromise && !this.restored) { await this.restorePromise.catch(() => undefined); } // Nothing was ever buffered or sent → no upload to finalize. if (!this.startResolved && this.bufferBytes === 0 && this.queue.length === 0) { return; } // Resolve start first so the tail part gets the correct (possibly resumed) // number before we cut it. if (this.bufferBytes > 0 || this.queue.length > 0) { await this.ensureStarted(signal); this.startResolved = true; } // Flush the tail: the remaining buffer becomes the final part. Unlike a // mid-stream cut, this is allowed to be below the 5 MB floor. if (this.bufferBytes > 0) this.cutPart(this.bufferBytes); void this.processNext(); while (this.inFlight || this.queue.length > 0) { if (this.stopped || this.failed) return; if (signal.aborted) throw abortError(); await sleepSignal(50, signal); } // Nothing ever uploaded → nothing to complete (no manifest started). if (!this.mediaUploadId) return; const status = await this.complete(signal); this.config.onCompleted?.(status); if (status === "deferred" || status === "deferred-timeout") return; // Bytes are safely assembled in S3 now — drop the durable buffer so it // doesn't linger for the orphan sweep. await this.config.durability?.clear().catch(() => undefined); } /** * Abort the in-flight PUT and clear everything. Does NOT complete the * multipart upload — the server's finalize-on-abandon recovers whatever * parts reached S3. Idempotent. */ stop(): void { if (this.stopped) return; this.stopped = true; this.lifecycleAbort.abort(); this.currentAbort?.abort(); this.currentAbort = null; for (const p of this.queue) { this.config.onDropped?.(p.partNumber, "aborted", { message: "uploader stopped before part uploaded", }); } this.queue.length = 0; this.queuedBytes = 0; this.buffer = []; this.bufferBytes = 0; } // ── internals ───────────────────────────────────────────────────────── /** Slice `bytes` off the front of the buffer into a new enqueued part. */ private cutPart(bytes: number): void { const take: Blob[] = []; let taken = 0; while (taken < bytes && this.buffer.length > 0) { const head = this.buffer[0]!; const need = bytes - taken; if (head.size <= need) { take.push(head); taken += head.size; this.buffer.shift(); } else { // Split the head blob: first `need` bytes complete this part, the // remainder stays buffered. take.push(head.slice(0, need)); this.buffer[0] = head.slice(need); taken += need; } } this.bufferBytes -= taken; const blob = take.length === 1 ? take[0]! : new Blob(take); this.queue.push({ partNumber: this.nextPartNumber++, blob, attemptCount: 0 }); this.queuedBytes += blob.size; } /** * Memory backstop: while the buffer + enqueued parts exceed the cap, drop * the OLDEST enqueued part. Dropping a part creates a gap, so the server * will complete only up to the part before it — we surface that as a drop. */ private enforceBufferCap(): void { while ( this.queuedBytes + this.bufferBytes > this.config.maxBufferBytes && this.queue.length > 0 ) { const dropped = this.queue.shift()!; this.queuedBytes -= dropped.blob.size; this.config.onDropped?.(dropped.partNumber, "overflow", { message: "buffered media exceeded maxBufferBytes", }); } } /** Pull the next part and PUT it; re-runs until the queue drains. */ private async processNext(): Promise { if (this.inFlight || this.stopped || this.failed) return; const next = this.queue[0]; if (!next) return; this.inFlight = true; try { await this.sendPart(next); } finally { this.inFlight = false; if (!this.stopped && !this.failed) void this.processNext(); } } /** * Lazily start the multipart upload on the server (once), returning the * manifest handle. Concurrency-safe via a memoised promise. */ private async ensureStarted(signal?: AbortSignal): Promise { if (this.mediaUploadId) return; if (!this.startPromise) { const startPromise = (async () => { const json = await this.postJson<{ mediaUploadId: string; uploadId: string; nextPartNumber: number; }>( "multipart-start", "/uploads/multipart/start", { sessionId: this.config.sessionId, kind: this.config.kind, ...(this.config.fingerprintId ? { fingerprintId: this.config.fingerprintId } : {}), ...(this.config.mimeType ? { mimeType: this.config.mimeType } : {}), ...(this.recordingStartedAt !== null ? { recordingStartedAt: this.recordingStartedAt } : {}), }, signal ?? this.lifecycleAbort.signal, ); this.mediaUploadId = json.mediaUploadId; // Resume: on a refresh/reconnect the server returns the SAME open // multipart upload and the next part number to write. We MUST continue // numbering from there — restarting at 1 would overwrite the parts a // previous page already PUT, corrupting the assembled recording. Only // advance (never rewind) so a fresh upload still starts at 1, and any // parts this instance already cut keep their higher numbers. if (typeof json.nextPartNumber === "number" && json.nextPartNumber > this.nextPartNumber) { this.nextPartNumber = json.nextPartNumber; } })(); this.startPromise = startPromise; void startPromise.catch(() => { if (this.startPromise === startPromise) this.startPromise = null; }); } await this.startPromise; } private async sendPart(part: PendingPart): Promise { const controller = new AbortController(); this.currentAbort = controller; let lastTimeout: RequestDeadlineError | undefined; while (part.attemptCount <= this.config.maxRetries) { if (this.stopped || this.failed) return; try { await this.ensureStarted(); const mediaUploadId = this.mediaUploadId!; // 1) Presign this part. const { url } = await this.postJson<{ url: string }>( "multipart-sign", "/uploads/multipart/sign", { sessionId: this.config.sessionId, mediaUploadId, partNumber: part.partNumber, sizeBytes: part.blob.size, }, controller.signal, ); // 2) PUT the bytes straight to S3. The ETag on the response is what // the server needs to complete the part. const putRes = await withRequestDeadline( "multipart-part-upload", mediaRequestTimeoutMs(part.blob.size), (signal) => fetch(url, { method: "PUT", body: part.blob, signal, credentials: "omit", }), controller.signal, ); if (!putRes.ok) { lastTimeout = undefined; // S3 5xx → retry; 4xx → the presign/part is bad, fail the stream. if ( putRes.status >= 400 && putRes.status < 500 && putRes.status !== 408 && putRes.status !== 429 ) { this.failStream(part.partNumber, putRes.status, putRes.statusText); return; } part.attemptCount += 1; await this.backoff(part.attemptCount, controller.signal); continue; } const etag = putRes.headers.get("etag") ?? putRes.headers.get("ETag") ?? ""; // 3) Ack (best-effort: the server reconciles with ListParts anyway). await this.postJson( "multipart-ack", "/uploads/multipart/ack", { sessionId: this.config.sessionId, mediaUploadId, partNumber: part.partNumber, etag, byteSize: part.blob.size, }, controller.signal, ).catch(() => undefined); // Done — dequeue and report. this.queue.shift(); this.queuedBytes -= part.blob.size; this.currentAbort = null; this.config.onPartUploaded?.(part.partNumber, part.blob.size); // GC the durable store up to this part's byte end. Every non-final // part is exactly partSize, so part N ends at N * partSize — the same // absolute offset space the store records against. this.highestAckedPart = Math.max(this.highestAckedPart, part.partNumber); void this.config.durability?.commit(this.highestAckedPart * this.config.partSizeBytes); return; } catch (err) { lastTimeout = err instanceof RequestDeadlineError ? err : undefined; if ((err as Error).name === "AbortError") { this.config.onDropped?.(part.partNumber, "aborted", { retryCount: part.attemptCount, message: "request aborted", }); this.currentAbort = null; return; } // A server sign/start 4xx surfaces here as a thrown HttpError. const status = (err as { httpStatus?: number }).httpStatus; if (status && status >= 400 && status < 500 && status !== 408 && status !== 429) { this.failStream(part.partNumber, status, (err as Error).message); return; } part.attemptCount += 1; await this.backoff(part.attemptCount, controller.signal); } } // Retry budget exhausted — drop this part (breaks contiguity here). this.queue.shift(); this.queuedBytes -= part.blob.size; this.config.onDropped?.(part.partNumber, "max-retries", { retryCount: part.attemptCount, message: lastTimeout ? "part upload timed out and exhausted retry budget" : "part upload exhausted retry budget", ...(lastTimeout ? { timedOut: true, timeoutMs: lastTimeout.timeoutMs } : {}), }); this.currentAbort = null; } private async complete(signal: AbortSignal): Promise { for (let attempt = 1; attempt <= 2; attempt += 1) { try { const json = await this.postJson<{ status: string }>( "multipart-complete", "/uploads/multipart/complete", { sessionId: this.config.sessionId, mediaUploadId: this.mediaUploadId, }, signal, ); return json.status; } catch (error) { const status = (error as { httpStatus?: number }).httpStatus; const retryable = error instanceof RequestDeadlineError || status === undefined || status === 408 || status === 429 || status >= 500; if (attempt === 1 && retryable && !signal.aborted) { await sleepSignal(500, signal); continue; } // Complete is best-effort from the client — the server's abandon // finalize recovers the upload if this never lands. return error instanceof RequestDeadlineError ? "deferred-timeout" : "deferred"; } } return "deferred"; } /** A 4xx anywhere means retrying is pointless; stop the whole stream. */ private failStream(partNumber: number, status: number, statusText?: string): void { this.failed = true; this.currentAbort = null; this.config.onDropped?.(partNumber, "rejected", { httpStatus: status, ...(statusText ? { statusText } : {}), }); } private async backoff(attempt: number, signal: AbortSignal): Promise { if (attempt > this.config.maxRetries) return; const delayMs = Math.min(1_000 * 2 ** (attempt - 1), 16_000); await sleepSignal(delayMs, signal); } /** POST JSON to a server endpoint; throws HttpError on non-2xx. */ private async postJson( operation: string, path: string, body: unknown, signal?: AbortSignal, ): Promise { const headers: Record = { "content-type": "application/json" }; if (this.config.appId) headers["x-app-id"] = this.config.appId; if (this.config.fingerprintId) { headers["x-fingerprint-id"] = this.config.fingerprintId; } return withRequestDeadline( operation, CONTROL_REQUEST_TIMEOUT_MS, async (requestSignal) => { const res = await fetch(`${this.config.baseUrl}${path}`, { method: "POST", headers, body: JSON.stringify(body), credentials: "omit", signal: requestSignal, }); if (!res.ok) { const err = new Error(`${path} → ${res.status}`) as Error & { httpStatus: number; }; err.httpStatus = res.status; throw err; } return (await res.json()) as T; }, signal, ); } } function resolveConfig(raw: MultipartUploaderConfig): ResolvedConfig { // Default to the S3 5 MB floor: minimizes the un-flushed buffer at risk on // a hard crash. A larger part size is more efficient but loses more on // crash; callers can raise it if they prefer fewer, larger PUTs. const partSize = Math.max(raw.partSizeBytes ?? S3_MIN_PART_BYTES, S3_MIN_PART_BYTES); return { sessionId: raw.sessionId, kind: raw.kind, baseUrl: stripTrailingPath(raw.ingestUrl), appId: raw.appId, fingerprintId: raw.fingerprintId, mimeType: raw.mimeType, partSizeBytes: partSize, maxBufferBytes: Math.max(raw.maxBufferBytes ?? 64 * 1024 * 1024, partSize * 2), maxRetries: raw.maxRetries ?? 5, onPartUploaded: raw.onPartUploaded, onCompleted: raw.onCompleted, onDropped: raw.onDropped, durability: raw.durability, }; } /** Origin of an events ingest URL (`https://api/ingest` → `https://api`). */ function stripTrailingPath(url: string): string { try { const u = new URL(url); return `${u.protocol}//${u.host}`; } catch { return url.replace(/\/[^/]*$/, ""); } } function sleepSignal(ms: number, signal: AbortSignal): Promise { return new Promise((resolve) => { if (signal.aborted) { resolve(); return; } const timer = setTimeout(() => { signal.removeEventListener("abort", onAbort); resolve(); }, ms); const onAbort = (): void => { clearTimeout(timer); resolve(); }; signal.addEventListener("abort", onAbort, { once: true }); }); } function abortError(): Error { const error = new Error("The request was aborted"); error.name = "AbortError"; return error; }