/**
* The client half of the Oxy API's large uploads (ADR 0043 stage 4b), for
* `OxyServer.assets.uploadAppFileLarge` / `uploadUserMediaLarge`.
*
* hash SHA-256 of the source, read 8 MiB at a time (never whole in memory)
* open POST → session (or the file, if already held)
* parts PUT each part to its presigned URL, `partConcurrency` at a time,
* each retried; URLs are fetched 100 at a time, just in time
* complete POST /:uploadId/complete → the file, or 202 `verifying`
* poll GET /:uploadId until `completed` or `failed`
*
* Resume is free: opening the same bytes again answers the open session with
* the parts S3 already holds, so a process that crashed mid-upload sends only
* what is missing when it calls again.
*/
import { createHash } from 'node:crypto';
import type { ServiceLane } from '../client/context';
import type { AudioRenditionOptions, FileVisibility } from '../models/interfaces';
/** What a large upload reads from: a Blob (`fs.openAsBlob(path)` for a file), bytes, or a file path. */
export type LargeUploadSource = Blob | Uint8Array | ArrayBuffer | string;
export interface LargeUploadProgressEvent {
/** `hash` reads the source once; `upload` sends parts; `verify` waits for Oxy to hash it. */
phase: 'hash' | 'upload' | 'verify';
loaded: number;
total: number;
}
export interface LargeServiceUploadOptions {
/**
* The file's type. Executables and active documents (HTML, XML, scripts)
* are refused (415); an SVG (at most 5 MiB) is sanitized when Oxy verifies
* it, and the sanitized bytes are what is stored (ADR 0043 stage 4c).
*/
mime: string;
/** How an audio file is rendered (ignored for other types). */
audio?: AudioRenditionOptions;
/** Stored as the original name; also how a type with no known extension gets one. */
filename?: string;
metadata?: Record;
/** Register this application's use of the file once it is stored (needs `files:references:write`). */
reference?: { entityType: string; entityId: string };
/** The source's SHA-256 (lowercase hex) when already known: skips the hashing pass. */
sha256?: string;
/** Parts sent at once (default 4). */
partConcurrency?: number;
/** Attempts per part (default 4). */
partAttempts?: number;
/** How long to wait for Oxy to verify a large file after completion (default 30 min). */
verifyTimeoutMs?: number;
onProgress?: (progress: LargeUploadProgressEvent) => void;
/** Stops the upload; the session stays open (until it expires) so a later call resumes it. */
signal?: AbortSignal;
}
/** One part's presigned URL: PUT exactly `size` bytes to it. */
interface PartUrl {
partNumber: number;
url: string;
size: number;
}
/** The wire view of an upload session (`/assets/service/{app,user-media}/uploads`). */
export interface LargeUploadSession {
uploadId: string | null;
status: 'uploading' | 'verifying' | 'completed' | 'failed' | 'aborted';
size: number;
partSize?: number;
partCount?: number;
expiresAt?: string;
parts?: PartUrl[];
uploadedParts?: { partNumber: number; size: number }[];
file?: TFile;
deduplicated?: boolean;
reference?: { result: 'created' | 'exists' | 'not_found' };
error?: { code: string; message: string };
}
/** A finished large upload. */
export interface LargeServiceUploadResult {
file: TFile;
/** The owner already held these bytes: nothing was stored. */
deduplicated: boolean;
/** `null` when no upload was needed. */
uploadId: string | null;
reference?: { result: 'created' | 'exists' | 'not_found' };
}
/** A large upload that ended without a file (checksum mismatch, a takedown, an abort…). */
export class LargeUploadFailedError extends Error {
constructor(
readonly code: string,
message: string,
readonly uploadId: string | null,
) {
super(message);
this.name = 'LargeUploadFailedError';
}
}
const HASH_CHUNK_BYTES = 8 * 1024 * 1024;
const PART_URL_BATCH = 100;
const POLL_INTERVAL_MS = 2_000;
async function toBlob(source: LargeUploadSource): Promise {
if (typeof source === 'string') {
const fs = await import('node:fs');
if (typeof fs.openAsBlob !== 'function') {
throw new Error('A file path needs Node 20+ (fs.openAsBlob); pass a Blob instead');
}
return fs.openAsBlob(source);
}
if (source instanceof Blob) return source;
// A copy into a plain ArrayBuffer: a view over a SharedArrayBuffer is not a BlobPart.
const bytes = source instanceof ArrayBuffer ? new Uint8Array(source) : source;
const copy = new Uint8Array(bytes.byteLength);
copy.set(bytes);
return new Blob([copy]);
}
async function sha256Of(
blob: Blob,
onProgress: LargeServiceUploadOptions['onProgress'],
signal: AbortSignal | undefined,
): Promise {
const hash = createHash('sha256');
for (let offset = 0; offset < blob.size; offset += HASH_CHUNK_BYTES) {
signal?.throwIfAborted();
const end = Math.min(offset + HASH_CHUNK_BYTES, blob.size);
hash.update(new Uint8Array(await blob.slice(offset, end).arrayBuffer()));
onProgress?.({ phase: 'hash', loaded: end, total: blob.size });
}
return hash.digest('hex');
}
function sleep(ms: number, signal?: AbortSignal): Promise {
return new Promise((resolve, reject) => {
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
'abort',
() => {
clearTimeout(timer);
reject(signal.reason);
},
{ once: true },
);
});
}
/**
* Run one large upload against `base` (`/assets/service/app/uploads` or
* `/assets/service/user-media/uploads`) with `headers` on every API call.
*/
export async function runLargeUpload(
service: ServiceLane,
base: string,
headers: Record,
visibility: FileVisibility | undefined,
source: LargeUploadSource,
options: LargeServiceUploadOptions,
): Promise> {
const { signal, onProgress } = options;
const blob = await toBlob(source);
if (blob.size === 0) throw new Error('Cannot upload an empty file');
const sha256 = options.sha256?.toLowerCase() ?? (await sha256Of(blob, onProgress, signal));
const call = (method: 'GET' | 'POST', path: string, body?: unknown): Promise =>
service.request(method, path, body, {
headers,
cache: false,
...(signal ? { signal } : {}),
});
let session = await call>('POST', base, {
sha256,
size: blob.size,
mime: options.mime,
...(visibility ? { visibility } : {}),
...(options.filename ? { filename: options.filename } : {}),
...(options.metadata ? { metadata: options.metadata } : {}),
...(options.reference ? { reference: options.reference } : {}),
...(options.audio ? { audio: options.audio } : {}),
});
if (session.status === 'uploading' && session.uploadId) {
const uploadId = session.uploadId;
const path = `${base}/${encodeURIComponent(uploadId)}`;
const partSize = session.partSize ?? blob.size;
const partCount = session.partCount ?? 1;
const urls = new Map((session.parts ?? []).map((part) => [part.partNumber, part]));
const sendParts = async (held: Set): Promise => {
const pending: number[] = [];
for (let part = 1; part <= partCount; part++) if (!held.has(part)) pending.push(part);
let sent = [...held].reduce(
(bytes, part) => bytes + Math.min(partSize, blob.size - (part - 1) * partSize),
0,
);
onProgress?.({ phase: 'upload', loaded: sent, total: blob.size });
const urlFor = async (partNumber: number, refresh = false): Promise => {
const known = urls.get(partNumber);
if (known && !refresh) return known;
const batch = pending.filter((part) => part >= partNumber).slice(0, PART_URL_BATCH);
const answer = await call>('POST', `${path}/parts`, {
partNumbers: batch.includes(partNumber) ? batch : [partNumber],
});
for (const part of answer.parts ?? []) urls.set(part.partNumber, part);
const fresh = urls.get(partNumber);
if (!fresh) throw new Error(`No upload URL for part ${partNumber}`);
return fresh;
};
const attempts = Math.max(1, options.partAttempts ?? 4);
const putPart = async (partNumber: number): Promise => {
const start = (partNumber - 1) * partSize;
const body = blob.slice(start, Math.min(start + partSize, blob.size));
let refresh = false;
for (let attempt = 1; ; attempt++) {
signal?.throwIfAborted();
const target = await urlFor(partNumber, refresh);
if (new URL(target.url).protocol !== 'https:') {
throw new Error('Part uploads require HTTPS');
}
try {
const response = await fetch(target.url, {
method: 'PUT',
body,
credentials: 'omit',
redirect: 'error',
...(signal ? { signal } : {}),
});
if (response.ok) return;
// An expired or refused signature: ask for a new URL.
refresh = response.status === 403;
if (attempt >= attempts) {
throw new Error(`Part ${partNumber} upload failed (${response.status})`);
}
} catch (error) {
if (signal?.aborted || attempt >= attempts) throw error;
}
await sleep(500 * 2 ** (attempt - 1), signal);
}
};
let next = 0;
const worker = async () => {
while (next < pending.length) {
const partNumber = pending[next++];
await putPart(partNumber);
sent += Math.min(partSize, blob.size - (partNumber - 1) * partSize);
onProgress?.({ phase: 'upload', loaded: sent, total: blob.size });
}
};
const concurrency = Math.max(1, Math.min(options.partConcurrency ?? 4, pending.length));
await Promise.all(Array.from({ length: concurrency }, worker));
};
await sendParts(new Set((session.uploadedParts ?? []).map((part) => part.partNumber)));
try {
session = await call>('POST', `${path}/complete`);
} catch (error) {
// A part S3 does not hold after all (lost, or the wrong size): send what
// the session says is missing, once, and complete again.
const status = (error as { status?: number; response?: { status?: number } })?.status;
if (status !== 409) throw error;
const state = await call>('GET', path);
if (state.status !== 'uploading') throw error;
urls.clear();
await sendParts(new Set((state.uploadedParts ?? []).map((part) => part.partNumber)));
session = await call>('POST', `${path}/complete`);
}
}
if (session.status === 'verifying' && session.uploadId) {
const path = `${base}/${encodeURIComponent(session.uploadId)}`;
const deadline = Date.now() + (options.verifyTimeoutMs ?? 30 * 60_000);
while (session.status === 'verifying') {
onProgress?.({ phase: 'verify', loaded: 0, total: blob.size });
if (Date.now() > deadline) {
throw new LargeUploadFailedError(
'VERIFY_TIMEOUT',
'Oxy is still verifying the upload; call again later to collect it',
session.uploadId,
);
}
await sleep(POLL_INTERVAL_MS, signal);
session = await call>('GET', path);
}
}
if (session.status !== 'completed' || !session.file) {
throw new LargeUploadFailedError(
session.error?.code ?? 'UPLOAD_FAILED',
session.error?.message ?? `The upload ended ${session.status}`,
session.uploadId,
);
}
onProgress?.({ phase: 'verify', loaded: blob.size, total: blob.size });
return {
file: session.file,
deduplicated: session.deduplicated ?? false,
uploadId: session.uploadId,
...(session.reference ? { reference: session.reference } : {}),
};
}