import { parseContentRangeSize, readBody, splitReadFileOptions, toBytes, toBytesWithProgress, } from './util.ts' import type { BufferEncoding, Fetcher, FilehandleOptions, GenericFilehandle, ProgressCallback, ReadFileOptions, ReadFileTextOptions, Stats, } from './filehandle.ts' // header names are case-insensitive but object keys are not, so a caller's // `Range` and the `range` a ranged GET adds are two keys — and fetch folds them // into one comma-joined multi-range request, whose multipart/byteranges body we // would hand back as file bytes. The same folding joins a constructor's // `Authorization` to a per-call `authorization`. A later source wins, however // either one spelled the name. function mergeHeaders(...sources: (Record | undefined)[]) { const merged = new Map() for (const source of sources) { for (const [name, value] of Object.entries(source ?? {})) { merged.set(name.toLowerCase(), [name, value]) } } return Object.fromEntries(merged.values()) } // under node, a body nobody reads holds its connection until it is collected. // A custom fetch may return a Response-like whose body has no cancel(). function discardBody(res: Response) { const body = res.body as Partial | null if (typeof body?.cancel === 'function') { body.cancel().catch(() => undefined) } } function isByteOffset(n: number) { return Number.isSafeInteger(n) && n >= 0 } // fetch reports a request that got no response as a bare TypeError naming no // URL. Matched by name, since a custom fetch can throw one from another realm. function isNetworkFailure(e: unknown) { return ( typeof e === 'object' && e !== null && 'name' in e && e.name === 'TypeError' ) } function getMessage(e: unknown) { const r = typeof e === 'object' && e !== null && 'message' in e && typeof e.message === 'string' ? e.message : `${e}` // strip trailing period so the wrapped form `${msg} fetching ${url}` reads cleanly return r.replace(/\.$/, '') } export default class RemoteFile implements GenericFilehandle { protected url: string /** the URL this handle fetches — see {@link GenericFilehandle.source} */ public get source() { return this.url } private _stat?: Stats private statProbe?: Promise private fetchImplementation: Fetcher private baseHeaders: Record private baseOverrides: Omit private baseSignal?: AbortSignal public constructor(source: string, opts: FilehandleOptions = {}) { this.url = source this.baseHeaders = opts.headers ?? {} this.baseOverrides = opts.overrides ?? {} this.baseSignal = opts.signal this.fetchImplementation = opts.fetch ?? globalThis.fetch.bind(globalThis) } protected buildRequest( opts: FilehandleOptions, extraHeaders?: Record, ): RequestInit { // a per-call signal beats the constructor's, which beats one supplied via // overrides; omit the key entirely when there is none, so we don't write // `signal: undefined` over an overrides-supplied signal const signal = opts.signal ?? this.baseSignal return { // defaults first: `overrides` is documented as extra fetch params, so a // caller passing e.g. mode/redirect/method has to be able to win method: 'GET', redirect: 'follow', mode: 'cors', ...this.baseOverrides, ...opts.overrides, headers: mergeHeaders(this.baseHeaders, opts.headers, extraHeaders), ...(signal ? { signal } : {}), } } public async fetch( input: RequestInfo, init?: RequestInit, ): Promise { const wrapError = (e: unknown) => isNetworkFailure(e) && !init?.signal?.aborted ? new Error(`${getMessage(e)} fetching ${input}`, { cause: e }) : e let response: Response try { response = await this.fetchImplementation(input, init) } catch (e) { if (`${e}`.includes('Failed to fetch')) { // refetch to help work around a chrome bug (discussed in // generic-filehandle issue #72) in which the chrome cache returns a // CORS error for content in its cache. see also // https://github.com/GMOD/jbrowse-components/pull/1511 console.warn( `generic-filehandle: refetching ${input} to attempt to work around chrome CORS header caching bug`, ) try { response = await this.fetchImplementation(input, { ...init, cache: 'reload', }) } catch (e) { throw wrapError(e) } } else { throw wrapError(e) } } return response } public async read( length: number, position: number, opts: FilehandleOptions = {}, ): Promise> { // NaN is the one that motivated this — a corrupt index yields it from // ordinary arithmetic — but a fractional or negative byte offset is just as // unsendable, and reaches the server as a range header it can only reject if (!isByteOffset(length) || !isByteOffset(position)) { throw new TypeError( `read() called with an invalid length or position (length=${length}, position=${position}). The index file may be corrupt.`, ) } return length === 0 ? new Uint8Array(0) : this.fetchBytes(length, position, opts) } /** * The bytes for one byte range — the seam a subclass that can serve those * bytes some other way overrides. * * The alternative is overriding {@link fetch}, and for a subclass holding a * byte cache that is the wrong altitude: it has bytes, so it has to wrap them * in a `Response` that `read` immediately unwraps again. Both halves copy. * Measured against JBrowse's `RemoteFileWithRangeCache` on a fully warm cache * with no network at all, that round trip was **69-77% of the entire read** — * 6.15ms vs 1.90ms for 16MB, and the same ratio down to 256KB. * * So: override this to serve bytes, override {@link fetch} to change how * requests are made. Overriding neither leaves the ranged GET below, and the * default implementation still goes through `this.fetch`, so a subclass that * only wraps `fetch` keeps working untouched. */ protected async fetchBytes( length: number, position: number, opts: FilehandleOptions, ): Promise> { const res = await this.fetch( this.url, this.buildRequest(opts, { range: `bytes=${position}-${position + length - 1}`, }), ) // HTTP 416 Range Not Satisfiable: the requested range starts past EOF. // Translate to an empty read so callers can detect EOF via short/empty // returns instead of needing a separate size oracle (stat) to stay clear of // the end of the file. if (res.status === 416) { discardBody(res) return new Uint8Array(0) } this.checkOk(res) if ((res.status === 200 && position === 0) || res.status === 206) { // try to parse out the size of the remote file const size = parseContentRangeSize(res.headers.get('content-range')) if (size !== undefined) { this._stat = { size } } const resData = opts.onProgress ? await toBytesWithProgress(res, opts.onProgress) : await toBytes(res) // server didn't honor the range request and returned the full file — // the body length is the actual file size if (!this._stat && res.status === 200) { this._stat = { size: resData.byteLength } } // the server over-delivered (it ignored our range header and sent the // whole file). copy out the requested slice rather than returning a // subarray view, which would pin the entire body in memory for as long // as the caller holds those few bytes. return resData.byteLength <= length ? resData : resData.slice(0, length) } discardBody(res) throw new Error( res.status === 200 ? `${this.url} fetch returned status 200, expected 206` : `HTTP ${res.status} fetching ${this.url}`, ) } private checkOk(res: Response) { if (!res.ok) { discardBody(res) throw new Error(`HTTP ${res.status} fetching ${this.url}`) } } public async readFile( options?: ReadFileOptions, ): Promise> public async readFile(options: ReadFileTextOptions): Promise public async readFile( options?: FilehandleOptions | BufferEncoding, ): Promise | string> { const { encoding, opts } = splitReadFileOptions(options) const res = await this.fetch(this.url, this.buildRequest(opts)) this.checkOk(res) const body = await this.readFileBody(res, encoding, opts.onProgress) // a 200 means we hold the entire file, so its length is the file size — // record it so a subsequent stat() doesn't need another request. a 206 // (caller supplied their own range header) tells us nothing. if (res.status === 200 && typeof body !== 'string') { this._stat = { size: body.byteLength } } return body } /** * Reads the body of a whole-file response. A subclass overrides it to learn * when that body starts and ends, which `fetch` cannot tell it: `fetch` * returns at the headers. */ protected readFileBody( res: Response, encoding: BufferEncoding | undefined, onProgress?: ProgressCallback, ) { return readBody(res, encoding, onProgress) } public async stat(): Promise { if (!this._stat) { // share one probe between concurrent stat() callers instead of each // firing its own request; cleared afterwards so a failed probe retries this.statProbe ??= this.read(10, 0).finally(() => { this.statProbe = undefined }) await this.statProbe } // Content-Range may not be exposed due to CORS — return size 0 rather // than crashing so callers can degrade gracefully. return this._stat ?? { size: 0 } } public close(): Promise { return Promise.resolve() } }