/**
* Charset-aware body decoding shared by the web-access halves.
*
* HTTP lets a response name its encoding in `Content-Type`, and an HTML page can
* name one in its markup, so decoding every body as UTF-8 turns a GBK/GB18030
* page into replacement characters. `http.ts` honours the BOM and the declared
* charset; `page.ts` adds the `` fallback for HTML. An unknown
* label falls back to UTF-8 instead of failing the fetch.
*/
const FALLBACK_CHARSET = "utf-8";
/** How much of an HTML document is scanned for a `` declaration. */
const SNIFF_BYTES = 4096;
/** The `charset` parameter of a content type, lowercased. */
export function charsetOf(contentType: string): string | undefined {
const match = /;\s*charset\s*=\s*"?([^";\s]+)"?/i.exec(contentType);
return match?.[1]?.toLowerCase();
}
/** The charset a leading byte-order mark declares. */
export function bomCharset(bytes: Uint8Array): string | undefined {
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) return "utf-8";
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) return "utf-16le";
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) return "utf-16be";
return undefined;
}
/**
* The charset a document declares in its first few kilobytes. Both
* `` and ``
* are matched by the same `charset=` scan.
*/
export function sniffHtmlCharset(bytes: Uint8Array): string | undefined {
// Latin-1 keeps every byte addressable and ASCII tags intact.
const head = new TextDecoder("latin1").decode(bytes.subarray(0, SNIFF_BYTES));
const match = /]+charset\s*=\s*["']?\s*([a-z0-9_.:-]+)/i.exec(head);
return match?.[1]?.toLowerCase();
}
/** The label type `TextDecoder` accepts; a runtime label is validated by the constructor. */
type DecoderLabel = ConstructorParameters[0];
/** Decode bytes with `charset`, falling back to UTF-8 for an absent or unknown label. */
export function decodeBytes(bytes: Uint8Array, charset?: string): string {
if (!charset) return new TextDecoder(FALLBACK_CHARSET).decode(bytes);
try {
return new TextDecoder(charset as DecoderLabel).decode(bytes);
} catch {
return new TextDecoder(FALLBACK_CHARSET).decode(bytes);
}
}
/** Decode an HTML body: BOM first, then the declared charset, then its own ``, then UTF-8. */
export function decodeHtml(bytes: Uint8Array, contentType: string): string {
return decodeBytes(bytes, bomCharset(bytes) ?? charsetOf(contentType) ?? sniffHtmlCharset(bytes));
}