// `` discovery for the webfetch thin-extraction fallback.
//
// When the extractor + pandoc/w3m pipeline produces thin output (typically
// because the page is a JS-rendered shell or a localized interstitial), a
// useful clean alternative is often advertised right there in via the
// W3C-standard alternate-link discovery mechanism (oEmbed et al.). Following
// it costs us one HTTP round-trip and yields title/author/description for an
// entire class of sites — without a single line of per-host code.
//
// Allowlist, not blocklist: unknown `type` values are skipped silently. New
// media types are added in follow-up PRs with evidence.
//
// Explicitly excluded:
// - application/rss+xml, application/atom+xml — feeds OF other content,
// not alternate representations of the current page.
// - — same shell page in another
// viewport.
// - android-app:// / ios-app:// hrefs — not http(s).
// - / "amphtml" — out of scope (see issue #128
// non-goals).
export const ALLOWED_ALTERNATE_TYPES: ReadonlySet = new Set([
// oEmbed standard JSON form. Highest signal: YouTube, Vimeo, Flickr,
// WordPress.com, Substack, Spotify, DeviantArt, Slideshare.
"application/json+oembed",
// oEmbed XML form — same providers, less common.
"application/xml+oembed",
// Real-world variants verified against live HTML during PR #134 review:
// - YouTube ships `text/xml+oembed` (NOT `application/xml+oembed`)
// - SoundCloud ships both `text/xml+oembed` and `text/json+oembed`
// The oEmbed spec itself only blesses the `application/*+oembed` forms,
// but allowlisting the `text/*+oembed` siblings costs nothing (the
// `+oembed` suffix is the discriminating signal, not the top-level type)
// and unlocks the motivating providers. Anything without the literal
// `+oembed` suffix — plain `application/json`, `text/xml` — stays out:
// those are catch-all generic-data hints with no contract about content.
"text/xml+oembed",
"text/json+oembed",
// Author-served markdown alternate. Rare, but unambiguous when present.
"text/markdown",
]);
export interface Alternate {
/** href as written in the link tag — may be absolute or relative. */
url: string;
/** type attribute, lowercased and trimmed. */
type: string;
/** title attribute if present (some sites use it as a human-readable label). */
title?: string;
}
// Scope the scan to .... Fallbacks in priority order:
// 1. closer (well-formed)
// 2. closer at all and no either; the
// no--at-all case is caught earlier by the outer
// `if (!headMatch)` guard, which also rejects HTML where the entire
// open tag is missing)
//
// Case (3) is intentional even though it can over-scan a truly malformed
// page (e.g. a fragment response with `` opened but neither closed
// nor followed by ``): the alternative — refusing to scan such
// pages — silently regresses the YouTube/Substack/etc. happy path the
// instant a server ships a non-conforming variant. Any `` we surface from the over-scanned tail still has to pass
// the allowlist, same-origin, and SSRF gates at the call site, so the
// blast radius of "we treated more bytes as than the spec allows"
// is bounded.
//
// Without this scope a stray in (some sites
// inject in-content video embeds with link tags) would be picked up too,
// against the spec — alternate-link discovery is a -only mechanism.
// Known limitation: HEAD_RE / LINK_RE both terminate the tag at the first
// `>`, which is wrong for the (rare but legal) case of an attribute value
// containing a literal `>` inside quotes — e.g. ``.
// HTML5 doesn't actually require quoting `>` in attribute values, so this
// is technically a parser bug. In practice it's a non-issue: the only
// allowlisted attribute we read is `href` (URL-escaped) and `type` (a
// media-type token); a stray `>` would only ever land in `title`, and the
// truncated tag we'd then process still has well-formed `rel`/`type`/`href`
// attributes if any followed the title. Documenting rather than fixing
// because a real HTML tokenizer here would be wildly out of scope.
// Same anti-substring-match rationale as the meta-charset sniffer in
// webfetch.ts — keep these two regexes in sync if either grows attribute-
// syntax support.
const HEAD_RE = /]*>([\s\S]*?)(?:<\/head\s*>|]*)>/gi;
// Same attribute tokenizer as the meta-charset sniffer in webfetch.ts.
const ATTR_RE = /([A-Za-z_:][A-Za-z0-9_.:-]*)\s*(?:=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>`]+)))?/g;
/**
* Scan the of `html` and return every with an
* allowlistable shape (has `type`, has `href`, no `media`, http/https or
* relative `href`). Type filtering against ALLOWED_ALTERNATE_TYPES happens
* at the call site so callers can also see denied entries in tests/logs.
*
* Order matches document order. First-match-wins is a caller policy.
*
* Pure: no I/O, no side effects. Unit-testable without mocks.
*/
export function findAlternates(html: string): Alternate[] {
const headMatch = HEAD_RE.exec(html);
if (!headMatch) return [];
const head = headMatch[1] ?? "";
// Strip HTML comments first so a commented-out can't leak through.
// The global replace below removes only well-formed `` pairs,
// so any `/g, "");
const unterm = scope.indexOf("