/** Meta-refresh, alternate links, and page metadata from HTML. */
import { validateUrl } from "../utils.js";
import type { ReadFormat } from "../types.js";
export interface PageMeta {
title?: string;
author?: string;
published?: string;
site?: string;
language?: string;
}
const MAX_REFRESH_HOPS = 5;
/** Follow meta-refresh only when delay is short (true redirect shells). */
const MAX_REFRESH_DELAY_SEC = 10;
function attr(tag: string, name: string): string | undefined {
const re = new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i");
const m = tag.match(re);
return m?.[1] ?? m?.[2] ?? m?.[3];
}
function resolveAgainst(baseUrl: string, href: string): string | null {
try {
const abs = new URL(href, baseUrl).href;
return validateUrl(abs) ? null : abs;
} catch {
return null;
}
}
function parseRefreshContent(content: string): { delay: number; url: string } | null {
// "0;url=https://…" | "0; URL='/path'" | "5;url=…"
const m = content
.trim()
.match(/^(\d+(?:\.\d+)?)\s*;\s*url\s*=\s*['"]?([^'";]+)['"]?\s*$/i);
if (!m) return null;
const delay = Number(m[1]);
if (!Number.isFinite(delay)) return null;
const url = m[2].trim();
return url ? { delay, url } : null;
}
/**
* Parse a short meta-refresh redirect target.
* Returns null for long delays, missing URL, or SSRF-blocked targets.
*/
export function findMetaRefreshUrl(html: string, baseUrl: string): string | null {
const head = html.slice(0, 32_768);
const re = /]*>/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(head))) {
const tag = m[0];
const httpEquiv = (attr(tag, "http-equiv") ?? "").toLowerCase();
if (httpEquiv !== "refresh") continue;
const content = attr(tag, "content");
if (!content) continue;
const parsed = parseRefreshContent(content);
if (!parsed || parsed.delay > MAX_REFRESH_DELAY_SEC) continue;
const next = resolveAgainst(baseUrl, parsed.url);
if (next) return next;
}
return null;
}
/** MIME types preferred per output format (first match wins). */
const ALTERNATE_TYPES: Record = {
markdown: [
"text/markdown",
"text/x-markdown",
"text/plain",
"application/json",
"text/html",
],
text: ["text/plain", "text/markdown", "text/x-markdown", "text/html"],
html: ["text/html", "application/xhtml+xml", "text/plain"],
};
function normalizeMime(type: string): string {
return type.split(";")[0].trim().toLowerCase();
}
/**
* Collect rel=alternate hrefs that match the requested format, preferred-type order.
* Same-document and SSRF-blocked targets are dropped.
*/
export function findAlternateUrls(
html: string,
baseUrl: string,
format: ReadFormat,
): string[] {
const preferred = ALTERNATE_TYPES[format] ?? ALTERNATE_TYPES.markdown;
const head = html.slice(0, 64_768);
const byType = new Map();
const re = /]*>/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(head))) {
const tag = m[0];
const rel = (attr(tag, "rel") ?? "").toLowerCase();
if (!/\balternate\b/.test(rel)) continue;
const type = normalizeMime(attr(tag, "type") ?? "");
const href = attr(tag, "href");
if (!href || !type || !preferred.includes(type)) continue;
const abs = resolveAgainst(baseUrl, href);
if (!abs) continue;
try {
if (new URL(abs).href === new URL(baseUrl).href) continue;
} catch {
continue;
}
const list = byType.get(type) ?? [];
if (!list.includes(abs)) list.push(abs);
byType.set(type, list);
}
const out: string[] = [];
for (const type of preferred) {
for (const u of byType.get(type) ?? []) {
if (!out.includes(u)) out.push(u);
}
}
return out;
}
function metaByKeys(html: string, keys: string[]): string | undefined {
// Index once, then pick by preferred key order (not document order).
const found = new Map();
const head = html.slice(0, 64_768);
const re = /]*>/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(head))) {
const tag = m[0];
const name = (
attr(tag, "name") ??
attr(tag, "property") ??
attr(tag, "itemprop") ??
""
).toLowerCase();
if (!name || found.has(name)) continue;
const content = attr(tag, "content")?.trim();
if (content) found.set(name, content);
}
for (const key of keys) {
const hit = found.get(key.toLowerCase());
if (hit) return hit;
}
return undefined;
}
/** Drop empty / placeholder meta (e.g. bare "@" from twitter:creator shells). */
function cleanMetaValue(value: string | undefined, kind: "author" | "generic" = "generic"): string | undefined {
if (!value) return undefined;
const t = value.trim();
if (!t) return undefined;
if (kind === "author") {
if (t === "@" || t === "@null" || t.length < 2) return undefined;
}
return t;
}
export function parsePageMeta(html: string, url: string): PageMeta {
const head = html.slice(0, 64_768);
const titleMatch = head.match(/]*>([\s\S]*?)<\/title>/i);
const title = cleanMetaValue(
metaByKeys(html, ["og:title", "twitter:title"]) ??
titleMatch?.[1]?.replace(/<[^>]+>/g, "").trim(),
);
const author = cleanMetaValue(
metaByKeys(html, [
"author",
"article:author",
"og:article:author",
"twitter:creator",
]),
"author",
);
const published = cleanMetaValue(
metaByKeys(html, [
"article:published_time",
"og:article:published_time",
"date",
"publish_date",
"dc.date",
"DC.date.issued",
"pubdate",
]),
);
let site = cleanMetaValue(
metaByKeys(html, ["og:site_name", "application-name", "twitter:site"]),
);
if (!site) {
try {
site = new URL(url).hostname;
} catch {
// ignore
}
}
const langMatch = head.match(/]*\blang\s*=\s*["']([^"']+)["']/i);
const language = cleanMetaValue(
langMatch?.[1]?.trim() || metaByKeys(html, ["og:locale", "language", "dc.language"]),
);
return {
title,
author,
published,
site,
language,
};
}
/** True when extracted content is thin enough to try alternates. */
export const THIN_CONTENT_CHARS = 400;
export function isThinContent(chars: number): boolean {
return chars < THIN_CONTENT_CHARS;
}
/** Follow meta-refresh hops via the given fetch function. */
export async function fetchWithMetaRefresh(
url: string,
doFetch: (url: string) => Promise,
maxHops: number = MAX_REFRESH_HOPS,
): Promise {
const seen = new Set();
let current = url;
let last: T | undefined;
for (let hop = 0; hop < maxHops; hop++) {
if (seen.has(current)) break;
seen.add(current);
last = await doFetch(current);
const next = findMetaRefreshUrl(last.html, last.finalUrl || current);
if (!next || seen.has(next) || next === last.finalUrl) {
return last;
}
current = next;
}
return last ?? doFetch(current);
}