/**
 * Shared types and utilities for web-fetch handlers
 */
import { scheduler } from "node:timers/promises";
import { ptree } from "@oh-my-pi/pi-utils";
import { decodeEntities } from "@oh-my-pi/pi-utils/dom/parser";
import type TurndownService from "@oh-my-pi/pi-utils/turndown";

import type { ModelRegistry } from "../../config/model-registry";
import type { AgentStorageApi } from "../../session/agent-storage";
import { ToolAbortError } from "../../tools/tool-errors";
import { decodeBody, readBodyCapped } from "../response-body";

export { formatNumber } from "@oh-my-pi/pi-utils";

export interface RenderResult {
	url: string;
	finalUrl: string;
	contentType: string;
	method: string;
	content: string;
	fetchedAt: string;
	/** True when the response body was cut mid-stream at {@link MAX_BYTES}. */
	truncated: boolean;
	notes: string[];
}

/**
 * Site-specific URL renderer: returns `null` for URLs it does not own.
 * `modelRegistry` is the session's, for handlers that read through a model (X via Grok).
 */
export type SpecialHandler = (
	url: string,
	timeout: number,
	signal?: AbortSignal,
	storage?: AgentStorageApi | null,
	modelRegistry?: ModelRegistry,
) => Promise<RenderResult | null>;

/** Network read bound for a fetched body; the only size limit on rendered URL content. */
export const MAX_BYTES = 50 * 1024 * 1024;

const USER_AGENTS = [
	"curl/8.0",
	"Mozilla/5.0 (compatible; TextBot/1.0)",
	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
];

/**
 * Statuses sites answer a user agent they refuse with (999 is LinkedIn's), whatever the body
 * says: a bare nginx "403 Forbidden" page, SEC's "Undeclared Automated Tool" page or a 429
 * keyed on the agent all serve the page to the next user agent.
 */
const BLOCKED_STATUSES = new Set([403, 429, 503, 999]);

/** A challenge interstitial carries no content (Reddit's is about 8 KB); a larger page is no wall. */
const CHALLENGE_PAGE_MAX_CHARS = 64 * 1024;

/**
 * True when a page small enough to be an interstitial holds an `<input name="js_challenge">` field
 * outside `<template>`. Bun's HTMLRewriter tokenizes the page as a browser does, so a form quoted in
 * a comment, script, raw-text element or attribute is no field. A field gets an attribute named with
 * a fresh nonce, which the page cannot already carry; a tag cut off by the end of the page is
 * written back as text, so its mark never reaches the output.
 */
function hasChallengeForm(html: string): boolean {
	if (html.length > CHALLENGE_PAGE_MAX_CHARS) return false;
	const mark = `data-omp-challenge-${crypto.randomUUID()}`;
	const output = new HTMLRewriter()
		.on("input[name]", {
			element(input) {
				if (decodeEntities(input.getAttribute("name") ?? "") === "js_challenge") input.setAttribute(mark, "");
			},
		})
		.on("template input[name]", {
			element(input) {
				input.removeAttribute(mark);
			},
		})
		.transform(html);
	return output.includes(mark);
}

/**
 * Name the bot protection whose challenge or block page an HTML response is. A 2xx page counts
 * only from Cloudflare's header or a live challenge form, since an article can quote any marker
 * as text; vendor markers in the body count on an error status.
 */
function botWallOf(response: Response, contentType: string, content: string): string | undefined {
	if (!contentType.includes("html")) return undefined;
	if (response.headers.get("cf-mitigated") === "challenge") return "Cloudflare";
	if (hasChallengeForm(content)) return "JavaScript challenge";
	if (response.ok) return undefined;
	if (content.includes("_cf_chl_opt")) return "Cloudflare";
	if (content.includes("captcha-delivery.com")) return "DataDome";
	if (content.includes("px-captcha")) return "PerimeterX";
	if (content.includes("blocked by network security")) return "network security block";
	return undefined;
}

/**
 * Collapse blank-line runs and trim. Never shortens content: URL reads page the preview
 * themselves, and selectors, artifacts, and URL search must see the whole body.
 */
export function finalizeOutput(content: string): string {
	return content.replace(/\n{3,}/g, "\n\n").trim();
}

export interface LoadPageOptions {
	timeout?: number;
	headers?: Record<string, string>;
	method?: string;
	body?: string;
	maxBytes?: number;
	signal?: AbortSignal;
	/**
	 * Return true to skip reading the response body for this content type
	 * (lowercased mime, no params). The caller is expected to re-fetch the
	 * payload as binary; this avoids streaming + decoding huge binaries twice.
	 */
	skipBodyForContentType?: (contentType: string) => boolean;
}

export interface LoadPageResult {
	content: string;
	contentType: string;
	finalUrl: string;
	ok: boolean;
	status?: number;
	/** True when the body was cut mid-stream at maxBytes. */
	truncated?: boolean;
	/** Last transport-level error message when ok is false. */
	error?: string;
	/** True when the body read was skipped via skipBodyForContentType. */
	bodySkipped?: boolean;
	/**
	 * Bot protection that answered with its challenge or block page instead of the page. Such a
	 * response is never `ok`, whatever its status, so a caller checking `ok` does not take it as content.
	 */
	blockedBy?: string;
}

const RETRY_AFTER_MAX_MS = 10_000;

/** Parse a Retry-After header (seconds or HTTP-date) into a bounded delay. */
function parseRetryAfterMs(value: string | null): number {
	if (!value) return 1_000;
	const seconds = Number(value);
	if (Number.isFinite(seconds)) return Math.min(Math.max(seconds, 0) * 1000, RETRY_AFTER_MAX_MS);
	const date = Date.parse(value);
	if (!Number.isNaN(date)) return Math.min(Math.max(date - Date.now(), 0), RETRY_AFTER_MAX_MS);
	return 1_000;
}

/**
 * Fetch a page with timeout and size limit
 */
export async function loadPage(url: string, options: LoadPageOptions = {}): Promise<LoadPageResult> {
	const { timeout = 20, headers = {}, maxBytes = MAX_BYTES, signal, method = "GET", body } = options;

	let lastError: string | undefined;
	let lastBlockedResponse: LoadPageResult | undefined;
	let retried429 = false;
	// Header names are case-insensitive, so a caller's header replaces ours whatever its spelling.
	const callerHeaders = new Headers(headers);
	// A caller's own User-Agent replaces every rotated one, so a refusal of it is final; a
	// transport error still retries.
	const rotates = !callerHeaders.has("user-agent");
	for (let attempt = 0; attempt < USER_AGENTS.length; attempt++) {
		if (signal?.aborted) {
			throw new ToolAbortError();
		}

		const userAgent = USER_AGENTS[attempt];
		const requestSignal = ptree.combineSignals(signal, timeout * 1000);

		try {
			const requestHeaders = new Headers({
				"User-Agent": userAgent,
				Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
				"Accept-Language": "en-US,en;q=0.5",
				"Accept-Encoding": "identity", // Cloudflare Markdown-for-Agents returns corrupted bytes when compression is negotiated
			});
			for (const [name, value] of callerHeaders) requestHeaders.set(name, value);
			const requestInit: RequestInit = {
				signal: requestSignal,
				method,
				headers: requestHeaders,
				redirect: "follow",
			};

			if (body !== undefined) {
				requestInit.body = body;
			}

			const response = await fetch(url, requestInit);

			const rawContentType = response.headers.get("content-type") ?? "";
			const contentType = rawContentType.split(";")[0]?.trim().toLowerCase() ?? "";
			const finalUrl = response.url;

			const lastAgent = !rotates || attempt === USER_AGENTS.length - 1;
			if (!lastAgent && BLOCKED_STATUSES.has(response.status)) {
				void response.body?.cancel().catch(() => {});
				// A refusal whose header names its wall is kept in case later agents fare no better; its
				// body is not read, so rotation costs what it did. A 2xx challenge kept later wins.
				const headerWall = lastBlockedResponse ? undefined : botWallOf(response, contentType, "");
				if (headerWall) {
					lastBlockedResponse = {
						content: "",
						contentType,
						finalUrl,
						ok: false,
						status: response.status,
						blockedBy: headerWall,
					};
				}
				continue;
			}

			if (response.status === 429 && !retried429) {
				// Rate limited under the last user agent: retry it once, honoring a bounded
				// Retry-After. The wait observes the caller's signal so an Esc during the backoff
				// does not stall for up to the full delay.
				retried429 = true;
				const delayMs = parseRetryAfterMs(response.headers.get("retry-after"));
				void response.body?.cancel().catch(() => {});
				try {
					await scheduler.wait(delayMs, { signal });
				} catch {
					throw new ToolAbortError();
				}
				attempt--; // Reuse the same user agent for the retry.
				continue;
			}

			if (response.ok && options.skipBodyForContentType?.(contentType)) {
				void response.body?.cancel().catch(() => {});
				return { content: "", contentType, finalUrl, ok: true, status: response.status, bodySkipped: true };
			}

			const responseBody = await readBodyCapped(response, maxBytes);
			if (!responseBody) {
				// Once a wall is known, an error under another agent only means trying the next one.
				if (!response.ok && lastBlockedResponse) {
					if (!lastAgent) continue;
					return lastBlockedResponse;
				}
				// A successful response can still be bodyless (HTTP 204, or a null
				// body stream): keep the transport's own success verdict instead of
				// rewriting it, so the caller's status handling (a 204 is a valid
				// empty read, not a failed fetch) runs on the real outcome.
				return { content: "", contentType, finalUrl, ok: response.ok, status: response.status };
			}

			const content = decodeBody(responseBody.bytes, rawContentType);
			const blockedBy = botWallOf(response, contentType, content);
			const result: LoadPageResult = {
				content,
				contentType,
				finalUrl,
				ok: response.ok && !blockedBy,
				status: response.status,
				truncated: responseBody.truncated,
				blockedBy,
			};
			if (blockedBy && !lastAgent) {
				// A 2xx challenge outranks a refusal's wall: remote readers can still get past it.
				if (!lastBlockedResponse || response.ok) lastBlockedResponse = result;
				continue;
			}
			// The last agent's error yields to a saved wall, unless both are walls and the saved one is a refusal.
			if (!response.ok && lastBlockedResponse && (!blockedBy || (lastBlockedResponse.status ?? 0) < 300)) {
				if (!lastAgent) continue;
				return lastBlockedResponse;
			}
			return result;
		} catch (error) {
			if (signal?.aborted) {
				throw new ToolAbortError();
			}
			lastError = error instanceof Error ? error.message : String(error);
			if (attempt === USER_AGENTS.length - 1) {
				return lastBlockedResponse ?? { content: "", contentType: "", finalUrl: url, ok: false, error: lastError };
			}
		}
	}

	return { content: "", contentType: "", finalUrl: url, ok: false, error: lastError };
}

/** Module-level Turndown instance — built lazily on first use. */
let turndownPromise: Promise<TurndownService> | undefined;

function getTurndown(): Promise<TurndownService> {
	turndownPromise ||= initTurndown();
	return turndownPromise;
}

async function initTurndown(): Promise<TurndownService> {
	// Lazy import keeps turndown/turndown-plugin-gfm off the startup graph.
	const { createTurndown } = await import("../../utils/turndown");
	return createTurndown();
}

/**
 * Convert HTML to markdown using Turndown with GFM support.
 * Strips script/style tags before conversion.
 */
export async function htmlToBasicMarkdown(html: string): Promise<string> {
	const cleaned = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "");
	const turndown = await getTurndown();
	return turndown.turndown(cleaned).trim();
}

/**
 * Build a RenderResult from markdown content. Calls finalizeOutput internally.
 */
export function buildResult(
	md: string,
	opts: {
		url: string;
		finalUrl?: string;
		method: string;
		fetchedAt: string;
		notes?: string[];
		contentType?: string;
		/** Pass the source `LoadPageResult.truncated` when `md` embeds a fetched body verbatim. */
		truncated?: boolean;
	},
): RenderResult {
	return {
		url: opts.url,
		finalUrl: opts.finalUrl ?? opts.url,
		contentType: opts.contentType ?? "text/markdown",
		method: opts.method,
		content: finalizeOutput(md),
		fetchedAt: opts.fetchedAt,
		truncated: opts.truncated ?? false,
		notes: opts.notes ?? [],
	};
}

/**
 * Format a date value as YYYY-MM-DD. Returns empty string on invalid input.
 */
export function formatIsoDate(value?: string | number | Date): string {
	if (value == null) return "";
	if (typeof value === "string") {
		const datePrefix = value.match(/^\d{4}-\d{2}-\d{2}/);
		if (datePrefix) return datePrefix[0];
	}
	try {
		return new Date(value).toISOString().split("T")[0];
	} catch {
		return "";
	}
}

/**
 * Decode common HTML entities.
 */
export function decodeHtmlEntities(text: string): string {
	return text
		.replace(/&lt;/g, "<")
		.replace(/&gt;/g, ">")
		.replace(/&amp;/g, "&")
		.replace(/&quot;/g, '"')
		.replace(/&#0?39;/g, "'")
		.replace(/&#x27;/g, "'")
		.replace(/&#x2F;/g, "/")
		.replace(/&nbsp;/g, " ");
}

/**
 * Format seconds into HH:MM:SS or MM:SS.
 */
export function formatMediaDuration(totalSeconds: number): string {
	const hours = Math.floor(totalSeconds / 3600);
	const minutes = Math.floor((totalSeconds % 3600) / 60);
	const secs = Math.floor(totalSeconds % 60);
	if (hours > 0) return `${hours}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
	return `${minutes}:${String(secs).padStart(2, "0")}`;
}

/**
 * Extract localized text, preferring en-US/en.
 */
export type LocalizedText = string | Record<string, string | null> | null | undefined;

export function getLocalizedText(value: LocalizedText, defaultLocale?: string): string | undefined {
	if (value == null) return undefined;
	if (typeof value === "string") return value;
	if (defaultLocale && value[defaultLocale]) return value[defaultLocale];
	return (
		value["en-US"] ?? value.en_US ?? value.en ?? Object.values(value).find(v => typeof v === "string") ?? undefined
	);
}

/**
 * Check if content looks like HTML by inspecting the leading tag.
 */
export function looksLikeHtml(content: string): boolean {
	// Only the leading tag matters; lowercase a short prefix instead of the whole body.
	const trimmed = content.trimStart().slice(0, 16).toLowerCase();
	return (
		trimmed.startsWith("<!doctype") ||
		trimmed.startsWith("<html") ||
		trimmed.startsWith("<head") ||
		trimmed.startsWith("<body")
	);
}