import { loadFetchContentDomainPolicy, loadSsrfConfig, validateRemoteUrl } from "../net/ssrf.ts";
import { extractHeadingTitle } from "./http-extractor.ts";
import type { FetchOptions, FetchResult } from "./types.ts";

const DEFAULT_TIMEOUT_MS = 30_000;

export async function fetchWithJina(url: string, signal?: AbortSignal, options: FetchOptions = {}): Promise<FetchResult | null> {
	const ssrf = loadSsrfConfig();
	await validateRemoteUrl(url, {
		allowRanges: ssrf.allowRanges,
		trustEnvProxy: options.proxy === undefined && ssrf.trustEnvProxy,
		domainPolicy: loadFetchContentDomainPolicy(),
		...(options.lookup ? { lookup: options.lookup } : {}),
	});
	const timeout = AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
	const response = await fetch(`https://r.jina.ai/${url}`, {
		headers: { Accept: "text/markdown", "X-No-Cache": "true" },
		signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
	});
	if (!response.ok) return null;
	const text = await response.text();
	const marker = text.indexOf("Markdown Content:");
	const content = (marker >= 0 ? text.slice(marker + "Markdown Content:".length) : text).trim();
	return content ? { url, title: extractHeadingTitle(content) || new URL(url).hostname, content, error: null } : null;
}