/**
 * Deep Research — direct Firecrawl HTTP client
 *
 * Calls the self-hosted Firecrawl API directly (same approach as the
 * firecrawl.ts extension)
 */
import * as fs from "node:fs";
import * as path from "node:path";
import { parse as parseYaml } from "yaml";
import type { SearchResult, EnrichedSearchResult, ContentType } from "./types";
import { getAgentDir } from "@oh-my-pi/pi-coding-agent";

/* ── Config ──────────────────────────────────────────────────────── */

/**
 * Read and merge Firecrawl settings from omp's config.yml files.
 *
 * Resolution order (later wins):
 *   1. env vars FIRECRAWL_BASE_URL / FIRECRAWL_API_KEY
 *   2. global ~/.omp/agent/config.yml → firecrawl.*
 *   3. project .omp/config.yml → firecrawl.*
 *   4. default http://localhost:3002 (if no baseUrl configured)
 */
function loadFirecrawlConfig() {
	// Start with env var defaults
	let baseUrl = process.env.FIRECRAWL_BASE_URL ?? "http://localhost:3002";
	let apiKey = process.env.FIRECRAWL_API_KEY;

	const agentDir = getAgentDir();

	// Helper: read a config.yml and merge its firecrawl.* keys
	const tryReadConfig = (configPath: string): void => {
		try {
			const raw = parseYaml(fs.readFileSync(configPath, "utf-8")) as Record<
				string,
				unknown
			>;
			const fc = (raw?.firecrawl ?? {}) as Record<string, unknown>;
			if (typeof fc.baseUrl === "string" && fc.baseUrl.length > 0) {
				baseUrl = fc.baseUrl;
			}
			if (typeof fc.apiKey === "string" && fc.apiKey.length > 0) {
				apiKey = fc.apiKey;
			}
		} catch {
			// File missing or unparseable — skip
		}
	};

	// 1. Global config
	tryReadConfig(path.join(agentDir, "config.yml"));

	// 2. Project config (override global)
	tryReadConfig(path.join(process.cwd(), ".omp", "config.yml"));

	return {
		baseUrl: baseUrl.replace(/\/+$/, ""),
		apiKey,
	};
}

const { baseUrl: BASE_URL, apiKey: API_KEY } = loadFirecrawlConfig();

/* ── Domain Authority Heuristics ─────────────────────────────────── */

/**
 * Known high-authority domains and their authority scores (0.0 – 1.0).
 * Academic, official, and established technical sources score highest.
 */
const AUTHORITY_DOMAINS: Record<string, number> = {
	// Academic & scholarly
	"arxiv.org": 0.95,
	"scholar.google.com": 0.95,
	"pubmed.ncbi.nlm.nih.gov": 0.95,
	"semanticscholar.org": 0.9,
	"ieee.org": 0.95,
	"acm.org": 0.95,
	"springer.com": 0.9,
	"sciencedirect.com": 0.9,
	"wiley.com": 0.85,
	"nature.com": 0.95,
	"science.org": 0.95,
	"plos.org": 0.85,
	// Official documentation
	"docs.python.org": 0.9,
	"developer.mozilla.org": 0.9,
	"learn.microsoft.com": 0.85,
	"developer.apple.com": 0.85,
	"kubernetes.io": 0.85,
	"react.dev": 0.85,
	"nextjs.org": 0.8,
	// Official language/platform docs
	"go.dev": 0.9,
	"golang.org": 0.9,
	"rust-lang.org": 0.9,
	"nodejs.org": 0.85,
	"python.org": 0.85,
	"typescriptlang.org": 0.85,
	"openai.com": 0.8,
	"anthropic.com": 0.8,
	"cloud.google.com": 0.8,
	"aws.amazon.com": 0.8,
	"azure.microsoft.com": 0.8,
	"postgresql.org": 0.85,
	"sqlite.org": 0.85,
	"redis.io": 0.85,
	"docker.com": 0.75,
	"elastic.co": 0.75,
	"grafana.com": 0.75,
	"datadoghq.com": 0.75,
	"cloudflare.com": 0.8,
	"blog.cloudflare.com": 0.8,
	"techempower.com": 0.8,
	"goframe.org": 0.75,
	"corrode.dev": 0.6,
	"evrone.com": 0.4,
	"rustify.rs": 0.4,
	"core.cz": 0.4,
	// Medical / clinical
	"mayoclinic.org": 0.9,
	"heart.org": 0.85,
	"researchgate.net": 0.6,
	"healthline.com": 0.5,
	"medicalnewstoday.com": 0.5,
	"webmd.com": 0.45,
	"verywellhealth.com": 0.5,
	// Databases & dev tools
	"mysql.com": 0.85,
	"mariadb.org": 0.85,
	"cockroachlabs.com": 0.7,
	"timescale.com": 0.7,
	"mongodb.com": 0.8,
	"liquibase.com": 0.6,
	"sqlpipe.com": 0.5,
	"data-tune.com": 0.4,
	"binaryigor.com": 0.4,
	// Government & non-profits
	".gov": 0.9,
	".edu": 0.85,
	"who.int": 0.9,
	"worldbank.org": 0.85,
	"oecd.org": 0.85,
	// Established tech & news
	"github.com": 0.8,
	"stackoverflow.com": 0.7,
	"medium.com": 0.4,
	"dev.to": 0.5,
	"wikipedia.org": 0.7,
	"reuters.com": 0.8,
	"apnews.com": 0.8,
	"bbc.com": 0.75,
	"nytimes.com": 0.75,
	"theguardian.com": 0.7,
	"techcrunch.com": 0.6,
	"arstechnica.com": 0.65,
	"wired.com": 0.65,
	"infoworld.com": 0.55,
	// Practitioner/aggregator content with measurable quality
	"github.io": 0.6,
	"crates.io": 0.7,
	"docs.rs": 0.75,
	"digitalocean.com": 0.6,
	"freecodecamp.org": 0.6,
	"geeksforgeeks.org": 0.35,
	"stackexchange.com": 0.65,
	"huggingface.co": 0.65,
	"nasa.gov": 0.9,
	"mit.edu": 0.9,
	"stanford.edu": 0.9,
	"harvard.edu": 0.9,
	"ox.ac.uk": 0.9,
	"cam.ac.uk": 0.9,
	// Low-authority: personal social / SEO content
	"linkedin.com": 0.25,
	"reddit.com": 0.25,
	"x.com": 0.3,
	"twitter.com": 0.3,
	"youtube.com": 0.3,
	"blogspot.com": 0.25,
	"substack.com": 0.3,
	"hashnode.dev": 0.35,
	"quora.com": 0.3,
	"netguru.com": 0.3,
	"relisoftware.com": 0.3,
	"dasroot.net": 0.3,
	"devgenius.io": 0.3,
	"devnewsletter.com": 0.3,
};

/**
 * Known low-quality SEO/comparison-spam domains. Content is often
 * auto-generated, republished from other sites, or thin on substance.
 * These get a hard authority floor so they never rank above real content.
 */
const LOW_AUTHORITY_DOMAINS: Record<string, number> = {
	"markaicode.com": 0.15,
	"bytegoblin.io": 0.2,
	"towardsdev.com": 0.2,
	"rustvsgo.com": 0.3,
	"seekingalpha.com": 0.3,
	"investopedia.com": 0.55,
	"devops-daily.com": 0.3,
};

/** Content-type hints based on domain patterns */
const CONTENT_TYPE_HINTS: [RegExp, ContentType][] = [
	[
		/arxiv\.org|semanticscholar|ieee\.org|acm\.org|springer|sciencedirect|pubmed\.ncbi/,
		"paper",
	],
	[
		/docs\.|learn\.|developer\.|kubernetes\.io|react\.dev|nextjs\.org/,
		"documentation",
	],
	[/wikipedia\.org|stackoverflow\.com|medium\.com|dev\.to/, "forum"],
	[
		/reuters\.com|apnews\.com|bbc\.com|nytimes\.com|techcrunch|arstechnica|wired/,
		"news",
	],
	[/\.gov|\.edu|who\.int|worldbank|oecd\.org/, "official"],
	[/github\.com/, "documentation"],
];

/* ── Source enrichment helpers ───────────────────────────────────── */

/**
 * Extract the registered domain from a URL (e.g., "blog.example.com" → "example.com").
 * Uses a simple 2-part TLD heuristic. For common cases like .co.uk this is approximate.
 */
function extractDomain(url: string): string {
	try {
		const hostname = new URL(url).hostname.toLowerCase();
		// Special-case common multi-part TLDs
		const multiPartTlds =
			/\.(co\.uk|org\.uk|ac\.uk|gov\.uk|com\.au|co\.jp|co\.kr|com\.br)$/;
		const parts = hostname.split(".");
		if (multiPartTlds.test(hostname) && parts.length >= 3) {
			return parts.slice(-3).join(".");
		}
		return parts.slice(-2).join(".");
	} catch {
		return url.replace(/^https?:\/\//, "").split("/")[0] ?? url;
	}
}

function computeAuthorityScore(domain: string): number {
	// Hard floor for known low-authority domains first
	if (LOW_AUTHORITY_DOMAINS[domain] !== undefined)
		return LOW_AUTHORITY_DOMAINS[domain];

	// Direct match first
	if (AUTHORITY_DOMAINS[domain]) return AUTHORITY_DOMAINS[domain];

	// Suffix matches (.gov, .edu, etc.)
	for (const [key, score] of Object.entries(AUTHORITY_DOMAINS)) {
		if (key.startsWith(".") && domain.endsWith(key)) return score;
	}

	// Subdomain matches (e.g., blog.example.com matches example.com)
	const parent = domain.split(".").slice(-2).join(".");
	if (parent !== domain && AUTHORITY_DOMAINS[parent]) {
		return AUTHORITY_DOMAINS[parent] * 0.9;
	}

	// github.io personal sites: treat as practitioner content (medium)
	if (domain.endsWith(".github.io")) return 0.55;

	return 0.3; // Unknown / low-authority default
}

function detectContentType(url: string, description: string): ContentType {
	const lowerUrl = url.toLowerCase();
	const lowerDesc = description.toLowerCase();

	for (const [pattern, type] of CONTENT_TYPE_HINTS) {
		if (pattern.test(lowerUrl)) return type;
	}

	// Heuristics from description text
	if (/paper|research|study|experiment|analysis\b/.test(lowerDesc))
		return "paper";
	if (/documentation|guide|tutorial|api|reference/.test(lowerDesc))
		return "documentation";
	if (/blog|post|article|opinion/.test(lowerDesc)) return "blog";
	if (/news|report|announce|release/.test(lowerDesc)) return "news";
	if (/forum|discussion|question|answer|thread/.test(lowerDesc)) return "forum";

	return "other";
}

function tryParseDate(dateStr: string | undefined | null): Date | null {
	if (!dateStr) return null;
	const d = new Date(dateStr);
	return isNaN(d.getTime()) ? null : d;
}

/**
 * Normalize a title for near-duplicate detection: lowercase, strip
 * punctuation, collapse whitespace, drop common filler words.
 * Two syndicated copies of the same article normalize identically.
 */
export function normalizeTitle(title: string): string {
	return title
		.toLowerCase()
		.replace(/[^a-z0-9\s]/g, " ")
		.replace(
			/\b(?:the|a|an|of|for|and|or|in|on|with|vs|versus|to|how|what|why|2024|2025|2026)\b/g,
			" ",
		)
		.replace(/\s+/g, " ")
		.trim();
}

/**
 * Near-duplicate check between two titles: normalized forms must share
 * a substantial token overlap (same core words in the same order).
 */
export function isNearDuplicateTitle(a: string, b: string): boolean {
	const normA = normalizeTitle(a);
	const normB = normalizeTitle(b);
	if (!normA || !normB) return false;
	if (normA === normB) return true;

	const tokensA = normA.split(" ");
	const tokensB = normB.split(" ");
	if (tokensA.length < 3 || tokensB.length < 3) return normA === normB;

	// Check if one title is a substring of the other (after normalization)
	if (normA.includes(normB) || normB.includes(normA)) return true;

	// Jaccard-ish overlap on the shorter token set
	const [short, long] =
		tokensA.length <= tokensB.length ? [tokensA, tokensB] : [tokensB, tokensA];
	const overlap = short.filter((t) => long.includes(t)).length;
	return overlap / short.length >= 0.75;
}

/**
 * Enrich a raw search result with source authority metadata.
 * Accepts extra fields (e.g. date) from the Firecrawl API response.
 */
export function enrichResult(
	result: SearchResult & Record<string, unknown>,
): EnrichedSearchResult {
	const domain = extractDomain(result.url);
	return {
		...result,
		domain,
		authorityScore: computeAuthorityScore(domain),
		publishedDate: tryParseDate(result.date as string | undefined),
		contentType: detectContentType(result.url, result.description),
	};
}

/* ── Helpers ──────────────────────────────────────────────────────── */

async function firecrawlRequest(
	endpoint: string,
	body: Record<string, unknown>,
	signal?: AbortSignal,
): Promise<unknown> {
	const headers: Record<string, string> = {
		"Content-Type": "application/json",
	};
	if (API_KEY) {
		headers["Authorization"] = `Bearer ${API_KEY}`;
	}

	const res = await fetch(`${BASE_URL}/v1/${endpoint}`, {
		method: "POST",
		headers,
		body: JSON.stringify(body),
		signal,
	});

	if (!res.ok) {
		const text = await res.text();
		throw new Error(
			`Firecrawl ${endpoint} failed (${res.status}): ${text.slice(0, 500)}`,
		);
	}

	return res.json();
}

/**
 * firecrawlRequest with retry-with-backoff for transient failures
 * (429 rate limits, 5xx server errors, network blips). Does NOT retry
 * 4xx client errors (invalid requests) or aborts.
 */
async function firecrawlRequestWithRetry(
	endpoint: string,
	body: Record<string, unknown>,
	signal?: AbortSignal,
	retries: number = 2,
): Promise<unknown> {
	let lastError: unknown;
	for (let attempt = 0; attempt <= retries; attempt++) {
		if (signal?.aborted) throw new Error("Aborted");
		try {
			return await firecrawlRequest(endpoint, body, signal);
		} catch (error) {
			lastError = error;
			const status =
				error instanceof Error
					? Number(/failed \((\d+)\)/.exec(error.message)?.[1] ?? 0)
					: 0;
			// Don't retry aborts or 4xx client errors (other than 429)
			if (
				signal?.aborted ||
				(status >= 400 && status < 500 && status !== 429)
			) {
				throw error;
			}
			if (attempt < retries) {
				const delayMs = 400 * 2 ** attempt + Math.random() * 200;
				await new Promise((r) => setTimeout(r, delayMs));
			}
		}
	}
	throw lastError;
}

export async function isFirecrawlReachable(): Promise<boolean> {
	try {
		const res = await fetch(`${BASE_URL}/v1/scrape`, {
			method: "POST",
			headers: {
				"Content-Type": "application/json",
				...(API_KEY ? { Authorization: `Bearer ${API_KEY}` } : {}),
			},
			body: JSON.stringify({ url: "https://example.com", formats: ["links"] }),
			signal: AbortSignal.timeout(10_000),
		});
		return res.ok;
	} catch {
		return false;
	}
}

/* ── Search ───────────────────────────────────────────────────────── */

/**
 * Search the web and return structured, enriched results.
 * Uses Firecrawl's search endpoint with scrape to get full page content.
 */
export async function searchWeb(
	query: string,
	limit: number = 5,
	signal?: AbortSignal,
): Promise<EnrichedSearchResult[]> {
	const body: Record<string, unknown> = {
		query,
		limit: Math.min(limit, 10),
		scrapeOptions: {
			formats: ["markdown"],
			onlyMainContent: true,
		},
	};

	const result = await firecrawlRequestWithRetry("search", body, signal);

	if (!result || typeof result !== "object") return [];

	const res = result as {
		success?: boolean;
		data?: Record<string, unknown>[];
		error?: string;
	};

	if (!res.success || !res.data) return [];

	const rawResults: (SearchResult & Record<string, unknown>)[] = res.data
		.map((doc) => ({
			title: (doc.title as string) ?? "",
			url: (doc.url as string) ?? "",
			description: (doc.description as string) ?? "",
			markdown: (doc.markdown as string) ?? "",
			// Preserve extra fields for date extraction
			...doc,
		}))
		.filter((r) => {
			// Keep results with a meaningful body OR a substantive description.
			// Filters out stub pages / pure navigation results that would
			// waste analysis tokens.
			const hasBody = (r.markdown ?? "").trim().length >= 150;
			const hasSubstantiveDesc = (r.description ?? "").trim().length >= 40;
			return hasBody || hasSubstantiveDesc;
		});

	// Enrich each result with source metadata
	return rawResults.map(enrichResult);
}

/* ── Scrape ───────────────────────────────────────────────────────── */

/**
 * Scrape a single URL and return its markdown content.
 */
export async function scrapeUrl(
	url: string,
	signal?: AbortSignal,
): Promise<{ title: string; markdown: string; links: string[] } | null> {
	const result = await firecrawlRequestWithRetry(
		"scrape",
		{ url, formats: ["markdown"] },
		signal,
	);

	if (!result || typeof result !== "object") return null;

	const res = result as {
		success?: boolean;
		data?: Record<string, unknown>;
		error?: string;
	};

	if (!res.success || !res.data) return null;

	return {
		title: (res.data.title as string) ?? "",
		markdown: (res.data.markdown as string) ?? "",
		links: (res.data.links as string[]) ?? [],
	};
}