import { parseHTML } from "linkedom"; import { activityMonitor } from "./activity.ts"; import type { SearchOptions, SearchResult, SearchResponse } from "./perplexity.ts"; const SEARCH_URL = "https://html.duckduckgo.com/html/"; const SEARCH_TIMEOUT_MS = 30_000; interface NormalizedDomainFilters { allowed: string[]; blocked: string[]; } function normalizeCount(value: number | undefined): number { if (typeof value !== "number" || !Number.isFinite(value)) return 5; return Math.max(1, Math.min(Math.floor(value), 20)); } function normalizeDomain(value: string): string | null { let input = value.trim().toLowerCase(); if (!input) return null; if (input.startsWith("-")) input = input.slice(1).trim(); if (!input) return null; try { const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`); input = parsed.hostname; } catch { input = input.split("/")[0]?.split(":")[0] ?? ""; } input = input.replace(/^\.+|\.+$/g, ""); return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null; } function normalizeDomainFilters(domainFilter: string[] | undefined): NormalizedDomainFilters { const filters: NormalizedDomainFilters = { allowed: [], blocked: [] }; for (const raw of domainFilter ?? []) { const domain = normalizeDomain(raw); if (!domain) continue; const target = raw.trim().startsWith("-") ? filters.blocked : filters.allowed; if (!target.includes(domain)) target.push(domain); } return filters; } function hostMatchesDomain(hostname: string, domain: string): boolean { return hostname === domain || hostname.endsWith(`.${domain}`); } function matchesDomainFilters(url: string, filters: NormalizedDomainFilters): boolean { if (filters.allowed.length === 0 && filters.blocked.length === 0) return true; const hostname = new URL(url).hostname.toLowerCase(); if (filters.allowed.length > 0 && !filters.allowed.some(domain => hostMatchesDomain(hostname, domain))) return false; return !filters.blocked.some(domain => hostMatchesDomain(hostname, domain)); } function decodeResultUrl(href: string): string | null { try { const link = new URL(href, SEARCH_URL); const destination = link.searchParams.get("uddg") ?? link.href; const url = new URL(destination); return url.protocol === "http:" || url.protocol === "https:" ? url.href : null; } catch { return null; } } export function isDuckDuckGoAvailable(): boolean { return true; } export async function searchWithDuckDuckGo(query: string, options: SearchOptions = {}): Promise { const url = new URL(SEARCH_URL); url.searchParams.set("q", query); const activityId = activityMonitor.logStart({ type: "api", query }); try { const response = await fetch(url, { method: "GET", headers: { Accept: "text/html", "User-Agent": "Mozilla/5.0 (compatible; pi-web-access/1.0; +https://github.com/nicobailon/pi-web-access)", }, signal: options.signal ? AbortSignal.any([AbortSignal.timeout(SEARCH_TIMEOUT_MS), options.signal]) : AbortSignal.timeout(SEARCH_TIMEOUT_MS), }); if (!response.ok) { const body = await response.text(); throw new Error(`DuckDuckGo search error ${response.status}: ${body.slice(0, 300)}`); } const { document } = parseHTML(await response.text()); const filters = normalizeDomainFilters(options.domainFilter); const results: SearchResult[] = []; let parseableResults = 0; for (const container of document.querySelectorAll(".result")) { if (container.classList.contains("result--ad")) continue; const anchor = container.querySelector(".result__a"); const title = anchor?.textContent?.trim() ?? ""; const href = anchor?.getAttribute("href")?.trim() ?? ""; const resultUrl = href ? decodeResultUrl(href) : null; if (!title || !resultUrl) continue; parseableResults++; if (!matchesDomainFilters(resultUrl, filters)) continue; const snippet = container.querySelector(".result__snippet")?.textContent?.trim() ?? ""; results.push({ title, url: resultUrl, snippet }); if (results.length >= normalizeCount(options.numResults)) break; } if (parseableResults === 0) { throw new Error("DuckDuckGo returned no parseable results (invalid response)"); } activityMonitor.logComplete(activityId, response.status); const answer = results.map(result => result.snippet ? `${result.snippet}\nSource: ${result.title} (${result.url})` : `Source: ${result.title} (${result.url})`).join("\n\n"); return { answer, results }; } catch (err) { const message = err instanceof Error ? err.message : String(err); if (message.toLowerCase().includes("abort")) activityMonitor.logComplete(activityId, 0); else activityMonitor.logError(activityId, message); throw err; } }