import { parseHTML } from 'linkedom';
export type SearchResult = {
title: string;
url: string;
snippet: string;
};
export type WebSearchOptions = {
allowed_domains?: string[];
blocked_domains?: string[];
signal?: AbortSignal;
};
const USER_AGENT = 'pi-browser-tools/1.0';
const FETCH_TIMEOUT_MS = 15_000;
const MAX_RESULTS = 10;
const env =
(
globalThis as typeof globalThis & {
process?: { env?: Record };
}
).process?.env ?? {};
function getFetchSignal(signal?: AbortSignal): AbortSignal {
return signal
? AbortSignal.any([signal, AbortSignal.timeout(FETCH_TIMEOUT_MS)])
: AbortSignal.timeout(FETCH_TIMEOUT_MS);
}
function normalizeUrl(rawUrl: string): string | null {
try {
const url = new URL(rawUrl);
url.hash = '';
return url.toString();
} catch {
return null;
}
}
function matchesDomain(hostname: string, domain: string): boolean {
const normalizedHost = hostname.toLowerCase();
const normalizedDomain = domain.toLowerCase();
return normalizedHost === normalizedDomain || normalizedHost.endsWith(`.${normalizedDomain}`);
}
function filterResults(results: SearchResult[], options: WebSearchOptions): SearchResult[] {
const seen = new Set();
const allowed = (options.allowed_domains ?? []).map((domain) => domain.toLowerCase());
const blocked = (options.blocked_domains ?? []).map((domain) => domain.toLowerCase());
const filtered: SearchResult[] = [];
for (const result of results) {
const normalizedUrl = normalizeUrl(result.url);
if (!normalizedUrl || seen.has(normalizedUrl)) continue;
let hostname: string;
try {
hostname = new URL(normalizedUrl).hostname;
} catch {
continue;
}
if (blocked.some((domain) => matchesDomain(hostname, domain))) continue;
if (allowed.length > 0 && !allowed.some((domain) => matchesDomain(hostname, domain))) continue;
seen.add(normalizedUrl);
filtered.push({
title: result.title.trim(),
url: normalizedUrl,
snippet: result.snippet.trim(),
});
if (filtered.length >= MAX_RESULTS) break;
}
return filtered;
}
function decodeDuckDuckGoUrl(href: string): string | null {
try {
const url = new URL(href, 'https://html.duckduckgo.com');
const uddg = url.searchParams.get('uddg');
if (uddg) return decodeURIComponent(uddg);
return url.toString();
} catch {
return null;
}
}
async function searchDuckDuckGo(query: string, signal?: AbortSignal): Promise {
const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
const response = await fetch(url, {
method: 'GET',
redirect: 'follow',
signal: getFetchSignal(signal),
headers: {
'user-agent': USER_AGENT,
accept: 'text/html,application/xhtml+xml',
},
});
if (!response.ok) {
throw new Error(`DuckDuckGo search failed: ${response.status} ${response.statusText}`);
}
const html = await response.text();
const { document } = parseHTML(html);
const anchors = Array.from(document.querySelectorAll('a.result__a'));
return anchors
.map((anchor) => {
const title = anchor.textContent?.trim() ?? '';
const href = anchor.getAttribute('href') ?? '';
const resultRoot =
anchor.closest('.result') ?? anchor.parentElement?.parentElement ?? anchor.parentElement;
const snippet = resultRoot?.querySelector('.result__snippet')?.textContent?.trim() ?? '';
return {
title,
url: decodeDuckDuckGoUrl(href) ?? href,
snippet,
};
})
.filter((result) => Boolean(result.title && result.url));
}
async function searchGoogle(query: string, signal?: AbortSignal): Promise {
const apiKey = env.GOOGLE_CSE_API_KEY;
const cseId = env.GOOGLE_CSE_ID;
if (!apiKey || !cseId) {
throw new Error('Google search requires GOOGLE_CSE_API_KEY and GOOGLE_CSE_ID');
}
const url = new URL('https://www.googleapis.com/customsearch/v1');
url.searchParams.set('key', apiKey);
url.searchParams.set('cx', cseId);
url.searchParams.set('q', query);
const response = await fetch(url, {
method: 'GET',
signal: getFetchSignal(signal),
headers: {
'user-agent': USER_AGENT,
accept: 'application/json',
},
});
if (!response.ok) {
throw new Error(`Google search failed: ${response.status} ${response.statusText}`);
}
const json = (await response.json()) as {
items?: Array<{ title?: string; link?: string; snippet?: string }>;
};
return (json.items ?? [])
.map((item) => ({
title: item.title ?? '',
url: item.link ?? '',
snippet: item.snippet ?? '',
}))
.filter((result) => Boolean(result.title && result.url));
}
async function searchBrave(query: string, signal?: AbortSignal): Promise {
const apiKey = env.BRAVE_SEARCH_API_KEY;
if (!apiKey) {
throw new Error('Brave search requires BRAVE_SEARCH_API_KEY');
}
const url = new URL('https://api.search.brave.com/res/v1/web/search');
url.searchParams.set('q', query);
const response = await fetch(url, {
method: 'GET',
signal: getFetchSignal(signal),
headers: {
'user-agent': USER_AGENT,
accept: 'application/json',
'x-subscription-token': apiKey,
},
});
if (!response.ok) {
throw new Error(`Brave search failed: ${response.status} ${response.statusText}`);
}
const json = (await response.json()) as {
web?: {
results?: Array<{ title?: string; url?: string; description?: string }>;
};
};
return (json.web?.results ?? [])
.map((item) => ({
title: item.title ?? '',
url: item.url ?? '',
snippet: item.description ?? '',
}))
.filter((result) => Boolean(result.title && result.url));
}
export async function webSearch(
query: string,
options: WebSearchOptions = {},
): Promise<{ results: SearchResult[] }> {
const provider = (env.WEB_SEARCH_PROVIDER ?? 'duckduckgo').toLowerCase();
const rawResults =
provider === 'google'
? await searchGoogle(query, options.signal)
: provider === 'brave'
? await searchBrave(query, options.signal)
: await searchDuckDuckGo(query, options.signal);
return {
results: filterResults(rawResults, options),
};
}