/** * Provides search query autocomplete/suggestions from various search engines. */ import { parseHTML } from "linkedom"; // Some suggest APIs (notably Google) return 403 Forbidden for requests // without a browser-like User-Agent, especially from datacenter/Cloudflare // egress IPs. Sent by default; per-backend headers can still override it. const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; /** * Fetch wrapper compatible with grab-url interface */ async function grab(url: string, options: any = {}): Promise { const timeout = options.timeout ? options.timeout * 1000 : 5000; const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeout); try { const response = await fetch(url, { ...options, headers: { "User-Agent": DEFAULT_USER_AGENT, ...options.headers, }, signal: controller.signal, }); clearTimeout(timeoutId); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } if (options.responseType === "json") { return { data: await response.json() }; } if (options.responseType === "arraybuffer") { return await response.arrayBuffer(); } if (options.responseType === "text") { return await response.text(); } // Default to json for backwards compatibility try { return { data: await response.json() }; } catch { return await response.text(); } } catch (error) { clearTimeout(timeoutId); throw error; } } /** * Autocomplete function type */ type AutocompleteFunction = ( query: string, locale?: string, ) => Promise; /** * Get autocomplete suggestions with error handling */ async function getSuggestions(url: string, options: any = {}): Promise { try { const response = await grab(url, { timeout: 3, responseType: options.responseType || "json", ...options, }); if (typeof response === "object" && "data" in response) { return response.data; } return response; } catch (error) { console.error("Autocomplete error:", error); return null; } } /** * Baidu autocomplete */ export async function baidu( query: string, _locale?: string, ): Promise { const params = new URLSearchParams({ ie: "utf-8", json: "1", prod: "pc", wd: query, }); const url = `https://www.baidu.com/sugrec?${params.toString()}`; const data = await getSuggestions(url); const results: string[] = []; if (data && data.g) { for (const item of data.g) { results.push(item.q); } } return results; } /** * Brave autocomplete */ export async function brave( query: string, _locale?: string, ): Promise { const params = new URLSearchParams({ q: query }); const url = `https://search.brave.com/api/suggest?${params.toString()}`; const data = await getSuggestions(url, { headers: { Cookie: "country=all", }, }); if (data && Array.isArray(data) && data.length > 1) { return data[1]; } return []; } /** * DuckDuckGo autocomplete */ export async function duckduckgo( query: string, locale: string = "en-US", ): Promise { // Extract region code (e.g., 'us-en' from 'en-US') const region = locale.toLowerCase().split("-").reverse().join("-"); const params = new URLSearchParams({ q: query, kl: region, }); const url = `https://duckduckgo.com/ac/?type=list&${params.toString()}`; const data = await getSuggestions(url); if (data && Array.isArray(data) && data.length > 1) { return data[1]; } return []; } /** * Google autocomplete */ export async function google( query: string, locale: string = "en", ): Promise { // Map locale to Google subdomain const subdomainMap: { [key: string]: string } = { de: "google.de", fr: "google.fr", es: "google.es", it: "google.it", nl: "google.nl", pt: "google.pt", ru: "google.ru", ja: "google.co.jp", zh: "google.com.hk", ko: "google.co.kr", }; const lang = locale.split("-")[0]; const subdomain = subdomainMap[lang] || "google.com"; const params = new URLSearchParams({ q: query, client: "gws-wiz", hl: lang, }); const url = `https://${subdomain}/complete/search?${params.toString()}`; const response = await getSuggestions(url, { responseType: "text" }); const results: string[] = []; if (response) { try { // Extract JSON from response const jsonStart = response.indexOf("["); const jsonEnd = response.lastIndexOf("]") + 1; if (jsonStart >= 0 && jsonEnd > jsonStart) { const jsonText = response.substring(jsonStart, jsonEnd); const data = JSON.parse(jsonText); if (data[0]) { for (const item of data[0]) { // Parse HTML entities const { document } = parseHTML(item[0]); const text = document.body?.textContent?.trim(); if (text) { results.push(text); } } } } } catch (e) { // Ignore parse errors } } return results; } /** * Qwant autocomplete */ export async function qwant( query: string, locale: string = "en_US", ): Promise { const params = new URLSearchParams({ q: query, locale: locale.replace("-", "_"), version: "2", }); const url = `https://api.qwant.com/v3/suggest?${params.toString()}`; const data = await getSuggestions(url); const results: string[] = []; if (data && data.status === "success" && data.data && data.data.items) { for (const item of data.data.items) { results.push(item.value); } } return results; } /** * Startpage autocomplete */ export async function startpage( query: string, locale: string = "en", ): Promise { const langMap: { [key: string]: string } = { da: "dansk", de: "deutsch", en: "english", es: "espanol", fr: "francais", nb: "norsk", nl: "nederlands", pl: "polski", pt: "portugues", sv: "svenska", }; const baseLang = locale.split("-")[0]; const lui = langMap[baseLang] || "english"; const params = new URLSearchParams({ q: query, format: "opensearch", segment: "startpage.defaultffx", lui: lui, }); const url = `https://www.startpage.com/suggestions?${params.toString()}`; const data = await getSuggestions(url, { headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", }, }); if ( data && Array.isArray(data) && data.length >= 2 && Array.isArray(data[1]) ) { return data[1]; } return []; } /** * Wikipedia autocomplete */ export async function wikipedia( query: string, locale: string = "en", ): Promise { const langMap: { [key: string]: string } = { en: "en.wikipedia.org", de: "de.wikipedia.org", fr: "fr.wikipedia.org", es: "es.wikipedia.org", it: "it.wikipedia.org", nl: "nl.wikipedia.org", pt: "pt.wikipedia.org", ru: "ru.wikipedia.org", ja: "ja.wikipedia.org", zh: "zh.wikipedia.org", ar: "ar.wikipedia.org", ko: "ko.wikipedia.org", }; const lang = locale.split("-")[0]; const netloc = langMap[lang] || "en.wikipedia.org"; const params = new URLSearchParams({ action: "opensearch", format: "json", formatversion: "2", search: query, namespace: "0", limit: "10", }); const url = `https://${netloc}/w/api.php?${params.toString()}`; const data = await getSuggestions(url); if (data && Array.isArray(data) && data.length > 1) { return data[1]; } return []; } /** * Yandex autocomplete */ export async function yandex( query: string, _locale?: string, ): Promise { const params = new URLSearchParams({ part: query }); const url = `https://suggest.yandex.com/suggest-ff.cgi?${params.toString()}`; const data = await getSuggestions(url); if (data && Array.isArray(data) && data.length > 1) { return data[1]; } return []; } /** * Available autocomplete backends */ export const backends: { [key: string]: AutocompleteFunction } = { baidu, brave, duckduckgo, google, qwant, startpage, wikipedia, yandex, }; /** * Get autocomplete suggestions from a specific backend * * @param backendName - Name of the autocomplete backend * @param query - Search query * @param locale - Locale/language code (e.g., 'en-US', 'de-DE') * @returns Array of suggestion strings */ export async function searchAutocomplete( backendName: string, query: string, locale: string = "en-US", ): Promise { const backend = backends[backendName]; if (!backend) { console.warn(`Autocomplete backend '${backendName}' not found`); return []; } try { return await backend(query, locale); } catch (error) { console.error(`Autocomplete error for ${backendName}:`, error); return []; } } /** * Get autocomplete suggestions from multiple backends and merge them * * @param backendNames - Array of backend names to query * @param query - Search query * @param locale - Locale/language code * @returns Merged and deduplicated array of suggestions */ export async function searchAutocompleteMulti( backendNames: string[], query: string, locale: string = "en-US", ): Promise { const promises = backendNames.map((name) => searchAutocomplete(name, query, locale), ); const results = await Promise.all(promises); // Merge and deduplicate const merged = new Set(); for (const result of results) { for (const suggestion of result) { merged.add(suggestion); } } return Array.from(merged); }