import type { FetchProviderAdapter } from "./provider.ts"; export interface FetchProviderDefinition { id: FetchProviderAdapter["id"]; label: string; hosted: boolean; load(): Promise; } function define(id: FetchProviderDefinition["id"], label: string, hosted: boolean, factory: () => Promise): FetchProviderDefinition { let loaded: Promise | undefined; return { id, label, hosted, load: () => loaded ??= factory() }; } const definitions: FetchProviderDefinition[] = [ define("http", "HTTP", false, async () => { const provider = await import("./http-extractor.ts"); return { id: "http", label: "HTTP", hosted: false, isAvailable: () => true, fetch: provider.fetchWithHttp }; }), define("firecrawl", "Firecrawl", false, async () => { const [provider, ssrf] = await Promise.all([import("../providers/firecrawl.ts"), import("../net/network-config.ts")]); return { id: "firecrawl", label: "Firecrawl", hosted: false, isAvailable: provider.isFirecrawlAvailable, fetch: (url, signal, options = {}) => provider.extractWithFirecrawl(url, signal, { timeoutMs: options.timeoutMs, ...(options.lookup ? { lookup: options.lookup } : {}), ssrf: ssrf.loadSsrfConfig() }) }; }), define("jina", "Jina", true, async () => { const provider = await import("./jina.ts"); return { id: "jina", label: "Jina", hosted: true, isAvailable: () => true, fetch: provider.fetchWithJina }; }), define("tinyfish", "TinyFish", true, async () => { const provider = await import("../providers/tinyfish.ts"); return { id: "tinyfish", label: "TinyFish", hosted: true, isAvailable: provider.isTinyFishAvailable, fetch: provider.extractWithTinyFish }; }), define("search1api", "Search1API", true, async () => { const provider = await import("../providers/search1api.ts"); return { id: "search1api", label: "Search1API", hosted: true, isAvailable: provider.isSearch1APIAvailable, fetch: provider.extractWithSearch1API }; }), define("querit", "Querit", true, async () => { const provider = await import("../providers/querit.ts"); return { id: "querit", label: "Querit", hosted: true, isAvailable: provider.isQueritAvailable, fetch: provider.extractWithQuerit }; }), define("kagi", "Kagi", true, async () => { const [provider, ssrf] = await Promise.all([import("../providers/kagi.ts"), import("../net/network-config.ts")]); return { id: "kagi", label: "Kagi", hosted: true, isAvailable: provider.isKagiExtractAvailable, fetch: (url, signal, options = {}) => provider.extractWithKagi(url, signal, { timeoutMs: options.timeoutMs, ...(options.lookup ? { lookup: options.lookup } : {}), ssrf: ssrf.loadSsrfConfig() }) }; }), define("ollama", "Ollama", true, async () => { const [provider, ssrf] = await Promise.all([import("../providers/ollama.ts"), import("../net/network-config.ts")]); return { id: "ollama", label: "Ollama", hosted: true, isAvailable: provider.isOllamaFetchAvailable, fetch: (url, signal, options = {}) => provider.extractWithOllama(url, signal, { timeoutMs: options.timeoutMs, ...(options.lookup ? { lookup: options.lookup } : {}), ssrf: ssrf.loadSsrfConfig() }) }; }), define("parallel", "Parallel", true, async () => { const provider = await import("../providers/parallel.ts"); return { id: "parallel", label: "Parallel", hosted: true, isAvailable: provider.isParallelAvailable, fetch: provider.extractWithParallel }; }), define("parallel-mcp", "Parallel MCP", true, async () => { const provider = await import("../providers/parallel-mcp.ts"); return { id: "parallel-mcp", label: "Parallel MCP", hosted: true, isAvailable: () => true, fetch: provider.extractWithParallelMcp }; }), define("brightdata", "Bright Data", true, async () => { const [provider, ssrf] = await Promise.all([import("../providers/brightdata/fetch.ts"), import("../net/network-config.ts")]); return { id: "brightdata", label: "Bright Data", hosted: true, isAvailable: provider.isBrightDataUnlockerAvailable, fetch: (url, signal, options = {}) => provider.extractWithBrightDataUnlocker(url, signal, { timeoutMs: options.timeoutMs, ...(options.lookup ? { lookup: options.lookup } : {}), ssrf: ssrf.loadSsrfConfig() }) }; }), define("gemini", "Gemini", true, async () => { const provider = await import("../providers/gemini/fetch.ts"); return { id: "gemini", label: "Gemini", hosted: true, isAvailable: () => true, fetch: async (url, signal) => await provider.extractWithUrlContext(url, signal) ?? await provider.extractWithGeminiWeb(url, signal) }; }), ]; export const fetchProviderRegistry = new Map(definitions.map(provider => [provider.id, provider])); export function getFetchProviderDefinition(id: FetchProviderDefinition["id"]): FetchProviderDefinition { const provider = fetchProviderRegistry.get(id); if (!provider) throw new Error(`Unknown fetch Provider: ${id}`); return provider; } export async function loadFetchProvider(id: FetchProviderDefinition["id"]): Promise { return getFetchProviderDefinition(id).load(); }