import { errorMessage, formatServiceError, isAbortError } from "../shared/errors"; import { fetchJson, fetchText } from "../shared/http"; import { extractWithTavily, mapTavilyError } from "./tavily"; import { EXA_CONTENTS_URL, JINA_READER_URL, METASO_READER_URL } from "../types"; export interface ReadContent { url: string; content: string; } export interface ReadFailure { url: string; error: string; } export interface ReadResponse { results: ReadContent[]; failed: ReadFailure[]; } function normalizeUrlKey(url: string): string { return url.trim().replace(/\/+$/, "").toLowerCase(); } function exaContentText(result: any): string { if (typeof result?.text === "string" && result.text.trim().length > 0) return result.text; if (Array.isArray(result?.highlights) && result.highlights.length > 0) return result.highlights.join("\n\n"); if (typeof result?.summary === "string" && result.summary.trim().length > 0) return result.summary; return ""; } export async function extractWithTavilyEngine(urls: string[], format: "markdown" | "text", apiKey: string, signal?: AbortSignal): Promise { try { const data = await extractWithTavily(urls, format, apiKey, signal); return { results: (data.results ?? []).map((result) => ({ url: result.url, content: result.raw_content ?? "" })), failed: (data.failed_results ?? []).map((result) => ({ url: result.url, error: result.error })), }; } catch (error) { if (isAbortError(error)) throw error; throw new Error(mapTavilyError(error)); } } export async function extractWithJina(urls: string[], format: "markdown" | "text", apiKey: string, signal?: AbortSignal): Promise { const responses = await Promise.all(urls.map(async (url) => { try { const content = await fetchText(`${JINA_READER_URL}${url}`, { headers: { Authorization: `Bearer ${apiKey}`, Accept: "text/plain", "X-Respond-With": format === "text" ? "text" : "markdown", }, signal, }); return { ok: true as const, url, content }; } catch (error) { if (isAbortError(error)) throw error; return { ok: false as const, url, error: formatServiceError("Jina", error) }; } })); return { results: responses.filter((item): item is { ok: true; url: string; content: string } => item.ok).map(({ url, content }) => ({ url, content })), failed: responses.filter((item): item is { ok: false; url: string; error: string } => !item.ok).map(({ url, error }) => ({ url, error })), }; } export async function extractWithExa(urls: string[], _format: "markdown" | "text", apiKey: string, signal?: AbortSignal): Promise { try { const data = await fetchJson(EXA_CONTENTS_URL, { method: "POST", headers: { "Content-Type": "application/json", "x-api-key": apiKey, }, body: JSON.stringify({ urls, text: { maxCharacters: 4000 }, highlights: { maxCharacters: 4000 }, }), signal, }); const rawResults = Array.isArray(data.results) ? data.results : Array.isArray(data.content) ? data.content : []; const mapped = new Map(); for (const result of rawResults) { const url = result?.url ?? result?.id ?? ""; const content = exaContentText(result); if (!url || !content) continue; mapped.set(normalizeUrlKey(url), { url, content }); } const results: ReadContent[] = []; const failed: ReadFailure[] = []; for (const url of urls) { const match = mapped.get(normalizeUrlKey(url)); if (match) { results.push(match); } else { failed.push({ url, error: "Exa returned no readable content" }); } } return { results, failed }; } catch (error) { if (isAbortError(error)) throw error; throw new Error(formatServiceError("Exa", error)); } } export async function extractWithMetaso(urls: string[], _format: "markdown" | "text", apiKey: string, signal?: AbortSignal): Promise { const responses = await Promise.all(urls.map(async (url) => { try { const content = await fetchText(METASO_READER_URL, { method: "POST", headers: { Authorization: `Bearer ${apiKey}`, Accept: "text/plain", "Content-Type": "application/json", }, body: JSON.stringify({ url }), signal, }); return { ok: true as const, url, content }; } catch (error) { if (isAbortError(error)) throw error; return { ok: false as const, url, error: formatServiceError("Metaso", error) }; } })); return { results: responses.filter((item): item is { ok: true; url: string; content: string } => item.ok).map(({ url, content }) => ({ url, content })), failed: responses.filter((item): item is { ok: false; url: string; error: string } => !item.ok).map(({ url, error }) => ({ url, error })), }; } export function stringifyReadError(error: unknown): string { return error instanceof Error ? error.message : errorMessage(error); }