import { describe, expect, it, vi } from "vitest"; import { fetchPage, htmlToMarkdown, parseHttpUrl } from "./fetch.js"; import type { FetchLike } from "./results.js"; const paragraph = "Exa 3.0 adds a cheaper Lite search type and a larger index for agents. ".repeat(6); const articleHtml = `Exa 3.0

Introducing Exa 3.0

${paragraph}

Pricing

TypePrice
Lite$4
curl https://api.exa.ai/search
`; function htmlResponse(body: string, contentType = "text/html; charset=utf-8", status = 200): Response { return new Response(body, { status, headers: { "content-type": contentType } }); } describe("parseHttpUrl", () => { it("accepts http(s) URLs only", () => { expect(parseHttpUrl("https://exa.ai/blog").hostname).toBe("exa.ai"); expect(() => parseHttpUrl("file:///etc/passwd")).toThrow(/Only http\(s\)/); expect(() => parseHttpUrl("not a url")).toThrow(/Invalid URL/); }); }); describe("htmlToMarkdown", () => { it("keeps headings, tables, and fenced code", () => { const markdown = htmlToMarkdown( "

Pricing

TypePrice
Lite$4
exa search
", ); expect(markdown).toContain("## Pricing"); expect(markdown).toContain("| Type | Price |"); expect(markdown).toContain("```\nexa search\n```"); }); }); describe("fetchPage", () => { it("returns the readable article as markdown", async () => { const fetchMock = vi.fn().mockResolvedValue(htmlResponse(articleHtml)); const page = await fetchPage("https://exa.ai/blog/exa-3", undefined, fetchMock); expect(page.title).toBe("Exa 3.0"); expect(page.text).toContain("Introducing Exa 3.0"); expect(page.text).toContain("| Lite | $4 |"); expect(page.text).not.toContain("Footer links"); }); it("returns non-HTML text bodies as-is", async () => { const fetchMock = vi.fn().mockResolvedValue(htmlResponse('{"ok":true}', "application/json")); await expect(fetchPage("https://api.example.com/status", undefined, fetchMock)).resolves.toEqual({ url: "https://api.example.com/status", text: '{"ok":true}', }); }); it("rejects HTTP errors, binary content, and pages without readable content", async () => { const fetchMock = vi .fn() .mockResolvedValueOnce(htmlResponse("blocked", "text/html", 403)) .mockResolvedValueOnce(htmlResponse("%PDF-1.7", "application/pdf")) .mockResolvedValueOnce(htmlResponse('
')); await expect(fetchPage("https://example.com/a", undefined, fetchMock)).rejects.toThrow(/HTTP 403/); await expect(fetchPage("https://example.com/b.pdf", undefined, fetchMock)).rejects.toThrow(/application\/pdf/); await expect(fetchPage("https://example.com/app", undefined, fetchMock)).rejects.toThrow(/needs? JavaScript/); }); });