import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { Type, type Static } from "typebox"; import { Firecrawl, type FirecrawlClientOptions } from "firecrawl"; const DEFAULT_TIMEOUT_MS = 60_000; const DEFAULT_MAX_AGE_MS = 172_800_000; const VISIBLE_TEXT_LIMIT = 12_000; const REGISTERED_TOOLS = ["web_search", "fetch_content", "extract_structured"] as const; const DEFAULT_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev"; const SearchSource = Type.Union([Type.Literal("web"), Type.Literal("news"), Type.Literal("images")]); const SearchCategory = Type.Union([Type.Literal("github"), Type.Literal("research"), Type.Literal("pdf")]); const ContentFormat = Type.Union([Type.Literal("markdown"), Type.Literal("summary")]); const ScrapeFormat = Type.Union([ Type.Literal("markdown"), Type.Literal("summary"), Type.Literal("html"), Type.Literal("rawHtml"), Type.Literal("links"), Type.Literal("images"), Type.Literal("screenshot"), Type.Literal("json"), Type.Literal("question"), Type.Literal("highlights"), ]); export const webSearchSchema = Type.Object({ query: Type.String({ maxLength: 500, description: "Search query" }), limit: Type.Optional(Type.Number({ minimum: 1, maximum: 20, default: 5 })), sources: Type.Optional(Type.Array(SearchSource, { default: ["web"] })), categories: Type.Optional(Type.Array(SearchCategory)), includeDomains: Type.Optional(Type.Array(Type.String())), excludeDomains: Type.Optional(Type.Array(Type.String())), country: Type.Optional(Type.String({ default: "US" })), location: Type.Optional(Type.String()), includeContent: Type.Optional(Type.Boolean({ default: false })), contentFormat: Type.Optional(ContentFormat), timeoutMs: Type.Optional(Type.Number({ minimum: 1, default: DEFAULT_TIMEOUT_MS })), }); export const fetchContentSchema = Type.Object({ url: Type.String({ format: "uri" }), formats: Type.Optional(Type.Array(ScrapeFormat, { default: ["markdown"] })), onlyMainContent: Type.Optional(Type.Boolean({ default: true })), onlyCleanContent: Type.Optional(Type.Boolean({ default: false })), includeTags: Type.Optional(Type.Array(Type.String())), excludeTags: Type.Optional(Type.Array(Type.String())), waitFor: Type.Optional(Type.Number({ minimum: 0 })), mobile: Type.Optional(Type.Boolean()), timeoutMs: Type.Optional(Type.Number({ minimum: 1, default: DEFAULT_TIMEOUT_MS })), maxAgeMs: Type.Optional(Type.Number({ minimum: 0, default: DEFAULT_MAX_AGE_MS })), prompt: Type.Optional(Type.String()), schema: Type.Optional(Type.Any()), removeBase64Images: Type.Optional(Type.Boolean({ default: true })), blockAds: Type.Optional(Type.Boolean({ default: true })), }); const SafeScrapeOptions = Type.Object({ formats: Type.Optional(Type.Array(ScrapeFormat)), onlyMainContent: Type.Optional(Type.Boolean()), onlyCleanContent: Type.Optional(Type.Boolean()), timeout: Type.Optional(Type.Number({ minimum: 1 })), waitFor: Type.Optional(Type.Number({ minimum: 0 })), mobile: Type.Optional(Type.Boolean()), blockAds: Type.Optional(Type.Boolean()), removeBase64Images: Type.Optional(Type.Boolean()), }); export const extractStructuredSchema = Type.Object({ urls: Type.Array(Type.String({ format: "uri" }), { minItems: 1 }), prompt: Type.Optional(Type.String()), schema: Type.Optional(Type.Any()), enableWebSearch: Type.Optional(Type.Boolean({ default: false })), showSources: Type.Optional(Type.Boolean({ default: true })), ignoreSitemap: Type.Optional(Type.Boolean({ default: false })), includeSubdomains: Type.Optional(Type.Boolean({ default: true })), ignoreInvalidURLs: Type.Optional(Type.Boolean({ default: true })), scrapeOptions: Type.Optional(SafeScrapeOptions), timeoutMs: Type.Optional(Type.Number({ minimum: 1, default: DEFAULT_TIMEOUT_MS })), }); export type WebSearchInput = Static; export type FetchContentInput = Static; export type ExtractStructuredInput = Static; type FirecrawlLike = Pick; type ToolResult = { content: Array<{ type: "text"; text: string }>; details: unknown; isError?: boolean; }; function getConfigObject(pi?: ExtensionAPI): Record { const candidate = (pi ?? {}) as unknown as { config?: unknown; getConfig?: () => unknown; extensionConfig?: unknown; }; const value = candidate.config ?? candidate.extensionConfig ?? candidate.getConfig?.(); return value && typeof value === "object" ? (value as Record) : {}; } export type FirecrawlConnectionConfig = { apiKey?: string; baseUrl: string; isSelfHosted: boolean; hasAuth: boolean; }; export function resolveApiKey(pi?: ExtensionAPI): string | undefined { const configKey = getConfigObject(pi).firecrawlApiKey; const envKey = process.env.FIRECRAWL_API_KEY; const value = typeof configKey === "string" && configKey.trim() ? configKey : envKey; return typeof value === "string" && value.trim() ? value.trim() : undefined; } export function resolveBaseUrl(pi?: ExtensionAPI): string { const configUrl = getConfigObject(pi).firecrawlBaseUrl; const value = typeof configUrl === "string" && configUrl.trim() ? configUrl : process.env.FIRECRAWL_BASE_URL; return (typeof value === "string" && value.trim() ? value.trim() : DEFAULT_FIRECRAWL_BASE_URL).replace(/\/$/, ""); } export function resolveFirecrawlConfig(pi?: ExtensionAPI): FirecrawlConnectionConfig { const apiKey = resolveApiKey(pi); const baseUrl = resolveBaseUrl(pi); return { apiKey, baseUrl, isSelfHosted: baseUrl !== DEFAULT_FIRECRAWL_BASE_URL, hasAuth: Boolean(apiKey), }; } export function createFirecrawlClientOptions(config: FirecrawlConnectionConfig): FirecrawlClientOptions { return { apiUrl: config.baseUrl, ...(config.apiKey ? { apiKey: config.apiKey } : {}), }; } export function buildAuthorizationHeaders(config: FirecrawlConnectionConfig): Record { return config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}; } function assertNotAborted(signal?: AbortSignal) { if (signal?.aborted) throw new Error("Operation aborted"); } export function isHttpUrl(value: string): boolean { try { const parsed = new URL(value); return parsed.protocol === "http:" || parsed.protocol === "https:"; } catch { return false; } } export function assertHttpUrl(value: string, field = "url") { if (!isHttpUrl(value)) throw new Error(`${field} must be an http:// or https:// URL. Other protocols are blocked.`); } export function isHostnameOnly(value: string): boolean { if (!value || value.includes("://") || value.includes("/") || value.includes("?") || value.includes("#")) return false; try { const parsed = new URL(`http://${value}`); return parsed.hostname === value.toLowerCase() || parsed.hostname === value; } catch { return false; } } function validateDomains(domains: string[] | undefined, field: string) { for (const domain of domains ?? []) { if (!isHostnameOnly(domain)) throw new Error(`${field} entries must be hostnames only, without protocol or path: ${domain}`); } } function friendlyError(error: unknown): string { if (error instanceof Error) return error.message.split("\n").slice(0, 3).join("\n"); return String(error); } function errorResult(error: unknown): ToolResult { return { isError: true, content: [{ type: "text", text: `Firecrawl request failed: ${friendlyError(error)}` }], details: { error: friendlyError(error) }, }; } function truncateVisible(text: string, limit = VISIBLE_TEXT_LIMIT): { text: string; truncated: boolean } { if (text.length <= limit) return { text, truncated: false }; return { text: `${text.slice(0, limit)}\n\n[Output truncated for display. Full response metadata is available in details.]`, truncated: true }; } function compactJson(value: unknown): string { return JSON.stringify(value, null, 0); } function normalizeSearchResult(raw: any, query: string) { const normalizeWeb = (item: any, index: number) => ({ title: item.title, url: item.url, description: item.description ?? item.snippet, position: item.position ?? index + 1, markdown: item.markdown, summary: item.summary, }); const normalizeNews = (item: any, index: number) => ({ title: item.title, url: item.url, snippet: item.snippet ?? item.description, date: item.date, position: item.position ?? index + 1, }); const normalizeImage = (item: any, index: number) => ({ title: item.title, imageUrl: item.imageUrl ?? item.image_url ?? item.url, url: item.url ?? item.sourceUrl, width: item.width, height: item.height, position: item.position ?? index + 1, }); return { query, ...(raw.web ? { web: raw.web.map(normalizeWeb) } : {}), ...(raw.news ? { news: raw.news.map(normalizeNews) } : {}), ...(raw.images ? { images: raw.images.map(normalizeImage) } : {}), }; } function formatSearchText(normalized: any): string { const lines = [`Firecrawl search results for: ${normalized.query}`]; for (const item of normalized.web ?? []) lines.push(`- [${item.position}] ${item.title ?? "Untitled"}: ${item.url}${item.description ? ` — ${item.description}` : ""}`); for (const item of normalized.news ?? []) lines.push(`- [news ${item.position}] ${item.title ?? "Untitled"}: ${item.url}${item.snippet ? ` — ${item.snippet}` : ""}`); for (const item of normalized.images ?? []) lines.push(`- [image ${item.position}] ${item.title ?? "Image"}: ${item.imageUrl}${item.url && item.url !== item.imageUrl ? ` (source: ${item.url})` : ""}`); return lines.join("\n"); } function createFirecrawl(config: FirecrawlConnectionConfig): FirecrawlLike { return new Firecrawl(createFirecrawlClientOptions(config)); } export async function executeWebSearch(params: WebSearchInput, app: FirecrawlLike, signal?: AbortSignal): Promise { assertNotAborted(signal); validateDomains(params.includeDomains, "includeDomains"); validateDomains(params.excludeDomains, "excludeDomains"); const includeContent = params.includeContent ?? false; const contentFormat = params.contentFormat ?? "markdown"; const options: Record = { limit: Math.min(Math.max(params.limit ?? 5, 1), 20), sources: params.sources ?? ["web"], categories: params.categories, includeDomains: params.includeDomains, excludeDomains: params.excludeDomains, country: params.country ?? "US", location: params.location, timeout: params.timeoutMs ?? DEFAULT_TIMEOUT_MS, ...(includeContent ? { scrapeOptions: { formats: [contentFormat] } } : {}), }; const raw = await app.search(params.query, options as never); assertNotAborted(signal); const normalized = normalizeSearchResult(raw, params.query); return { content: [{ type: "text", text: formatSearchText(normalized) }], details: { normalized, raw } }; } function buildScrapeOptions(params: FetchContentInput): Record { const formats = params.formats ?? ["markdown"]; const mappedFormats = formats.map((format) => (format === "summary" || format === "question" || format === "highlights" ? { type: format, prompt: params.prompt } : format === "json" ? { type: "json", schema: params.schema, prompt: params.prompt } : format)); return { formats: mappedFormats, onlyMainContent: params.onlyMainContent ?? true, onlyCleanContent: params.onlyCleanContent ?? false, includeTags: params.includeTags, excludeTags: params.excludeTags, waitFor: params.waitFor, mobile: params.mobile, timeout: params.timeoutMs ?? DEFAULT_TIMEOUT_MS, maxAge: params.maxAgeMs ?? DEFAULT_MAX_AGE_MS, removeBase64Images: params.removeBase64Images ?? true, blockAds: params.blockAds ?? true, }; } export async function executeFetchContent(params: FetchContentInput, app: FirecrawlLike, signal?: AbortSignal): Promise { assertHttpUrl(params.url); assertNotAborted(signal); const raw: any = await app.scrape(params.url, buildScrapeOptions(params) as never); assertNotAborted(signal); const formats = params.formats ?? ["markdown"]; if (formats.includes("json") && raw.json !== undefined) { return { content: [{ type: "text", text: compactJson(raw.json) }], details: raw }; } const visible = String(raw.markdown ?? raw.summary ?? raw.html ?? raw.rawHtml ?? raw.content ?? compactJson(raw)); const truncated = truncateVisible(visible); return { content: [{ type: "text", text: truncated.text }], details: { ...raw, displayTruncated: truncated.truncated } }; } function safeExtractScrapeOptions(options: ExtractStructuredInput["scrapeOptions"]): Record | undefined { if (!options) return undefined; return { formats: options.formats, onlyMainContent: options.onlyMainContent, onlyCleanContent: options.onlyCleanContent, timeout: options.timeout, waitFor: options.waitFor, mobile: options.mobile, blockAds: options.blockAds, removeBase64Images: options.removeBase64Images, }; } function hasWildcard(urls: string[]) { return urls.some((url) => url.includes("/*")); } export async function executeExtractStructured(params: ExtractStructuredInput, app: FirecrawlLike, signal?: AbortSignal): Promise { if (!params.prompt && !params.schema) throw new Error("extract_structured requires either prompt or schema."); for (const url of params.urls) assertHttpUrl(url, "urls[]"); assertNotAborted(signal); const timeoutMs = params.timeoutMs ?? DEFAULT_TIMEOUT_MS; const raw: any = await app.extract({ urls: params.urls, prompt: params.prompt, schema: params.schema, enableWebSearch: params.enableWebSearch ?? false, showSources: params.showSources ?? true, ignoreSitemap: params.ignoreSitemap ?? false, includeSubdomains: params.includeSubdomains ?? true, ignoreInvalidURLs: params.ignoreInvalidURLs ?? true, scrapeOptions: safeExtractScrapeOptions(params.scrapeOptions), timeout: Math.ceil(timeoutMs / 1000), } as never); assertNotAborted(signal); const warnings = [raw.warning, ...(raw.warnings ?? [])].filter(Boolean); if (hasWildcard(params.urls)) warnings.unshift("Warning: wildcard URLs ending in /* can consume many Firecrawl credits."); const body = raw.data ?? raw; const sourceLines = raw.sources?.length ? `\n\nSources:\n${raw.sources.map((source: string) => `- ${source}`).join("\n")}` : ""; const warningText = warnings.length ? `\n\n${warnings.join("\n")}` : ""; const text = `Firecrawl structured extraction result:\n${compactJson(body)}${sourceLines}${warningText}`; return { content: [{ type: "text", text: truncateVisible(text).text }], details: raw }; } export default function (pi: ExtensionAPI) { function getApp(): FirecrawlLike { return createFirecrawl(resolveFirecrawlConfig(pi)); } pi.registerTool({ name: "web_search", label: "Web Search", description: "Search the web using Firecrawl Search. Returns URLs, titles, descriptions, and optionally scraped markdown content.", promptSnippet: "Search the web with Firecrawl Search", promptGuidelines: ["Use web_search for open web discovery; keep limit small unless the user asks for broad coverage."], parameters: webSearchSchema, async execute(_toolCallId, params, signal, onUpdate) { onUpdate?.({ content: [{ type: "text", text: "Searching Firecrawl…" }], details: {} }); const app = getApp(); try { return await executeWebSearch(params, app, signal); } catch (error) { return errorResult(error); } }, }); pi.registerTool({ name: "fetch_content", label: "Fetch Content", description: "Fetch and clean content from a URL using Firecrawl Scrape. Supports markdown, summary, HTML, links, screenshots, and JSON extraction.", promptSnippet: "Fetch clean content from a known URL with Firecrawl Scrape", promptGuidelines: ["Use fetch_content when the user provides a specific URL or when web_search has found the URL to inspect."], parameters: fetchContentSchema, async execute(_toolCallId, params, signal, onUpdate) { onUpdate?.({ content: [{ type: "text", text: "Scraping URL…" }], details: {} }); const app = getApp(); try { return await executeFetchContent(params, app, signal); } catch (error) { return errorResult(error); } }, }); pi.registerTool({ name: "extract_structured", label: "Extract Structured", description: "Extract structured data from one or more URLs using Firecrawl Extract with optional JSON Schema.", promptSnippet: "Extract structured JSON from URLs with Firecrawl Extract", promptGuidelines: ["Use extract_structured when the user asks for specific fields, tables, lists, prices, or other structured data from URLs."], parameters: extractStructuredSchema, async execute(_toolCallId, params, signal, onUpdate) { onUpdate?.({ content: [{ type: "text", text: "Extracting structured data…" }], details: {} }); const app = getApp(); try { return await executeExtractStructured(params, app, signal); } catch (error) { return errorResult(error); } }, }); pi.registerCommand("firecrawl-status", { description: "Show Firecrawl extension status", handler: async (_args, ctx) => { const config = resolveFirecrawlConfig(pi); ctx.ui.notify( [ `Firecrawl base URL: ${config.baseUrl}${config.isSelfHosted ? " (self-hosted)" : " (cloud default)"}`, `Authentication: ${config.hasAuth ? "configured" : "not configured"}`, `Registered tools: ${REGISTERED_TOOLS.join(", ")}`, ].join("\n"), "info", ); }, }); }