{"version":3,"file":"firecrawl.mjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport { processContent } from './content';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private version: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n  private includeTags?: string[];\n  private excludeTags?: string[];\n  private waitFor?: number;\n  private maxAge?: number;\n  private mobile?: boolean;\n  private skipTlsVerification?: boolean;\n  private blockAds?: boolean;\n  private removeBase64Images?: boolean;\n  private parsePDF?: boolean;\n  private storeInCache?: boolean;\n  private zeroDataRetention?: boolean;\n  private headers?: Record<string, string>;\n  private location?: { country?: string; languages?: string[] };\n  private onlyMainContent?: boolean;\n  private changeTrackingOptions?: object;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    this.version = config.version ?? 'v2';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/${this.version}/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];\n    this.timeout = config.timeout ?? 7500;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    this.includeTags = config.includeTags;\n    this.excludeTags = config.excludeTags;\n    this.waitFor = config.waitFor;\n    this.maxAge = config.maxAge;\n    this.mobile = config.mobile;\n    this.skipTlsVerification = config.skipTlsVerification;\n    this.blockAds = config.blockAds;\n    this.removeBase64Images = config.removeBase64Images;\n    this.parsePDF = config.parsePDF;\n    this.storeInCache = config.storeInCache;\n    this.zeroDataRetention = config.zeroDataRetention;\n    this.headers = config.headers;\n    this.location = config.location;\n    this.onlyMainContent = config.onlyMainContent;\n    this.changeTrackingOptions = config.changeTrackingOptions;\n\n    if (!this.apiKey) {\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const payload = omitUndefined({\n        url,\n        formats: options.formats ?? this.defaultFormats,\n        includeTags: options.includeTags ?? this.includeTags,\n        excludeTags: options.excludeTags ?? this.excludeTags,\n        headers: options.headers ?? this.headers,\n        waitFor: options.waitFor ?? this.waitFor,\n        timeout: options.timeout ?? this.timeout,\n        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n        maxAge: options.maxAge ?? this.maxAge,\n        mobile: options.mobile ?? this.mobile,\n        skipTlsVerification:\n          options.skipTlsVerification ?? this.skipTlsVerification,\n        parsePDF: options.parsePDF ?? this.parsePDF,\n        location: options.location ?? this.location,\n        removeBase64Images:\n          options.removeBase64Images ?? this.removeBase64Images,\n        blockAds: options.blockAds ?? this.blockAds,\n        storeInCache: options.storeInCache ?? this.storeInCache,\n        zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,\n        changeTrackingOptions:\n          options.changeTrackingOptions ?? this.changeTrackingOptions,\n      });\n      const response = await axios.post(this.apiUrl, payload, {\n        headers: {\n          'Content-Type': 'application/json',\n          Authorization: `Bearer ${this.apiKey}`,\n        },\n        timeout: this.timeout,\n      });\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n\n// Helper function to clean up payload for firecrawl\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n  return Object.fromEntries(\n    Object.entries(obj).filter(([, v]) => v !== undefined)\n  ) as Partial<T>;\n}\n"],"names":[],"mappings":";;;;AAKA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,OAAO;AACP,IAAA,cAAc;AACd,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,WAAW;AACX,IAAA,WAAW;AACX,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,mBAAmB;AACnB,IAAA,QAAQ;AACR,IAAA,kBAAkB;AAClB,IAAA,QAAQ;AACR,IAAA,YAAY;AACZ,IAAA,iBAAiB;AACjB,IAAA,OAAO;AACP,IAAA,QAAQ;AACR,IAAA,eAAe;AACf,IAAA,qBAAqB;AAE7B,IAAA,WAAA,CAAY,SAAmC,EAAE,EAAA;AAC/C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;QAElE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;AAErC,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA,CAAA,EAAI,IAAI,CAAC,OAAO,SAAS;AAErE,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC;QAC/D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB;AACrD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB;AACnD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY;AACvC,QAAA,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB;AACjD,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe;AAC7C,QAAA,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,qBAAqB;AAEzD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC;QAC3E;QAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,4CAAA,EAA+C,IAAI,CAAC,MAAM,CAAA,CAAE,CAC7D;IACH;AAEA;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAoC,EAAE,EAAA;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;QACH;AAEA,QAAA,IAAI;YACF,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;AAC/C,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;AAChE,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,mBAAmB,EACjB,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB;AACzD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,kBAAkB,EAChB,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB;AACvD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;AACvD,gBAAA,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB;AACtE,gBAAA,qBAAqB,EACnB,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB;AAC9D,aAAA,CAAC;AACF,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACtD,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAA,CAAE;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CAAC;AAEF,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;QAC7B;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAA,8BAAA,EAAiC,YAAY,CAAA,CAAE;AACvD,iBAAA;aACF;QACH;IACF;AAEA;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAmC,EAAA;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;QACxB;AAEA,QAAA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAChE,YAAA,IAAI;gBACF,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,cAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB;AACD,gBAAA,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;YACzB;YAAE,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;YAC5C;QACF;aAAO,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;QAC5C;;QAGA,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;QACxC;;QAGA,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;QAC3C;AAEA,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;IACxB;AAEA;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAmC,EAAA;AACjD,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;QACX;AAEA,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;IAC/B;AACD;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAA,GAAmC,EAAE,KACjB;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;AAEA;AACA,SAAS,aAAa,CAAmB,GAAM,EAAA;IAC7C,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,CACzC;AACjB;;;;"}