{"version":3,"file":"serper-scraper.cjs","sources":["../../../../src/tools/search/serper-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Serper scraper implementation\n * Uses the Serper Scrape API (https://scrape.serper.dev) to scrape web pages\n *\n * Features:\n * - Simple API with single endpoint\n * - Returns both text and markdown content\n * - Includes metadata from scraped pages\n * - Credits-based pricing model\n *\n * @example\n * ```typescript\n * const scraper = createSerperScraper({\n *   apiKey: 'your-serper-api-key',\n *   includeMarkdown: true,\n *   timeout: 10000\n * });\n *\n * const [url, response] = await scraper.scrapeUrl('https://example.com');\n * if (response.success) {\n *   const [content] = scraper.extractContent(response);\n *   console.log(content);\n * }\n * ```\n */\nexport class SerperScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private timeout: number;\n  private logger: t.Logger;\n  private includeMarkdown: boolean;\n\n  constructor(config: t.SerperScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.SERPER_API_KEY ?? '';\n\n    this.apiUrl =\n      config.apiUrl ??\n      process.env.SERPER_SCRAPE_URL ??\n      'https://scrape.serper.dev';\n\n    this.timeout = config.timeout ?? 7500;\n    this.includeMarkdown = config.includeMarkdown ?? true;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('SERPER_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Serper scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.SerperScrapeOptions = {}\n  ): Promise<[string, t.SerperScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'SERPER_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const payload = {\n        url,\n        includeMarkdown: options.includeMarkdown ?? this.includeMarkdown,\n      };\n\n      const response = await axios.post(this.apiUrl, payload, {\n        headers: {\n          'X-API-KEY': this.apiKey,\n          'Content-Type': 'application/json',\n        },\n        timeout: options.timeout ?? this.timeout,\n      });\n\n      return [url, { success: true, data: response.data }];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Serper Scrape API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.SerperScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    if (response.data.text != null) {\n      return [response.data.text, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(\n    response: t.SerperScrapeResponse\n  ): Record<string, string | number | boolean | null | undefined> {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Serper scraper instance\n * @param config Scraper configuration\n * @returns Serper scraper instance\n */\nexport const createSerperScraper = (\n  config: t.SerperScraperConfig = {}\n): SerperScraper => {\n  return new SerperScraper(config);\n};\n"],"names":["createDefaultLogger"],"mappings":";;;;;AAIA;;;;;;;;;;;;;;;;;;;;;;;;AAwBG;MACU,aAAa,CAAA;AAChB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,eAAe;AAEvB,IAAA,WAAA,CAAY,SAAgC,EAAE,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;AAE/D,QAAA,IAAI,CAAC,MAAM;AACT,YAAA,MAAM,CAAC,MAAM;gBACb,OAAO,CAAC,GAAG,CAAC,iBAAiB;AAC7B,gBAAA,2BAA2B;QAE7B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QACrC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,IAAI;QAErD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAIA,yBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC;QACxE;QAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,yCAAA,EAA4C,IAAI,CAAC,MAAM,CAAA,CAAE,CAC1D;IACH;AAEA;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAiC,EAAE,EAAA;AAEnC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,2BAA2B;AACnC,iBAAA;aACF;QACH;AAEA,QAAA,IAAI;AACF,YAAA,MAAM,OAAO,GAAG;gBACd,GAAG;AACH,gBAAA,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;aACjE;AAED,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACtD,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,IAAI,CAAC,MAAM;AACxB,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;AACD,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACzC,aAAA,CAAC;AAEF,YAAA,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtD;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAA,kCAAA,EAAqC,YAAY,CAAA,CAAE;AAC3D,iBAAA;aACF;QACH;IACF;AAEA;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAgC,EAAA;QAEhC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;QACxB;QAEA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YAClC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;QAC5C;QAEA,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;QACxC;AAEA,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;IACxB;AAEA;;;;AAIG;AACH,IAAA,eAAe,CACb,QAAgC,EAAA;AAEhC,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;QACX;AAEA,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;IAC/B;AACD;AAED;;;;AAIG;MACU,mBAAmB,GAAG,CACjC,MAAA,GAAgC,EAAE,KACjB;AACjB,IAAA,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC;AAClC;;;;;"}