{"version":3,"sources":["../../../src/tools/web/webCrawler.ts"],"names":["WebCrawlerToolOutput","JSONToolOutput","getTextContent","result","url","statusCode","statusText","contentType","content","join","defaultParser","response","text","headers","get","includes","stripHtml","WebCrawlerTool","Tool","name","description","inputSchema","z","object","string","describe","client","parser","emitter","Emitter","root","child","namespace","creator","options","fetch","_run","_options","run","redirect","request","signal","status","createSnapshot","loadSnapshot","snapshot","Object","assign"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA2BO,MAAMA,6BAA6BC,uBAAAA,CAAAA;AAAAA,EAAAA;;;EAExCC,cAAAA,GAAyB;AACvB,IAAA,OAAO;MACL,CAAA,KAAA,EAAQ,IAAA,CAAKC,OAAOC,GAAG,CAAA,CAAA;AACvB,MAAA,CAAA,QAAA,EAAW,KAAKD,MAAAA,CAAOE,UAAU,CAAA,EAAA,EAAK,IAAA,CAAKF,OAAOG,UAAU,CAAA,CAAA,CAAA;MAC5D,CAAA,cAAA,EAAiB,IAAA,CAAKH,OAAOI,WAAW,CAAA,CAAA;MACxC,CAAA,SAAA,EAAY,IAAA,CAAKJ,OAAOK,OAAO,CAAA;AAC/BC,KAAAA,CAAAA,IAAAA,CAAK,IAAA,CAAA;AACT,EAAA;AACF;;;;;;;AAiBA,eAAeC,cAAcC,QAAAA,EAA4B;AACvD,EAAA,MAAMC,IAAAA,GAAO,MAAMD,QAAAA,CAASC,IAAAA,EAAI;AAChC,EAAA,IAAIA,IAAAA,EAAM;AACR,IAAA,MAAML,WAAAA,GAAcI,QAAAA,CAASE,OAAAA,CAAQC,GAAAA,CAAI,cAAA,CAAA,IAAmB,EAAA;AAC5D,IAAA,IAAIP,WAAAA,CAAYQ,QAAAA,CAAS,WAAA,CAAA,EAAc;AACrC,MAAA,OAAOC,yBAAAA,CAAUJ,IAAAA,CAAAA,CAAMT,MAAAA;AACzB,IAAA;AACF,EAAA;AACA,EAAA,OAAOS,IAAAA,IAAQ,YAAA;AACjB;AATeF,MAAAA,CAAAA,aAAAA,EAAAA,eAAAA,CAAAA;AAWR,MAAMO,uBAAuBC,aAAAA,CAAAA;AAAAA,EAAAA;;;EAClCC,IAAAA,GAAO,YAAA;EACPC,WAAAA,GAAc,CAAA,0CAAA,CAAA;EACdC,WAAAA,GAAc;AACZ,IAAA,OAAOC,MAAEC,MAAAA,CAAO;AACdnB,MAAAA,GAAAA,EAAKkB,MAAEE,MAAAA,EAAM,CAAGpB,GAAAA,EAAG,CAAGqB,SAAS,aAAA;KACjC,CAAA;AACF,EAAA;AAEUC,EAAAA,MAAAA;AACAC,EAAAA,MAAAA;EAEMC,OAAAA,GAA8DC,mBAAAA,CAAQC,KAAKC,KAAAA,CAAM;IAC/FC,SAAAA,EAAW;AAAC,MAAA,MAAA;AAAQ,MAAA;;IACpBC,OAAAA,EAAS;GACX,CAAA;AAEA,EAAA,WAAA,CAAY,EAAEP,MAAAA,EAAQC,MAAAA,EAAQ,GAAGO,OAAAA,EAAAA,GAAuC,EAAC,EAAG;AAC1E,IAAA,KAAA,CAAMA,OAAAA,CAAAA;AACN,IAAA,IAAA,CAAKR,SAASA,MAAAA,IAAUS,KAAAA;AACxB,IAAA,IAAA,CAAKR,SAASA,MAAAA,IAAUjB,aAAAA;AAC1B,EAAA;AAEA,EAAA,MAAgB0B,IAAAA,CACd,EAAEhC,GAAAA,EAAG,EACLiC,UACAC,GAAAA,EACA;AACA,IAAA,MAAM3B,QAAAA,GAAW,MAAM,IAAA,CAAKe,MAAAA,CAAOtB,GAAAA,EAAK;MACtCmC,QAAAA,EAAU,QAAA;AACV,MAAA,GAAG,KAAKL,OAAAA,CAAQM,OAAAA;AAChBC,MAAAA,MAAAA,EAAQH,GAAAA,CAAIG;KACd,CAAA;AAEA,IAAA,MAAMjC,OAAAA,GAAU,MAAM,IAAA,CAAKmB,MAAAA,CAAOhB,QAAAA,CAAAA;AAClC,IAAA,OAAO,IAAIX,oBAAAA,CAAqB;AAC9BI,MAAAA,GAAAA;AACAC,MAAAA,UAAAA,EAAYM,QAAAA,CAAS+B,MAAAA;AACrBpC,MAAAA,UAAAA,EAAYK,QAAAA,CAASL,UAAAA;AACrBC,MAAAA,WAAAA,EAAaI,QAAAA,CAASE,OAAAA,CAAQC,GAAAA,CAAI,cAAA,CAAA,IAAmB,SAAA;AACrDN,MAAAA;KACF,CAAA;AACF,EAAA;EAEAmC,cAAAA,GAAiB;AACf,IAAA,OAAO;AACL,MAAA,GAAG,MAAMA,cAAAA,EAAAA;AACTjB,MAAAA,MAAAA,EAAQ,IAAA,CAAKA,MAAAA;AACbC,MAAAA,MAAAA,EAAQ,IAAA,CAAKA;AACf,KAAA;AACF,EAAA;AAEAiB,EAAAA,YAAAA,CAAa,EAAElB,MAAAA,EAAQC,MAAAA,EAAQ,GAAGkB,UAAAA,EAAoD;AACpF,IAAA,KAAA,CAAMD,aAAaC,QAAAA,CAAAA;AACnBC,IAAAA,MAAAA,CAAOC,OAAO,IAAA,EAAM;AAClBrB,MAAAA,MAAAA,EAAQA,MAAAA,IAAUS,KAAAA;AAClBR,MAAAA,MAAAA,EAAQA,MAAAA,IAAUjB;KACpB,CAAA;AACF,EAAA;AACF","file":"webCrawler.cjs","sourcesContent":["/**\n * Copyright 2025 © BeeAI a Series of LF Projects, LLC\n * SPDX-License-Identifier: Apache-2.0\n */\n\nimport {\n  BaseToolOptions,\n  BaseToolRunOptions,\n  ToolEmitter,\n  JSONToolOutput,\n  Tool,\n  ToolInput,\n} from \"@/tools/base.js\";\nimport { z } from \"zod\";\nimport { Cache } from \"@/cache/decoratorCache.js\";\nimport { stripHtml } from \"string-strip-html\";\nimport { RunContext } from \"@/context.js\";\nimport { Emitter } from \"@/emitter/emitter.js\";\n\ninterface CrawlerOutput {\n  url: string;\n  statusCode: number;\n  statusText: string;\n  contentType: string;\n  content: string;\n}\n\nexport class WebCrawlerToolOutput extends JSONToolOutput<CrawlerOutput> {\n  @Cache()\n  getTextContent(): string {\n    return [\n      `URL: ${this.result.url}`,\n      `STATUS: ${this.result.statusCode} (${this.result.statusText})`,\n      `CONTENT-TYPE: ${this.result.contentType}`,\n      `CONTENT: ${this.result.content}`,\n    ].join(\"\\n\");\n  }\n}\n\nexport type HttpClient = (url: string, options?: RequestInit) => Promise<HttpClientResponse>;\ninterface HttpClientResponse {\n  status: number;\n  statusText: string;\n  headers: Headers;\n  text(): Promise<string>;\n}\n\ntype Parser = (response: HttpClientResponse) => Promise<string>;\ninterface WebsiteCrawlerToolOptions extends BaseToolOptions {\n  client?: HttpClient;\n  parser?: Parser;\n  request?: RequestInit;\n}\n\nasync function defaultParser(response: HttpClientResponse) {\n  const text = await response.text();\n  if (text) {\n    const contentType = response.headers.get(\"content-type\") ?? \"\";\n    if (contentType.includes(\"text/html\")) {\n      return stripHtml(text).result;\n    }\n  }\n  return text || \"No Content\";\n}\n\nexport class WebCrawlerTool extends Tool<WebCrawlerToolOutput, WebsiteCrawlerToolOptions> {\n  name = \"WebCrawler\";\n  description = `Retrieves content of an arbitrary website.`;\n  inputSchema() {\n    return z.object({\n      url: z.string().url().describe(\"Website URL\"),\n    });\n  }\n\n  protected client: HttpClient;\n  protected parser: Parser;\n\n  public readonly emitter: ToolEmitter<ToolInput<this>, WebCrawlerToolOutput> = Emitter.root.child({\n    namespace: [\"tool\", \"webCrawler\"],\n    creator: this,\n  });\n\n  constructor({ client, parser, ...options }: WebsiteCrawlerToolOptions = {}) {\n    super(options);\n    this.client = client ?? fetch;\n    this.parser = parser ?? defaultParser;\n  }\n\n  protected async _run(\n    { url }: ToolInput<this>,\n    _options: Partial<BaseToolRunOptions>,\n    run: RunContext<this>,\n  ) {\n    const response = await this.client(url, {\n      redirect: \"follow\",\n      ...this.options.request,\n      signal: run.signal,\n    });\n\n    const content = await this.parser(response);\n    return new WebCrawlerToolOutput({\n      url,\n      statusCode: response.status,\n      statusText: response.statusText,\n      contentType: response.headers.get(\"content-type\") ?? \"unknown\",\n      content,\n    });\n  }\n\n  createSnapshot() {\n    return {\n      ...super.createSnapshot(),\n      client: this.client,\n      parser: this.parser,\n    };\n  }\n\n  loadSnapshot({ client, parser, ...snapshot }: ReturnType<typeof this.createSnapshot>) {\n    super.loadSnapshot(snapshot);\n    Object.assign(this, {\n      client: client ?? fetch,\n      parser: parser ?? defaultParser,\n    });\n  }\n}\n"]}