{"version":3,"file":"web_retrieval.cjs","names":[],"sources":["../../../src/batteries/tools/web_retrieval/index.ts"],"sourcesContent":["/**\n * RAG glue: turn web-search and web-scrape results into `Retrievable` records for a turn.\n *\n * @module @nhtio/adk/batteries/tools/web_retrieval\n *\n * @remarks\n * The seam from \"I searched / scraped something\" to \"it is in the turn as a `Retrievable`\",\n * shared by the SearXNG and Scrapper batteries. It is deliberately **decoupled** from the ADK\n * core at runtime:\n *\n * - The converters are **pure** `(payload) => RawRetrievable[]` — they build plain data objects and\n *   never instantiate a core class, so the module's only core coupling is erased `import type`.\n * - The recommended spool-artifact type travels as an **open resolver**\n *   ({@link @nhtio/adk/forge!ArtifactConstructorResolver}), never a closed string enum — a consumer's\n *   future YAML/HTML `SpooledArtifact` subclass works with no change here. The converter hands the\n *   recommendation to the caller's `spool` hook; the caller owns the actual class import.\n * - The one helper that must construct a `Retrievable` ({@link storeRetrievables}) takes the\n *   constructor via a **resolver** (constructor / sync / async / dynamic-import), exactly like the\n *   vector battery's `createVectorStore` `client`.\n *\n * Web content is `'third-party-public'` by default — a definitional constant for open-web data\n * (NOT inferred from the URL, which CONTRIBUTING Design Decision #12 forbids); override via\n * `trustTier` when you know better.\n */\n\nimport { sha256 } from 'js-sha256'\nimport { isInstanceOf } from '@nhtio/adk/guards'\nimport type { SpooledArtifact } from '@nhtio/adk/spooled_artifact'\nimport type { ArtifactConstructorResolver } from '@nhtio/adk/forge'\nimport type { RawRetrievable, Retrievable, RetrievableTrustTier } from '@nhtio/adk/common'\n\n/** A constructor that builds a {@link @nhtio/adk!Retrievable} from a {@link @nhtio/adk/common!RawRetrievable}. */\nexport type RetrievableCtor = new (raw: RawRetrievable) => Retrievable\n\n/** A resolver of `T`: the value itself, or a (sync/async) thunk, optionally a module `{ default }`. */\nexport type Resolver<T> = T | (() => T | { default: T }) | (() => Promise<T | { default: T }>)\n\n/**\n * A reader-backed-artifact hook. Called by a converter for content that may be large; the\n * converter passes the artifact constructor it **recommends** for this content (an open\n * {@link @nhtio/adk/forge!ArtifactConstructorResolver}) so the caller can wrap with the right\n * subclass — preserving its forged query tools — using the caller's own core import. Return a\n * {@link @nhtio/adk!SpooledArtifact} to store the content reader-backed, or `undefined` to keep it\n * inline as a string.\n */\nexport type SpoolHook = (\n  id: string,\n  text: string,\n  recommended: ArtifactConstructorResolver\n) => SpooledArtifact | undefined\n\n/** Options common to every converter. */\nexport interface ToRetrievableOptions {\n  /**\n   * Trust tier for the produced records. Default `'third-party-public'` (web content is\n   * third-party by definition — this is a constant, not URL inference).\n   */\n  trustTier?: RetrievableTrustTier\n  /** Semantic `kind` label, e.g. `'web-search-result'`, `'web-article'`, `'web-links'`. */\n  kind?: string\n  /** Prefix for the stable, hashed record id (namespacing across sources). */\n  idPrefix?: string\n  /** Optional reader-backed-artifact hook for large content. See {@link SpoolHook}. */\n  spool?: SpoolHook\n  /** Whether spooled content should be materialized inline instead of rendered as a handle. */\n  inline?: boolean\n}\n\n/**\n * The artifact-resolver recommendations a caller may supply so the glue names no concrete class\n * itself. Each converter asks for the relevant key; if the caller omits it, content stays inline.\n */\nexport interface ArtifactRecommendations {\n  /** Recommended for plain-text / HTML content (base `SpooledArtifact`). */\n  text?: ArtifactConstructorResolver\n  /** Recommended for markdown content (`SpooledMarkdownArtifact`). */\n  markdown?: ArtifactConstructorResolver\n  /** Recommended for JSON content (`SpooledJsonArtifact`). */\n  json?: ArtifactConstructorResolver\n}\n\nconst nowIso = (): string => new Date().toISOString()\n\n/** A stable, unguessable id derived from a source string (URL) plus an optional prefix. */\nconst stableId = (prefix: string | undefined, source: string): string => {\n  const h = sha256(source)\n  return prefix ? `${prefix}:${h}` : h\n}\n\n/** Clamp a possibly-unbounded score into `[0, 1]`; drop non-finite. */\nconst clampScore = (score: unknown): number | undefined => {\n  if (typeof score !== 'number' || !Number.isFinite(score)) return undefined\n  if (score < 0) return 0\n  if (score > 1) return 1\n  return score\n}\n\n/**\n * Resolve content to either an inline string or a caller-provided {@link SpooledArtifact}. When a\n * `spool` hook is supplied it is offered the recommended resolver; whatever it returns (artifact or\n * `undefined`→inline) is used.\n */\nconst resolveContent = (\n  id: string,\n  text: string,\n  opts: ToRetrievableOptions,\n  recommended: ArtifactConstructorResolver\n): string | SpooledArtifact => {\n  if (opts.spool) {\n    const artifact = opts.spool(id, text, recommended)\n    if (artifact) return artifact\n  }\n  return text\n}\n\n// ── SearXNG ──────────────────────────────────────────────────────────────────\n\n/** Minimal structural shape of a SearXNG normalised result the converter reads. */\nexport interface SearxngResultLike {\n  /** Result URL (becomes the record's `source`). */\n  url?: string\n  /** Result title (joined into the inline content). */\n  title?: string\n  /** Result snippet (joined into the inline content). */\n  content?: string\n  /** Relevance score (clamped to `[0,1]` on the record). */\n  score?: number\n}\n/** Minimal structural shape of a SearXNG normalised payload. */\nexport interface SearxngPayloadLike {\n  /** The result list. */\n  results?: SearxngResultLike[]\n}\n\n/**\n * Convert a SearXNG normalised payload into one {@link @nhtio/adk/common!RawRetrievable} per result.\n *\n * @remarks\n * Snippets are short, so `content` stays an inline string (the `spool` hook, if any, is still\n * offered the `text` recommendation). `source` is the result URL; `score` is clamped to `[0,1]`.\n *\n * @param payload - The SearXNG normalised payload (`{ results: [{ url, title, content, score }] }`).\n * @param opts - Trust tier, kind, id prefix, optional spool hook.\n * @param recommend - Optional artifact-resolver recommendations (the glue names no class itself).\n * @returns One `RawRetrievable` per result.\n */\nexport const searxngResultsToRetrievables = (\n  payload: SearxngPayloadLike,\n  opts: ToRetrievableOptions = {},\n  recommend: ArtifactRecommendations = {}\n): RawRetrievable[] => {\n  const trustTier: RetrievableTrustTier = opts.trustTier ?? 'third-party-public'\n  const kind = opts.kind ?? 'web-search-result'\n  const created = nowIso()\n  const results = payload.results ?? []\n  return results.map((r, i) => {\n    const source = r.url ?? ''\n    const id = stableId(opts.idPrefix, source || `${kind}:${i}`)\n    const text = [r.title, r.content].filter((s): s is string => typeof s === 'string').join('\\n')\n    const recommended = recommend.text ?? recommend.markdown ?? recommend.json\n    const content = recommended ? resolveContent(id, text, opts, recommended) : text\n    const raw: RawRetrievable = {\n      id,\n      content,\n      trustTier,\n      kind,\n      createdAt: created,\n      updatedAt: created,\n      inline: opts.inline,\n    }\n    if (source) raw.source = source\n    const score = clampScore(r.score)\n    if (score !== undefined) raw.score = score\n    return raw\n  })\n}\n\n// ── Scrapper: article ──────────────────────────────────────────────────────────\n\n/** Minimal structural shape of a Scrapper normalised article. */\nexport interface ScrapperArticleLike {\n  /** The page URL (becomes the record's `source`). */\n  url?: string\n  /** Article title. */\n  title?: string\n  /** Article text with HTML stripped (the default content source). */\n  textContent?: string\n  /** Processed article HTML (the `'content'` content source). */\n  content?: string\n}\n\n/** Which article text field to use as the record content. */\nexport type ArticleContentSource = 'textContent' | 'content'\n\n/** Options for {@link scrapperArticleToRetrievable}. */\nexport interface ArticleToRetrievableOptions extends ToRetrievableOptions {\n  /** Which field to use as content (default `'textContent'`). `'content'` is HTML. */\n  contentSource?: ArticleContentSource\n  /**\n   * Whether the chosen content is markdown (recommend `markdown`) rather than plain text.\n   * Default false. Use when an output pipeline rendered the article to markdown.\n   */\n  asMarkdown?: boolean\n}\n\n/**\n * Convert a Scrapper normalised article into a single {@link @nhtio/adk/common!RawRetrievable}.\n *\n * @remarks\n * Long article text is exactly what a reader-backed {@link @nhtio/adk!SpooledArtifact} is for: pass a\n * `spool` hook and the converter offers it the recommended artifact resolver (markdown when\n * `asMarkdown`, else text/HTML) so the model gets the right forged query tools. Without a hook,\n * content stays inline.\n *\n * @param article - The Scrapper normalised article.\n * @param opts - Trust tier, kind, id prefix, content source, markdown flag, optional spool hook.\n * @param recommend - Optional artifact-resolver recommendations.\n * @returns A single `RawRetrievable`.\n */\nexport const scrapperArticleToRetrievable = (\n  article: ScrapperArticleLike,\n  opts: ArticleToRetrievableOptions = {},\n  recommend: ArtifactRecommendations = {}\n): RawRetrievable => {\n  const trustTier: RetrievableTrustTier = opts.trustTier ?? 'third-party-public'\n  const kind = opts.kind ?? 'web-article'\n  const created = nowIso()\n  const source = article.url ?? ''\n  const id = stableId(opts.idPrefix, source || kind)\n  const field = opts.contentSource ?? 'textContent'\n  const text = (field === 'content' ? article.content : article.textContent) ?? ''\n  const recommended = opts.asMarkdown ? (recommend.markdown ?? recommend.text) : recommend.text\n  const content = recommended ? resolveContent(id, text, opts, recommended) : text\n  const raw: RawRetrievable = {\n    id,\n    content,\n    inline: opts.inline,\n    artifactConstructor: recommended,\n    trustTier,\n    kind,\n    createdAt: created,\n    updatedAt: created,\n  }\n  if (source) raw.source = source\n  return raw\n}\n\n// ── Scrapper: links ──────────────────────────────────────────────────────────\n\n/** Minimal structural shape of a Scrapper normalised link. */\nexport interface ScrapperLinkLike {\n  /** The link's target URL (becomes the record's `source`). */\n  url?: string\n  /** The link's anchor text (becomes the record's content). */\n  text?: string\n}\n/** Minimal structural shape of a Scrapper normalised links payload. */\nexport interface ScrapperLinksLike {\n  /** The page URL the links were collected from. */\n  url?: string\n  /** The collected links. */\n  links?: ScrapperLinkLike[]\n}\n\n/**\n * Convert a Scrapper normalised links payload into one {@link @nhtio/adk/common!RawRetrievable} per link.\n *\n * @remarks\n * Each link's `text` becomes the content and its `url` the `source`. Spooling is opt-in, exactly\n * as for the other web converters; link records default to `inline: true` so short text remains\n * inline even when a storage layer auto-spools it.\n *\n * @param payload - The Scrapper normalised links payload (`{ links: [{ url, text }] }`).\n * @param opts - Trust tier, kind, id prefix, optional spool hook, and inline preference.\n * @param recommend - Optional artifact-resolver recommendations.\n * @returns One `RawRetrievable` per link.\n */\nexport const scrapperLinksToRetrievables = (\n  payload: ScrapperLinksLike,\n  opts: ToRetrievableOptions = {},\n  recommend: ArtifactRecommendations = {}\n): RawRetrievable[] => {\n  const trustTier: RetrievableTrustTier = opts.trustTier ?? 'third-party-public'\n  const kind = opts.kind ?? 'web-link'\n  const created = nowIso()\n  const links = payload.links ?? []\n  return links.map((l, i) => {\n    const source = l.url ?? ''\n    const id = stableId(opts.idPrefix, source || `${kind}:${i}`)\n    const text = l.text ?? source\n    const recommended = recommend.text ?? recommend.markdown ?? recommend.json\n    const content = recommended ? resolveContent(id, text, opts, recommended) : text\n    const raw: RawRetrievable = {\n      id,\n      content,\n      inline: opts.inline ?? true,\n      trustTier,\n      kind,\n      createdAt: created,\n      updatedAt: created,\n    }\n    if (source) raw.source = source\n    return raw\n  })\n}\n\n// ── Store helper (the single core-touching function) ─────────────────────────\n\n/** The minimal context surface {@link storeRetrievables} needs. */\nexport interface RetrievableStoreCtx {\n  /** Persist a single `Retrievable` and return the tracked (possibly auto-spooled) instance. */\n  storeRetrievable: (v: Retrievable) => Retrievable | Promise<Retrievable>\n}\n\n/**\n * Resolve a {@link Resolver} of the `Retrievable` constructor (sync / async / `{ default }`).\n *\n * @remarks\n * Both a bare class and a resolver are `typeof 'function'`, and we hold `Retrievable` only as an\n * `import type` (no runtime value to duck-type against). We disambiguate by behaviour: invoking a\n * real ES class without `new` throws, so a bare constructor is caught and returned as-is; a resolver\n * invokes cleanly and yields the constructor (possibly via a Promise and/or a `{ default }`).\n */\nconst resolveRetrievableCtor = async (\n  resolver: Resolver<RetrievableCtor>\n): Promise<RetrievableCtor> => {\n  if (typeof resolver !== 'function') {\n    throw new TypeError('retrievable must be a constructor or a resolver returning one')\n  }\n  let resolved: unknown\n  try {\n    resolved = (resolver as () => unknown)()\n  } catch {\n    return resolver as RetrievableCtor // bare class: threw on no-`new` invocation\n  }\n  if (isInstanceOf(resolved, 'Promise', Promise)) resolved = await resolved\n  if (resolved && typeof resolved === 'object' && 'default' in resolved) {\n    resolved = (resolved as { default?: unknown }).default\n  }\n  if (typeof resolved === 'function') return resolved as RetrievableCtor\n  return resolver as RetrievableCtor // resolver returned a non-function: it was itself the ctor\n}\n\n/**\n * Construct {@link @nhtio/adk!Retrievable}s from `RawRetrievable`s and store each via `ctx`.\n *\n * @remarks\n * This is the only function here that touches a core class, and it does so through an injected\n * **resolver** (`deps.retrievable`) so the glue itself never value-imports `Retrievable`. Each\n * record's `RawRetrievable` validation (including the required `trustTier`) fires at construction.\n * For reader-backed content, the caller's `spool` hook will typically have used\n * `ctx.storeRetrievableBytes` already; this helper just persists the records into the turn.\n *\n * @param ctx - Anything with a `storeRetrievable` method (a `DispatchContext`, or a stub).\n * @param raws - The plain records from the converters.\n * @param deps - `{ retrievable }`: the `Retrievable` constructor or a resolver of it.\n * @returns The constructed `Retrievable` instances, in input order.\n */\nexport const storeRetrievables = async (\n  ctx: RetrievableStoreCtx,\n  raws: RawRetrievable[],\n  deps: { retrievable: Resolver<RetrievableCtor> }\n): Promise<Retrievable[]> => {\n  const Ctor = await resolveRetrievableCtor(deps.retrievable)\n  const out: Retrievable[] = []\n  for (const raw of raws) {\n    const record = new Ctor(raw)\n    const stored = await ctx.storeRetrievable(record)\n    out.push(stored)\n  }\n  return out\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiFA,IAAM,gCAAuB,IAAI,KAAK,GAAE,YAAY;;AAGpD,IAAM,YAAY,QAA4B,WAA2B;CACvE,MAAM,KAAA,GAAA,UAAA,QAAW,MAAM;CACvB,OAAO,SAAS,GAAG,OAAO,GAAG,MAAM;AACrC;;AAGA,IAAM,cAAc,UAAuC;CACzD,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG,OAAO,KAAA;CACjE,IAAI,QAAQ,GAAG,OAAO;CACtB,IAAI,QAAQ,GAAG,OAAO;CACtB,OAAO;AACT;;;;;;AAOA,IAAM,kBACJ,IACA,MACA,MACA,gBAC6B;CAC7B,IAAI,KAAK,OAAO;EACd,MAAM,WAAW,KAAK,MAAM,IAAI,MAAM,WAAW;EACjD,IAAI,UAAU,OAAO;CACvB;CACA,OAAO;AACT;;;;;;;;;;;;;AAiCA,IAAa,gCACX,SACA,OAA6B,CAAC,GAC9B,YAAqC,CAAC,MACjB;CACrB,MAAM,YAAkC,KAAK,aAAa;CAC1D,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,UAAU,OAAO;CAEvB,QADgB,QAAQ,WAAW,CAAC,GACrB,KAAK,GAAG,MAAM;EAC3B,MAAM,SAAS,EAAE,OAAO;EACxB,MAAM,KAAK,SAAS,KAAK,UAAU,UAAU,GAAG,KAAK,GAAG,GAAG;EAC3D,MAAM,OAAO,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,MAAmB,OAAO,MAAM,QAAQ,EAAE,KAAK,IAAI;EAC7F,MAAM,cAAc,UAAU,QAAQ,UAAU,YAAY,UAAU;EAEtE,MAAM,MAAsB;GAC1B;GACA,SAHc,cAAc,eAAe,IAAI,MAAM,MAAM,WAAW,IAAI;GAI1E;GACA;GACA,WAAW;GACX,WAAW;GACX,QAAQ,KAAK;EACf;EACA,IAAI,QAAQ,IAAI,SAAS;EACzB,MAAM,QAAQ,WAAW,EAAE,KAAK;EAChC,IAAI,UAAU,KAAA,GAAW,IAAI,QAAQ;EACrC,OAAO;CACT,CAAC;AACH;;;;;;;;;;;;;;;AA4CA,IAAa,gCACX,SACA,OAAoC,CAAC,GACrC,YAAqC,CAAC,MACnB;CACnB,MAAM,YAAkC,KAAK,aAAa;CAC1D,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,UAAU,OAAO;CACvB,MAAM,SAAS,QAAQ,OAAO;CAC9B,MAAM,KAAK,SAAS,KAAK,UAAU,UAAU,IAAI;CAEjD,MAAM,SADQ,KAAK,iBAAiB,mBACZ,YAAY,QAAQ,UAAU,QAAQ,gBAAgB;CAC9E,MAAM,cAAc,KAAK,aAAc,UAAU,YAAY,UAAU,OAAQ,UAAU;CAEzF,MAAM,MAAsB;EAC1B;EACA,SAHc,cAAc,eAAe,IAAI,MAAM,MAAM,WAAW,IAAI;EAI1E,QAAQ,KAAK;EACb,qBAAqB;EACrB;EACA;EACA,WAAW;EACX,WAAW;CACb;CACA,IAAI,QAAQ,IAAI,SAAS;CACzB,OAAO;AACT;;;;;;;;;;;;;;AAgCA,IAAa,+BACX,SACA,OAA6B,CAAC,GAC9B,YAAqC,CAAC,MACjB;CACrB,MAAM,YAAkC,KAAK,aAAa;CAC1D,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,UAAU,OAAO;CAEvB,QADc,QAAQ,SAAS,CAAC,GACnB,KAAK,GAAG,MAAM;EACzB,MAAM,SAAS,EAAE,OAAO;EACxB,MAAM,KAAK,SAAS,KAAK,UAAU,UAAU,GAAG,KAAK,GAAG,GAAG;EAC3D,MAAM,OAAO,EAAE,QAAQ;EACvB,MAAM,cAAc,UAAU,QAAQ,UAAU,YAAY,UAAU;EAEtE,MAAM,MAAsB;GAC1B;GACA,SAHc,cAAc,eAAe,IAAI,MAAM,MAAM,WAAW,IAAI;GAI1E,QAAQ,KAAK,UAAU;GACvB;GACA;GACA,WAAW;GACX,WAAW;EACb;EACA,IAAI,QAAQ,IAAI,SAAS;EACzB,OAAO;CACT,CAAC;AACH;;;;;;;;;;AAmBA,IAAM,yBAAyB,OAC7B,aAC6B;CAC7B,IAAI,OAAO,aAAa,YACtB,MAAM,IAAI,UAAU,+DAA+D;CAErF,IAAI;CACJ,IAAI;EACF,WAAY,SAA2B;CACzC,QAAQ;EACN,OAAO;CACT;CACA,IAAI,eAAA,aAAa,UAAU,WAAW,OAAO,GAAG,WAAW,MAAM;CACjE,IAAI,YAAY,OAAO,aAAa,YAAY,aAAa,UAC3D,WAAY,SAAmC;CAEjD,IAAI,OAAO,aAAa,YAAY,OAAO;CAC3C,OAAO;AACT;;;;;;;;;;;;;;;;AAiBA,IAAa,oBAAoB,OAC/B,KACA,MACA,SAC2B;CAC3B,MAAM,OAAO,MAAM,uBAAuB,KAAK,WAAW;CAC1D,MAAM,MAAqB,CAAC;CAC5B,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,SAAS,IAAI,KAAK,GAAG;EAC3B,MAAM,SAAS,MAAM,IAAI,iBAAiB,MAAM;EAChD,IAAI,KAAK,MAAM;CACjB;CACA,OAAO;AACT"}