/** * Internal core shared by both Scrapper verbs (article / links). * * @remarks * No `@module` tag — this is a sibling of `index.ts`, relative-imported, not its own entrypoint. * Houses the per-parameter disposition machinery (schema building from `fixed`/`defaults`), the * snake→kebab wire mapping, the request/response contexts, and the `fetch`+pipeline handler core. * Generic harness helpers (artifact/header resolution, pipeline runners) come from `../_shared`. */ import { Tool } from "../../../forge"; import { type ToolGateFn, type ToolHeaders, type ToolHeadersResolver, type SpooledArtifactCtor } from "../_shared/index"; import type { Schema } from '@nhtio/validation'; import type { NextFn } from '@nhtio/middleware'; /** Throw the battery-scoped config error. */ export declare const failConfig: (reason: string) => never; /** The wire type of a Scrapper query parameter — controls serialisation. */ export type ScrapperParamType = 'string' | 'number' | 'boolean'; /** * One curated, model-facing Scrapper parameter: its snake_case key (used in the model schema and * in `fixed`/`defaults`), its kebab-case wire name, its type, the base validator, and a description. */ export interface ScrapperParamSpec { /** snake_case key as seen by the model and in `config.fixed` / `config.defaults`. */ key: string; /** kebab-case name sent to the Scrapper API. */ wire: string; /** Wire type, controlling string/number/boolean serialisation. */ type: ScrapperParamType; /** Base `@nhtio/validation` schema (no `.required()`/`.default()`/`.optional()` applied yet). */ schema: Schema; /** Human-readable description surfaced to the model. */ description: string; } /** * Build the model-facing input schema from a verb's param specs and the factory's disposition. * `url` is always required. A `fixed` param is omitted (the model can't set it); a `defaults` param * gets `.default(value)`; everything else is `.optional()`. */ export declare const buildScrapperSchema: (specs: ScrapperParamSpec[], fixed: Record | undefined, defaults: Record | undefined, extra?: Record) => Schema; /** * Assemble the wire-kebab query params for one request: each spec's value is `fixed` (if pinned) * else the validated model/default value; then `fixedQuery` raw passthrough is layered on. `url` * is handled separately (it is the search target, never pinned). * * @remarks * An empty string (`''`) is deliberately treated the same as `undefined`/`null` and never * forwarded — this is what lets the schema's `.allow('')` on optional string specs actually * mean "don't set this," instead of sending e.g. `user-agent=` to the wire. This applies * generically to every spec (no per-spec-type branching), including a `fixed`-pinned value that * happens to be `''` — an explicitly pinned empty string still means "don't send it." */ export declare const buildWireParams: (args: Record, specs: ScrapperParamSpec[], fixed: Record | undefined, fixedQuery: Record | undefined) => Record; /** * Mutable context handed to each input-pipeline stage **before** the HTTP request is sent. * Identical for both verbs. */ export interface ScrapperRequestContext { /** The tool's name (read-only). */ readonly toolName: string; /** The target page URL (the `url` argument). Mutable. */ url: string; /** Wire-kebab query params (everything except `url`). Mutable. */ params: Record; /** Resolved request headers sent to the SCRAPPER INSTANCE (auth). Mutable. */ headers: ToolHeaders; /** The Scrapper instance base URL (read-only). */ readonly instanceUrl: string; /** Cross-stage scratch space; also carried onto the response context. */ readonly stash: Map; /** Skip the fetch and return `result` verbatim as the tool's output (e.g. a cache hit). */ shortCircuit(result: string): void; } /** * Mutable context handed to each output-pipeline stage **after** the response JSON is parsed. * * @typeParam R - The verb's normalised result type (article object or links payload). */ export interface ScrapperResponseContext { /** The tool's name (read-only). */ readonly toolName: string; /** The request context as it was sent (post-input-pipeline). */ readonly request: ScrapperRequestContext; /** The parsed Scrapper JSON body. Mutable (used when `format` is `raw`). */ raw: unknown; /** The normalised result. Mutable — reshape, redact, enrich. */ result: R; /** The effective payload shape for this call. */ format: 'normalized' | 'raw'; /** When set, used verbatim as the tool's output (overrides serialisation). */ output?: string; /** Cross-stage scratch space; carried over from the request context. */ readonly stash: Map; } /** An input-pipeline stage. Onion middleware over {@link ScrapperRequestContext}. */ export type ScrapperInputMiddlewareFn = (ctx: ScrapperRequestContext, next: NextFn) => void | Promise; /** An output-pipeline stage over a verb's {@link ScrapperResponseContext}. */ export type ScrapperOutputMiddlewareFn = (ctx: ScrapperResponseContext, next: NextFn) => void | Promise; /** Configuration common to every Scrapper factory. `A` is the accepted `artifact` resolver type. */ export interface ScrapperBaseConfig { /** Base URL of the Scrapper instance, e.g. `https://scrapper.example.org`. Required. */ instanceUrl: string; /** Headers sent to the Scrapper INSTANCE for auth (X-API-Key / Basic) — static or resolver. */ headers?: ToolHeaders | ToolHeadersResolver; /** The tool's own `fetch` AbortController timeout in ms. Default `65_000` (> Scrapper's 60s browser default). */ requestTimeoutMs?: number; /** Output shape. `normalized`/`raw` pin it; `either` (default) exposes a `format` arg to the model. */ resultFormat?: 'normalized' | 'raw' | 'either'; /** Spool-artifact resolver for the output. Default `() => SpooledJsonArtifact`. */ artifact?: A; /** Tool name override. */ name?: string; /** Tool description override. */ description?: string; /** Pinned params — sent always, removed from the model schema. */ fixed?: Partial

; /** Model-overridable default param values. */ defaults?: Partial

; /** Raw, un-modeled wire params (kebab keys) — always sent, never model-visible. Keeps the battery generic. */ fixedQuery?: Record; /** * Optional per-call gate run before the HTTP request — the seam for human-approval/RBAC * flows built on `ctx.waitFor` (the ADK gates primitive). Throwing aborts the call through * the standard tool-error path. Scraping reaches the network on the agent's behalf, which * makes every call a candidate for gating. */ gate?: ToolGateFn; /** Stages run before the HTTP request. See {@link ScrapperRequestContext}. */ inputPipeline?: ScrapperInputMiddlewareFn[]; /** Stages run after the response is parsed. See {@link ScrapperResponseContext}. */ outputPipeline?: ScrapperOutputMiddlewareFn[]; } /** Verb-specific wiring passed to {@link assembleScrapperTool}. */ export interface ScrapperVerb { /** Scrapper endpoint path, e.g. `/api/article`. */ endpoint: string; /** The curated param specs for this verb. */ specs: ScrapperParamSpec[]; /** Default tool name (`scrapper_article` / `scrapper_links`). */ defaultName: string; /** Default tool description. */ defaultDescription: string; /** Map a parsed Scrapper body to the verb's normalised result. */ normalize: (body: Record) => R; } /** * Build a configured Scrapper {@link Tool} from validated config + an already-resolved sync * artifact constructor. Shared by every verb and by both the async and sync factories. */ export declare const assembleScrapperTool: (verb: ScrapperVerb, config: ScrapperBaseConfig, instanceUrl: string, artifactConstructor: () => SpooledArtifactCtor) => Tool; /** Validate `instanceUrl` and return the trailing-slash-normalised base. */ export declare const validateScrapperInstanceUrl: (config: { instanceUrl?: string; }) => string;