/** * Parameter schema, output schema, and request resolution for `web_fetch`. */ import { StringEnum } from "@earendil-works/pi-ai"; import { Type, type Static } from "typebox"; import { MAX_OUTPUT_CHARS, type WebFetchConfig } from "./config.ts"; import type { FindMode } from "./find.ts"; import type { FetchRequest } from "./types.ts"; const MAX_URL_LENGTH = 2048; export const MIN_CHARS = 200; const FIND_MODES = ["insensitive", "exact", "fuzzy"] as const; const DEFAULT_CONTEXT_CHARS = 200; const DEFAULT_MAX_MATCHES = 8; const MAX_FIND_TERMS = 10; const HTTP_METHODS = ["GET", "POST"] as const; const MAX_HEADERS = 20; const MAX_HEADER_VALUE = 4096; /** Headers that control the transport; letting the model set them would break framing or SSRF checks. */ const BLOCKED_HEADERS = new Set([ "host", "content-length", "connection", "transfer-encoding", "upgrade", "te", "trailer", "proxy-connection", "proxy-authenticate", "proxy-authorization", ]); const HEADER_NAME = /^[!#$%&'*+\-.^_`|~0-9a-zA-Z]+$/; export const WebFetchParams = Type.Object({ url: Type.String({ minLength: 1, maxLength: MAX_URL_LENGTH, description: "Absolute http(s) URL to fetch.", }), method: Type.Optional( StringEnum(HTTP_METHODS, { description: "HTTP method; POST suits read-style APIs such as GraphQL.", }), ), headers: Type.Optional( Type.Record(Type.String(), Type.String(), { description: "Extra request headers (for example an API token). Transport headers are refused.", }), ), body: Type.Optional( Type.String({ maxLength: 1_000_000, description: "Request body for a POST; set Content-Type via `headers`.", }), ), startIndex: Type.Optional( Type.Integer({ minimum: 0, description: "Code-point offset to start from; use it to page a truncated result.", }), ), maxChars: Type.Optional( Type.Integer({ minimum: MIN_CHARS, maximum: MAX_OUTPUT_CHARS, description: "Maximum characters to return (default and ceiling: the configured maxOutputChars).", }), ), find: Type.Optional( Type.Array(Type.String(), { minItems: 1, maxItems: MAX_FIND_TERMS, description: "Return passages matching these strings instead of the page text; offsets are usable as startIndex.", }), ), mode: Type.Optional(StringEnum(FIND_MODES, { description: "Match mode for `find`. Defaults to insensitive." })), contextChars: Type.Optional( Type.Integer({ minimum: 0, maximum: 2000, description: "Characters of context around each find match (default 200).", }), ), maxMatches: Type.Optional( Type.Integer({ minimum: 1, maximum: 50, description: "Maximum find matches to return (default 8)." }), ), refresh: Type.Optional(Type.Boolean({ description: "Bypass the page cache and refetch." })), }); export type WebFetchArgs = Static; export const WebFetchOutput = Type.Object({ url: Type.String(), finalUrl: Type.String(), title: Type.String(), status: Type.Number(), contentType: Type.String(), text: Type.String(), totalChars: Type.Number(), startIndex: Type.Number(), nextIndex: Type.Number(), truncated: Type.Boolean(), cached: Type.Boolean(), matches: Type.Array(Type.Object({ query: Type.String(), offset: Type.Number(), passage: Type.String() })), matchesTruncated: Type.Boolean(), fetchedAt: Type.String(), error: Type.String(), }); /** The output shape, derived from the runtime schema so the two cannot drift. */ export type FetchResponse = Static; function normalizeFind(raw: unknown): string[] { if (!Array.isArray(raw)) return []; return raw .filter((value): value is string => typeof value === "string") .map((value) => value.trim()) .filter(Boolean) .slice(0, MAX_FIND_TERMS); } /** Normalize custom headers: reject transport/control headers and CRLF, cap count and size. */ function normalizeHeaders(raw: unknown): Record { if (!raw || typeof raw !== "object" || Array.isArray(raw)) return {}; const entries = Object.entries(raw as Record); const headers: Record = {}; let count = 0; for (const [rawName, rawValue] of entries) { if (typeof rawValue !== "string") continue; const name = rawName.trim(); if (!name) continue; if (!HEADER_NAME.test(name)) throw new Error(`header name "${name}" is invalid.`); if (BLOCKED_HEADERS.has(name.toLowerCase())) throw new Error(`header "${name}" is not allowed.`); if (/[\r\n]/.test(rawValue)) throw new Error(`header "${name}" contains an invalid character.`); const value = rawValue.trim(); if (!value) continue; if (value.length > MAX_HEADER_VALUE) throw new Error(`header "${name}" is too long.`); if (count >= MAX_HEADERS) throw new Error(`at most ${MAX_HEADERS} headers are allowed.`); headers[name] = value; count++; } return headers; } /** Validate the arguments and clamp the optional fields. */ export function resolveRequest(args: WebFetchArgs, config: WebFetchConfig): FetchRequest { const url = typeof args.url === "string" ? args.url.trim() : ""; if (!url) throw new Error("url is required."); if (url.length > MAX_URL_LENGTH) throw new Error(`url must be at most ${MAX_URL_LENGTH} characters.`); const explicitMethod = args.method === "GET" || args.method === "POST" ? args.method : undefined; if (args.body !== undefined && explicitMethod === "GET") throw new Error('body requires method "POST".'); const method: "GET" | "POST" = explicitMethod ?? (args.body !== undefined ? "POST" : "GET"); if (typeof args.body === "string" && args.body.length > config.maxBodyChars) { throw new Error(`body is longer than ${config.maxBodyChars} characters.`); } const headers = normalizeHeaders(args.headers); const body = typeof args.body === "string" ? args.body : undefined; const startIndex = typeof args.startIndex === "number" ? Math.max(0, Math.round(args.startIndex)) : 0; const requested = typeof args.maxChars === "number" ? args.maxChars : config.maxOutputChars; const maxChars = Math.min(config.maxOutputChars, Math.max(MIN_CHARS, Math.round(requested))); const rawMode = typeof args.mode === "string" ? args.mode : ""; const mode: FindMode = (FIND_MODES as readonly string[]).includes(rawMode) ? (rawMode as FindMode) : "insensitive"; const contextChars = typeof args.contextChars === "number" ? Math.min(2000, Math.max(0, Math.round(args.contextChars))) : DEFAULT_CONTEXT_CHARS; const maxMatches = typeof args.maxMatches === "number" ? Math.min(50, Math.max(1, Math.round(args.maxMatches))) : DEFAULT_MAX_MATCHES; return { url, method, headers, body, startIndex, maxChars, find: normalizeFind(args.find), mode, contextChars, maxMatches, refresh: args.refresh === true, cacheable: method === "GET" && Object.keys(headers).length === 0 && body === undefined, }; }