import type { BrowserFacade } from "./runtime/browser.js"; import type { HumanizeAPI } from "./runtime/human.js"; /** * Context injected by the runtime into every scraper. * Provides transport (fetch/browser), output (pushData), * state (checkpoint), work distribution (tasks), and utilities. */ export interface ScraperContext { /** HTTP client — curl-equivalent. Proxy-rotation-ready. */ readonly fetch: (url: string, init?: RequestInit) => Promise; /** * Playwright browser facade — lazy, stealth-by-default. Calling * `newPage()` returns a fully patched Page that survives Akamai / * Cloudflare / DataDome / PerimeterX checks without extra setup. */ readonly browser: BrowserFacade; /** Stream a result out. Each call persists immediately. */ readonly pushData: (data: Record) => void; /** Key-value checkpoint store, persisted to disk. */ readonly checkpoint: { readonly get: (key: string) => Promise; readonly set: (key: string, value: unknown) => Promise; readonly has: (key: string) => Promise; readonly delete: (key: string) => Promise; }; /** Task queue with deduplication and persistence. */ readonly tasks: { readonly add: (task: { readonly id: string; readonly [key: string]: unknown; }) => void; readonly process: (handler: (task: Record) => Promise) => Promise; }; /** Structured logger. */ readonly log: { readonly info: (msg: string, data?: unknown) => void; readonly warn: (msg: string, data?: unknown) => void; readonly error: (msg: string, data?: unknown) => void; readonly debug: (msg: string, data?: unknown) => void; }; /** Rate-limit-aware sleep. */ readonly sleep: (ms: number) => Promise; /** * Human-like behavior helpers: jittered pauses, burst-pause patterns, * mouse movement, warmup navigation, smooth scrolling. Use these * instead of `sleep(N)` whenever you can — fixed intervals are the * easiest pattern for anti-bot stacks to fingerprint. */ readonly human: HumanizeAPI; /** * Run-time arguments passed via CLI flags (e.g. `--cities=delhi`). * Each scraper interprets its own args; the framework does NOT validate * keys or values. Empty if the run was invoked without flags. Reserved * framework flags (--out) are stripped before this map is built. */ readonly args: Record; } /** Definition of an agent-authored scraper. */ export interface ScraperDefinition { /** Unique identifier for this scraper. */ readonly id: string; /** Source domain (informational). */ readonly source?: string; /** The scraping logic. Receives the runtime context. */ readonly run: (ctx: ScraperContext) => Promise; } /** Identity function that returns the scraper definition. Provides type checking for scraper authors. */ export declare function defineScraper(definition: ScraperDefinition): ScraperDefinition;