/** * XHR / network response capture for recon. * * Some sites (Housing.com, NoBroker, many SPAs) don't put data in the * HTML — they fetch it via XHR after `DOMContentLoaded`. The schema walker * and visual audit see nothing useful because there's nothing in the * document to look at. The data is in the network responses. * * This helper registers a Playwright response listener BEFORE navigation, * collects JSON (and optionally HTML-partial) responses matching a URL * pattern, and returns them in the same `HydrationPayload` shape so * `walkSchema` works on them unchanged. * * Recon-time only. The production scraper, once it knows which endpoints * matter, should call those APIs directly via `ctx.fetch` (Tier 2) — far * more efficient than re-running the whole page. */ import type { Page } from "playwright"; import type { HydrationPayload } from "./schemaWalker.js"; export interface CapturedResponse { /** Full response URL. */ readonly url: string; /** HTTP status. */ readonly status: number; /** Request method. */ readonly method: string; /** Content type from response headers, lowercase. */ readonly contentType: string; /** Response body — parsed JSON if `application/json`, raw text otherwise. */ readonly data: unknown; /** True if the response was JSON-parsed successfully. */ readonly isJson: boolean; } export interface XhrCaptureOptions { /** * Only capture responses whose URL matches. Default: any URL containing * `/api/`, `/graphql`, `/_next/data/`, `/v1/`, `/v2/`, or `/rest/`. */ readonly urlMatcher?: RegExp; /** * Maximum responses to capture (safety cap). Default 100. */ readonly maxResponses?: number; /** * Also capture `text/html` partials (Turbo/HTMX/morphdom). Default false. */ readonly captureHtmlPartials?: boolean; /** * Skip responses larger than this (bytes). Default 5 MB. Avoids buffering * giant image / video / data dumps. */ readonly maxBytes?: number; } export interface XhrCaptureHandle { /** Stop listening and return the captured responses. */ readonly stop: () => readonly CapturedResponse[]; /** Get a live snapshot without stopping. */ readonly snapshot: () => readonly CapturedResponse[]; /** Number of responses captured so far. */ readonly count: () => number; } /** * Start capturing XHR / fetch responses. Call BEFORE `page.goto` so you * catch the initial-load XHRs. Call `.stop()` after `waitForLoadState` * or a fixed dwell to get the captured responses. * * Example: * ```typescript * const capture = await startXhrCapture(page, { urlMatcher: /\/api\// }) * await page.goto(url, { waitUntil: "networkidle" }) * await ctx.human.pause(2000, 4000) * const responses = capture.stop() * ctx.log.info("captured", { count: responses.length }) * ``` */ export declare function startXhrCapture(page: Page, options?: XhrCaptureOptions): Promise; /** * Convert captured JSON responses to `HydrationPayload` shape so * `walkSchema` and `filterLeaves` work on them unchanged. * * Example: * ```typescript * const responses = capture.stop() * const payloads = capturedToPayloads(responses) * for (const p of payloads) { * const leaves = walkSchema(p.data, { arraySampleSize: 2 }) * ctx.log.info(p.source, { count: leaves.length }) * } * ``` */ export declare function capturedToPayloads(responses: readonly CapturedResponse[]): readonly HydrationPayload[]; /** * Group captured responses by URL path (ignoring query params). Useful * for "this page fires 5 different endpoints; here's one per endpoint * with a sample response" recon summary. */ export declare function groupByEndpoint(responses: readonly CapturedResponse[]): ReadonlyMap;