/** * Visual audit helpers — open a page, take viewport-sized screenshots, * and dump the DOM text inventory so the agent can compare scraped data * to what's visible to a human. * * Used during Phase 5e of site reconnaissance ("visual ground truth"). * Stays out of the scraper hot path — recon-time only. * * Designed to be SITE-AGNOSTIC: we tried several real sites (99acres, * MagicBricks, IndiaMART) and the common shapes for label:value data are: * * - "Label: Value" in a single text node (99acres, generic SSR) * - (IndiaMART spec tables) * -
Label
Value
(definition lists, common) * - sibling spans/divs with "label" or "value" class hints (MagicBricks) * * We emit each as a separate field so consumers know provenance. */ import type { Page } from "playwright"; export interface LabelValue { readonly label: string; readonly value: string; } export interface AuditResult { readonly screenshotPaths: readonly string[]; /** All headings (h1/h2/h3) — gives you section structure. */ readonly headings: readonly string[]; /** "Label: Value" extracted from single text nodes (the old colon regex). */ readonly colonPairs: readonly LabelValue[]; /**
......
rows. */ readonly tableRows: readonly LabelValue[]; /**
term
def
entries. */ readonly definitionList: readonly LabelValue[]; /** Label/value pairs found via sibling-class heuristic (e.g. MagicBricks). */ readonly siblingPairs: readonly LabelValue[]; /** * Stacked layout pairs — two text children inside a flex/grid cell where * the first is short (≤40 chars) and visually sits above the second. * Catches the CommonFloor "label-above-value" pattern. */ readonly stackedPairs: readonly LabelValue[]; /** * Pairs derived from data-* attribute hints (`data-testid`, `data-aut-id`, * `data-qa`, `data-q`). Modern React stacks (OLX, Airbnb, Booking, …) tag * semantic fields here instead of via meaningful class names. */ readonly attrPairs: readonly LabelValue[]; /** * Merged + deduped union of all six sources. Use this for casual scans; * use the individual sources when you need provenance (e.g. table data is * usually more reliable than colon-regex hits). */ readonly labelValuePairs: readonly LabelValue[]; /** Status / badge text nodes. Conservatively filtered to avoid nav matches. */ readonly statusBadges: readonly string[]; /** Anchors that look like detail-page links. */ readonly detailLinks: readonly { href: string; text: string; }[]; /** Currency tokens visible in the rendered DOM. */ readonly currencyMentions: readonly string[]; } export interface AuditOptions { /** Output prefix for screenshots, e.g. "/tmp/99acres-srp-". */ readonly screenshotPrefix: string; /** Scroll fractions of viewport height for each capture. Default [0, 1.2, 2.5]. */ readonly scrollPoints?: readonly number[]; /** * Extra regex patterns to recognise as detail-page links. Merged with the * defaults (which catch `/spid-`, `/pdpid-`, `/proddetail/`, `/dp/`, `/p/N`, * and any path segment with 4+ consecutive digits). */ readonly detailLinkPatterns?: readonly RegExp[]; } /** * Audit a page that's already been navigated to. Caller is responsible for * launching the browser, navigating, and any anti-bot warmup. */ export declare function auditPage(page: Page, options: AuditOptions): Promise; /** * Diff a scraped record against an audit. Returns labels visible on the page * whose values don't appear anywhere in the scraped record. * * Filters out URL/phone "values" — those are 100% noise based on real-site * testing (IndiaMART contact numbers, partial URL fragments). */ export declare function findMissingFields(audit: AuditResult, scraped: Record): readonly LabelValue[];
labelvalue