/**
* Visual audit helpers — open a page, take viewport-sized screenshots,
* and dump the DOM text inventory so the agent can compare scraped data
* to what's visible to a human.
*
* Used during Phase 5e of site reconnaissance ("visual ground truth").
* Stays out of the scraper hot path — recon-time only.
*
* Designed to be SITE-AGNOSTIC: we tried several real sites (99acres,
* MagicBricks, IndiaMART) and the common shapes for label:value data are:
*
* - "Label: Value" in a single text node (99acres, generic SSR)
* -
| ... | ... |
(IndiaMART spec tables)
* - - Label
- Value
(definition lists, common)
* - sibling spans/divs with "label" or "value" class hints (MagicBricks)
*
* We emit each as a separate field so consumers know provenance.
*/
import type { Page } from "playwright";
export interface LabelValue {
readonly label: string;
readonly value: string;
}
export interface AuditResult {
readonly screenshotPaths: readonly string[];
/** All headings (h1/h2/h3) — gives you section structure. */
readonly headings: readonly string[];
/** "Label: Value" extracted from single text nodes (the old colon regex). */
readonly colonPairs: readonly LabelValue[];
/** | label | value |
rows. */
readonly tableRows: readonly LabelValue[];
/** - term
- def
entries. */
readonly definitionList: readonly LabelValue[];
/** Label/value pairs found via sibling-class heuristic (e.g. MagicBricks). */
readonly siblingPairs: readonly LabelValue[];
/**
* Stacked layout pairs — two text children inside a flex/grid cell where
* the first is short (≤40 chars) and visually sits above the second.
* Catches the CommonFloor "label-above-value" pattern.
*/
readonly stackedPairs: readonly LabelValue[];
/**
* Pairs derived from data-* attribute hints (`data-testid`, `data-aut-id`,
* `data-qa`, `data-q`). Modern React stacks (OLX, Airbnb, Booking, …) tag
* semantic fields here instead of via meaningful class names.
*/
readonly attrPairs: readonly LabelValue[];
/**
* Merged + deduped union of all six sources. Use this for casual scans;
* use the individual sources when you need provenance (e.g. table data is
* usually more reliable than colon-regex hits).
*/
readonly labelValuePairs: readonly LabelValue[];
/** Status / badge text nodes. Conservatively filtered to avoid nav matches. */
readonly statusBadges: readonly string[];
/** Anchors that look like detail-page links. */
readonly detailLinks: readonly {
href: string;
text: string;
}[];
/** Currency tokens visible in the rendered DOM. */
readonly currencyMentions: readonly string[];
}
export interface AuditOptions {
/** Output prefix for screenshots, e.g. "/tmp/99acres-srp-". */
readonly screenshotPrefix: string;
/** Scroll fractions of viewport height for each capture. Default [0, 1.2, 2.5]. */
readonly scrollPoints?: readonly number[];
/**
* Extra regex patterns to recognise as detail-page links. Merged with the
* defaults (which catch `/spid-`, `/pdpid-`, `/proddetail/`, `/dp/`, `/p/N`,
* and any path segment with 4+ consecutive digits).
*/
readonly detailLinkPatterns?: readonly RegExp[];
}
/**
* Audit a page that's already been navigated to. Caller is responsible for
* launching the browser, navigating, and any anti-bot warmup.
*/
export declare function auditPage(page: Page, options: AuditOptions): Promise;
/**
* Diff a scraped record against an audit. Returns labels visible on the page
* whose values don't appear anywhere in the scraped record.
*
* Filters out URL/phone "values" — those are 100% noise based on real-site
* testing (IndiaMART contact numbers, partial URL fragments).
*/
export declare function findMissingFields(audit: AuditResult, scraped: Record): readonly LabelValue[];