/** * Phase 2 record-and-replay cache for `rp ai-test`. * * On a green run we persist the navigation skeleton the agent used to reach the * product-module surface — expressed as DURABLE locator hints (role + accessible * name, visible text, or a URL), never the ephemeral `ref` ids @playwright/mcp * snapshots hand out (e17, e42…), which change between runs and can't be * replayed. On the next run we feed that skeleton back to the agent as a * fast-path to follow instead of re-discovering the whole route. * * The platform is a CONSTANT (see scenario-prompt.ts): the agent may freely heal * a step whose target moved/renamed and we re-save the corrected skeleton. The * product-module assertion is the VARIABLE under test and is never cached or * healed — `scenario_expected` is re-evaluated strictly every run. * * Cache invalidation is fingerprint-based: editing a scenario's * description/inputs/expected changes the fingerprint, so a stale skeleton for * an older version of the scenario is ignored rather than replayed. */ import { createHash } from 'node:crypto'; import path from 'node:path'; export type TraceAction = 'navigate' | 'click' | 'fill' | 'select' | 'expect' | 'hover'; export type TraceLocatorKind = 'url' | 'role' | 'text' | 'label' | 'placeholder' | 'testid' | 'css'; /** * A STRUCTURED locator — the single most important contract in fast-replay. * * The recording agent used to emit free-form prose ("heading 'Root Funeral' * card -> button 'Add'", "text Live | 1.0") which raw Playwright cannot resolve, * so replay healed on every scenario and the speed-up never materialised. Now * the agent emits exactly one Playwright locator primitive per step, and * deterministic-replay.ts maps `kind` straight onto `page.getBy*` with no * parsing or guessing. Exactly one of the kind-specific fields is populated to * match `kind`; `name`/`exact` are optional refinements for role/text/label. */ export interface TraceLocator { kind: TraceLocatorKind; /** kind: 'url' — absolute URL for a navigate step. */ url?: string; /** kind: 'role' — ARIA role (button, link, textbox, combobox, heading…). */ role?: string; /** kind: 'role' — accessible name refining the role match. */ name?: string; /** kind: 'text' — visible text. */ text?: string; /** kind: 'label' — form-field label. */ label?: string; /** kind: 'placeholder' — input placeholder. */ placeholder?: string; /** kind: 'testid' — data-testid value. */ testid?: string; /** kind: 'css' — raw CSS selector (last resort). */ css?: string; /** Case-sensitive whole-string match for role/text/label. Defaults to false. */ exact?: boolean; /** * Engine-synthesised scoping refinement (NEVER emitted by the recording agent, * which can't see the DOM). When a role/text locator matches several elements * that share an accessible name (duplicate table rows), replay narrows the set * to the one element ALSO containing this visible text — Playwright's * `.filter({ hasText })` — derived from a distinguishing value elsewhere in the * recorded step sequence. Absent → no filtering, identical to legacy behaviour. */ hasText?: string; } export interface TraceStep { action: TraceAction; /** Structured Playwright locator. URL kind for `navigate`. */ locator: TraceLocator; /** Value typed/selected for `fill` / `select`. Omitted for the rest. */ value?: string; } /** The single string field that must be populated for each locator kind. */ const LOCATOR_REQUIRED_FIELD: Record = { url: 'url', role: 'role', text: 'text', label: 'label', placeholder: 'placeholder', testid: 'testid', css: 'css', }; export interface ScenarioTrace { scenarioId: string; moduleKey: string; dashboardHost: string; /** Hash of the scenario's description+inputs+expected at record time. */ fingerprint: string; recordedAt: string; /** * The semantic verdict reason captured when this path was recorded (e.g. the * premium + policy number that was asserted). Replays re-execute the same steps * but produce only a mechanical "re-ran N steps" reason, so we carry the rich * recorded reason forward to keep replay verdicts meaningful. Optional: older * caches predate this field. */ recordedReason?: string; steps: TraceStep[]; } // Single top-level dir for ALL `rp ai-test` artifacts in a module, so a run // creates one tidy `ai-test/` folder (cache + per-run output) instead of // scattering sibling `ai-test-cache/` and `ai-test-runs/` dirs at the module root. export const AI_TEST_DIRNAME = 'ai-test'; export const CACHE_DIRNAME = `${AI_TEST_DIRNAME}/cache`; /** Minimal fs surface so the cache is unit-testable without touching disk. */ export interface CacheIo { readFile: (p: string) => string; writeFile: (p: string, c: string) => void; ensureDir: (p: string) => void; fileExists: (p: string) => boolean; } export interface ScenarioFingerprintInput { description: string; inputs: string; expected: string; } /** Stable short hash of the scenario's semantic content. */ export const scenarioFingerprint = (s: ScenarioFingerprintInput): string => createHash('sha256').update(`${s.description}\n${s.inputs}\n${s.expected}`).digest('hex').slice(0, 16); const toFsSafe = (v: string) => v.replaceAll(/[^a-z0-9_.-]/gi, '_'); /** Filesystem-safe path for a scenario's cache file under the module dir. */ export const cacheFilePath = (moduleDir: string, dashboardHost: string, scenarioId: string): string => path.join(moduleDir, CACHE_DIRNAME, `${toFsSafe(dashboardHost)}__${toFsSafe(scenarioId)}.json`); const isTraceLocator = (v: unknown): v is TraceLocator => { if (typeof v !== 'object' || v === null) return false; const l = v as Record; const kinds = Object.keys(LOCATOR_REQUIRED_FIELD) as TraceLocatorKind[]; if (!kinds.includes(l.kind as TraceLocatorKind)) return false; const field = LOCATOR_REQUIRED_FIELD[l.kind as TraceLocatorKind]; if (typeof l[field] !== 'string' || l[field].length === 0) return false; if (l.name !== undefined && typeof l.name !== 'string') return false; if (l.exact !== undefined && typeof l.exact !== 'boolean') return false; if (l.hasText !== undefined && typeof l.hasText !== 'string') return false; return true; }; const isTraceStep = (v: unknown): v is TraceStep => { if (typeof v !== 'object' || v === null) return false; const s = v as Record; const actions: TraceAction[] = ['navigate', 'click', 'fill', 'select', 'expect', 'hover']; if (!actions.includes(s.action as TraceAction)) return false; // Rejects the legacy string-`target` shape: those caches fail validation, // heal once via the AI agent, and re-save in the structured shape. if (!isTraceLocator(s.locator)) return false; if (s.value !== undefined && typeof s.value !== 'string') return false; return true; }; /** Keep only the kind-relevant fields so a persisted locator is minimal/stable. */ const cleanLocator = (l: TraceLocator): TraceLocator => { const field = LOCATOR_REQUIRED_FIELD[l.kind]; const out: TraceLocator = { kind: l.kind, [field]: l[field] } as TraceLocator; if (l.kind === 'role' && l.name !== undefined) out.name = l.name; if (l.exact !== undefined) out.exact = l.exact; if (l.hasText !== undefined) out.hasText = l.hasText; return out; }; /** * Field names the agent has been seen to use for a locator's primitive value * when it omits the contract's kind-specific field (e.g. `{kind:'css', * selector:'#id'}` or `{kind:'url', value:'http://…'}`). Tried in order. */ const LOCATOR_VALUE_FALLBACKS = ['value', 'selector', 'target'] as const; /** * Map a stray primitive field onto the field this locator's `kind` requires. * The recording agent is an LLM and drifts on which key holds the value between * runs; this folds the known aliases back onto the canonical field so the step * still validates. A locator that already has its required field is untouched. */ const normalizeLocator = (loc: Record): Record => { const kind = loc.kind; if (typeof kind !== 'string' || !(kind in LOCATOR_REQUIRED_FIELD)) return loc; const required = LOCATOR_REQUIRED_FIELD[kind as TraceLocatorKind]; const current = loc[required]; if (typeof current === 'string' && current.length > 0) return loc; for (const fb of LOCATOR_VALUE_FALLBACKS) { const v = loc[fb]; if (typeof v === 'string' && v.length > 0) return { ...loc, [required]: v }; } return loc; }; /** * Infer a missing `kind` on a NESTED locator object from which primitive field * it carries. Observed live on main-life-child (2026-07-03): the agent emitted * every locator nested under `locator` but WITHOUT the `kind` discriminant — * `{"css":"#cover_amount"}`, `{"role":"button","text":"New policy"}` — so every * step was rejected, the whole ~10-minute record was discarded ("agent passed * but emitted no trace") and the scenario re-recorded. The shapes are * unambiguous: exactly one kind's required field is present (checked in * LOCATOR_REQUIRED_FIELD order, so `role` wins over `text` when both appear — * on a role locator the drifted `text` holds the ACCESSIBLE NAME and is folded * onto `name`). A locator that already has `kind`, or carries no known field, * passes through untouched — this never invents a locator. */ const inferMissingKind = (loc: Record): Record => { if (typeof loc.kind === 'string') return loc; const kinds = Object.keys(LOCATOR_REQUIRED_FIELD) as TraceLocatorKind[]; for (const kind of kinds) { const field = LOCATOR_REQUIRED_FIELD[kind]; const v = loc[field]; if (typeof v === 'string' && v.length > 0) { const out: Record = { ...loc, kind }; if (kind === 'role' && typeof out.name !== 'string' && typeof out.text === 'string') { out.name = out.text; delete out.text; } return out; } } return loc; }; /** * Infer a `{kind, }` locator from a COMPACT FLAT step — one that carries * a locator primitive directly on the step with NO `kind` and NO nested * `locator`, e.g. `{"action":"click","role":"button","name":"New policy"}` or * `{"action":"fill","css":"#cover_amount","value":"50000"}`. Observed live on * policy-issue-main-member-child (2026-06-25): the agent emitted the whole trace * in this shape with no `::trace::` prefix, so every step was rejected and the * scenario saved no cache ("no usable trace") and re-recorded every run. We map * the first present primitive field (in LOCATOR_REQUIRED_FIELD order) to its * kind, carrying the `name`/`exact` refinements role needs. Returns undefined * when no known primitive field is present (step passes through and is rejected * downstream) — this never invents a locator. */ const inferLocatorFromCompactStep = (s: Record): Record | undefined => { const kinds = Object.keys(LOCATOR_REQUIRED_FIELD) as TraceLocatorKind[]; for (const kind of kinds) { const field = LOCATOR_REQUIRED_FIELD[kind]; const v = s[field]; if (typeof v === 'string' && v.length > 0) { return { kind, [field]: v, ...(typeof s.name === 'string' ? { name: s.name } : {}), ...(typeof s.exact === 'boolean' ? { exact: s.exact } : {}), }; } } return undefined; }; /** * Keys the agent has been seen to put the action verb under instead of the * contract's `action`. Observed live on policy-issue-main-member-spouse * (2026-06-22): the agent emitted every step as `{"step":"navigate",…}`, so * `isTraceStep` (which reads `s.action`) rejected all 35 steps and a fully * green issue flow cached nothing — re-recording (~380s) on every run. Tried * in order; `action` first so a correct step is untouched. */ const ACTION_KEY_ALIASES = ['action', 'step', 'act'] as const; /** * Verbs the agent emits that aren't in the replay engine's vocabulary * (`navigate|click|fill|select|expect|hover`), folded onto the equivalent the * engine can execute. A checkbox `check`/`uncheck` is a `click` on the same * label locator (a fresh replay form starts unchecked, so click→checked matches * the record). `goto`→`navigate`, `type`→`fill`, `assert`→`expect` cover the * other common synonyms. Canonical verbs map to themselves via lower-casing. */ const ACTION_VALUE_ALIASES: Record = { check: 'click', uncheck: 'click', tick: 'click', toggle: 'click', press: 'click', goto: 'navigate', go: 'navigate', open: 'navigate', type: 'fill', enter: 'fill', choose: 'select', assert: 'expect', verify: 'expect', }; /** * Keys the agent has been seen to nest the locator OBJECT under instead of the * contract's `locator`. Observed live on policy-issue-main-member-spouse * (2026-06-26): the agent emitted every step as `{action, target:{kind, value}}` * — `s.locator` was undefined, so all 31 steps of a fully green issue flow were * rejected, no cache was saved ("no usable trace"), and the scenario re-recorded * (~600s) every run. Tried in order; `locator` first so a correct step is * untouched. (`target` also appears in LOCATOR_VALUE_FALLBACKS for the distinct * case where it holds the primitive STRING; the two never collide because the * lookup here only accepts an object.) */ const LOCATOR_KEY_ALIASES = ['locator', 'target', 'element'] as const; /** Resolve a step's action across key drift (`step`/`act`) and verb synonyms. */ const resolveAction = (s: Record): unknown => { let raw: unknown; for (const k of ACTION_KEY_ALIASES) { const v = s[k]; if (typeof v === 'string' && v.length > 0) { raw = v; break; } } if (typeof raw !== 'string') return raw; const lc = raw.toLowerCase(); return ACTION_VALUE_ALIASES[lc] ?? lc; }; /** * Normalise a raw parsed step into the canonical `{action, locator, value}` * shape BEFORE validation. The recording agent drifts on JSON shape between * runs; the drifts observed live on policy-issue-main-member (2026-06-19) and * policy-issue-main-member-spouse (2026-06-22) all caused every step to be * rejected → the scenario re-recorded (~330-380s) every run. Tolerated here: * (1) a FLAT locator hoisted onto the step itself instead of nested under * `locator`; (2) the locator's primitive held in an aliased field (see * normalizeLocator); (3) the action verb under an aliased key or as a synonym * (see resolveAction). Unrecognised shapes pass through untouched and are * rejected downstream by isTraceStep — this rescues recognised drift, it never * invents a locator. */ const normalizeRawStep = (v: unknown): unknown => { if (typeof v !== 'object' || v === null) return v; const s = v as Record; const action = resolveAction(s); // The locator object is normally under `locator`, but the agent drifts to // `target`/`element`/`selector` (see LOCATOR_KEY_ALIASES). Take the first // alias that holds an object so the nested `{kind, value}` still validates. let locator: unknown; for (const k of LOCATOR_KEY_ALIASES) { if (typeof s[k] === 'object' && s[k] !== null) { locator = s[k]; break; } } if (locator === undefined && typeof s.kind === 'string') { const rest = { ...s }; for (const k of ACTION_KEY_ALIASES) delete rest[k]; delete rest.value; locator = rest; } else if (locator === undefined) { locator = inferLocatorFromCompactStep(s); } if (typeof locator === 'object' && locator !== null) { locator = normalizeLocator(inferMissingKind(locator as Record)); } return { action, locator, ...(s.value === undefined ? {} : { value: s.value }) }; }; /** * Pull the step array out of whatever the agent emitted. The contract is a bare * array, but the agent has been seen to wrap it as `{id, steps:[…]}` (2026-06-19) * — which made `sanitizeSteps` reject the whole payload as a non-array. Unwrap * the common wrapper keys; anything else yields null (treated as no steps). */ const asStepArray = (raw: unknown): unknown[] | null => { if (Array.isArray(raw)) return raw; if (typeof raw === 'object' && raw !== null) { const o = raw as Record; if (Array.isArray(o.steps)) return o.steps; if (Array.isArray(o.trace)) return o.trace; } return null; }; /** Coerce an untrusted parsed payload into TraceSteps, dropping malformed entries. */ export const sanitizeSteps = (raw: unknown): TraceStep[] => { const arr = asStepArray(raw); if (!arr) return []; return arr .map(normalizeRawStep) .filter(isTraceStep) .map((s) => ({ action: s.action, locator: cleanLocator(s.locator), ...(s.value === undefined ? {} : { value: s.value }), })); }; /** * A run-minted identifier — policy / quote / application reference: uppercase * alphanumeric, 8–12 chars, with at least one letter AND one digit (e.g. * KJVO0F8TLU, 4ER13VQZ93). These are unique per run, so an `expect` on one * NEVER matches on replay and forces a full re-record every single time. */ const GENERATED_REF = /^(?=.*[A-Z])(?=.*\d)[A-Z\d]{8,12}$/; const isGeneratedRefAssert = (s: TraceStep): boolean => { if (s.action !== 'expect') return false; const { kind, text, name } = s.locator; const asserted = kind === 'text' ? text : kind === 'role' ? name : undefined; return typeof asserted === 'string' && GENERATED_REF.test(asserted); }; /** * Strip `expect` steps that assert a run-minted reference number. The scenario's * real pass/fail still comes from the agent verdict plus the stable asserts * (status "Active", success headings); keeping the per-run id only guarantees a * replay miss. Applied on the RECORD path so the cache is born deterministic. */ export const dropGeneratedRefAsserts = (steps: TraceStep[]): TraceStep[] => steps.filter((s) => !isGeneratedRefAssert(s)); /** * Load a cached trace for this scenario, or null if there's no usable one. * Returns null (rather than throwing) for every miss reason — a stale/corrupt * cache must never block a run; it just falls back to fresh discovery. */ export const loadTrace = ( io: CacheIo, params: { moduleDir: string; scenarioId: string; moduleKey: string; dashboardHost: string; fingerprint: string; }, ): ScenarioTrace | null => { const { moduleDir, scenarioId, moduleKey, dashboardHost, fingerprint } = params; const file = cacheFilePath(moduleDir, dashboardHost, scenarioId); if (!io.fileExists(file)) return null; let parsed: Partial; try { parsed = JSON.parse(io.readFile(file)) as Partial; } catch { return null; // corrupt cache → ignore, re-discover } // Any mismatch means this skeleton isn't for the current scenario/target. if (parsed.moduleKey !== moduleKey) return null; if (parsed.dashboardHost !== dashboardHost) return null; if (parsed.fingerprint !== fingerprint) return null; const steps = sanitizeSteps(parsed.steps); if (steps.length === 0) return null; return { scenarioId, moduleKey, dashboardHost, fingerprint, recordedAt: typeof parsed.recordedAt === 'string' ? parsed.recordedAt : '', recordedReason: typeof parsed.recordedReason === 'string' ? parsed.recordedReason : undefined, steps, }; }; /** Persist a known-good (or healed) trace for this scenario. */ export const saveTrace = ( io: CacheIo, params: { moduleDir: string; scenarioId: string; moduleKey: string; dashboardHost: string; fingerprint: string; steps: TraceStep[]; recordedReason?: string; now?: () => Date; }, ): ScenarioTrace | null => { const { moduleDir, scenarioId, moduleKey, dashboardHost, fingerprint, recordedReason, now = () => new Date(), } = params; const steps = sanitizeSteps(params.steps); // Nothing durable to replay — don't write an empty skeleton. if (steps.length === 0) return null; const trace: ScenarioTrace = { scenarioId, moduleKey, dashboardHost, fingerprint, recordedAt: now().toISOString(), ...(recordedReason ? { recordedReason } : {}), steps, }; const file = cacheFilePath(moduleDir, dashboardHost, scenarioId); io.ensureDir(path.dirname(file)); io.writeFile(file, JSON.stringify(trace, null, 2)); return trace; };