import type { ActionPolicy } from "./core/discover/index.js"; import type { RunScenarioOptions } from "./run.js"; import type { Driver, Reporter, SkillStore, TraceSink } from "./core/ports.js"; import type { Assertion, RunUsage, Scenario, Verdict } from "./core/types.js"; export interface SuiteCase { /** Names the frozen skill (`/.skill.json`) — stable across runs, no path separators. */ id: string; /** What to do, in natural language — discover's input on a cache miss. */ intent: string; /** Case start URL; falls back to `SuiteOptions.baseUrl`. One of the two is required. */ url?: string; /** The USER's success criteria in natural language, frozen as `expect` assertions * (LLM-judged at replay — a case with none replays fully deterministically). */ expect?: string[]; /** Mechanical assertions the user wants frozen alongside the derived ones. */ assertions?: Assertion[]; /** Discover step cap for this case. */ maxSteps?: number; } /** A frozen skill's staleness fingerprint: the case's user-authored inputs (intent + the criteria * merged into the freeze) at the time it was discovered. Suite-local extension of `Scenario` — NOT * a core type change; `Scenario`/`SkillStore` stay shared with plain discover/replay/CLI (spec §2, * pattern ≠ data). An undeclared field rides through `Scenario` objects unchanged (plain JS object * spread), so a healed re-freeze that starts from the original scenario keeps its hash for free. */ export type FrozenSuiteScenario = Scenario & { caseHash: string; }; /** Fingerprints exactly the case fields that flow into the freeze: `intent`, the criteria * (`expect`/`assertions`), and the START URL — discover freezes `url ?? baseUrl` as the first * `goto`, so repointing either one changes what replays and must read as stale (#131). `id` and * `maxSteps` stay out: file key and step cap, neither changes what was discovered or judged. */ export declare function hashCase(c: SuiteCase, baseUrl?: string): string; export interface SuiteOptions extends Pick { /** Where frozen skills live. Default: a FileSkillStore with refs under `skillDir`. */ store?: SkillStore; /** Path prefix for the default store's refs. Default "skills". */ skillDir?: string; /** Fallback start URL for cases that don't carry their own. */ baseUrl?: string; /** Repair broken replays (locator + surgical + outcome heal) and re-freeze. Default true — * a suite is unattended by design; pass false for a strict regression gate. */ heal?: boolean; /** Builds the browser for each discovery and each replay — a FRESH driver per case isolates * case state (auth, storage, dialogs). Default: ChromeDevToolsDriver. The suite closes what * this factory returns (#98: whoever constructs owns). */ driverFactory?: () => Driver; /** Per-case result reporter, forwarded to `runScenario`. */ reporter?: Reporter; /** Gate discovered actions — forwarded to discover AND to the heal re-discovery. */ policy?: ActionPolicy; /** Fired after each case with its verdict — a host's live progress feed. */ onCase?: (verdict: SuiteVerdict) => void; /** Lifecycle event stream for the whole suite (spec/core/trace.md): one header, then every * case's events under its `caseRef` (= case id), then run-end. */ trace?: TraceSink; } export interface SuiteVerdict { id: string; intent: string; verdict: Verdict; skillRef: string; /** True when this run had to discover the case (cache miss); false = pure replay. */ discovered: boolean; /** True when discovery hit its step cap — the case failed closed and nothing was frozen. */ truncated?: boolean; /** Locator + surgical step heals the replay needed (0 on a clean replay). */ heals: number; /** Discovery + replay combined. A cached mechanical-only case shows llmCalls: 0. */ usage: RunUsage; } export interface SuiteResult { verdicts: SuiteVerdict[]; /** Every case passed. */ passed: boolean; /** Whole-suite LLM usage. */ usage: RunUsage; } export declare function runSuite(cases: SuiteCase[], opts?: SuiteOptions): Promise;