/** * Code-only predicate evaluation. No model call is ever made on this path, so a * criterion is ground truth the executor cannot talk its way past (D20). * * Written fresh for the new core (D34). The old src/plan/evaluate.ts informed the * shape only. */ import type { CheckResult, PageFacts, Predicate, Verification } from "./types.ts"; export const PREDICATE_KIND_LIST = [ "url_includes", "title_includes", "text_visible", "text_absent", "ref_exists", "control_exists", "control_absent", "value_equals", "value_includes", "no_console_error", "dialog_open", "all", "any", "not", ] as const; export const PREDICATE_KINDS = new Set(PREDICATE_KIND_LIST); export function allowedPredicateKinds(): string { return [...PREDICATE_KINDS].join(", "); } function includesInsensitive(haystack: string, needle: string): boolean { return haystack.toLowerCase().includes(needle.toLowerCase()); } /** * What the page says, including what has been typed into it and what its rows say. * * `facts.text` is the body's innerText, and three kinds of visible thing are missing * from it. The value of an input: type a name into a box and the page's text does not * change, so asking whether the text you just typed is visible answers no however well * the typing worked. A control's accessible name, which is what the model is shown and * what it addresses. And the row a control sits in, which is where a list keeps the * other half of an identity - a handle in the anchor, a display name in the span beside * it - and asking for the display name of a row the agent can plainly see should not * answer no. * * All three are things a person looking at the screen can read, and all three come from * the same snapshot the model is shown, where passwords are already redacted. */ function readableText(facts: PageFacts): string { const fromControls: string[] = []; for (const control of facts.observation.controls) { if (control.value) fromControls.push(control.value); if (control.name) fromControls.push(control.name); if (control.row) fromControls.push(control.row); } return fromControls.length === 0 ? facts.text : `${facts.text}\n${fromControls.join("\n")}`; } /** * Where the text was, or what we looked through and did not find it in. * * The detail used to be a fixed sentence written for the failing case, so a passing * check reported `pass text visible "x": text not found: "x"`. The model read that * contradiction on every successful check, and so did anyone reading the ledger. */ function describeTextMatch(haystack: string, needle: string, found: boolean): string { if (!found) return `no match in ${haystack.length} characters of visible text`; const at = haystack.toLowerCase().indexOf(needle.toLowerCase()); const from = Math.max(0, at - 20); const window = haystack.slice(from, at + needle.length + 20).replace(/\s+/g, " ").trim(); return `found in "${window}"`; } function findControl(facts: PageFacts, role: string | undefined, name: string | undefined) { return facts.observation.controls.find((control) => { if (role && control.role !== role) return false; if (name && !includesInsensitive(control.name, name)) return false; return true; }); } function describeControl(control: { ref: string; role: string; name: string }): string { return `${control.ref} ${control.role} "${control.name}"`; } /** * How one check reads, wherever it is reported. * * Both halves are needed and they live in different fields: what was wanted is the * predicate, what was seen is the detail. Reporting only the detail is what left a * recovery note saying "no match in 101 characters" without saying no match for what. */ export function describeCheck(check: CheckResult): string { return `${check.predicate}: ${check.detail}`; } function findByName(facts: PageFacts, name: string) { return facts.observation.controls.find((control) => includesInsensitive(control.name, name)); } export function describePredicate(pred: Predicate): string { switch (pred.kind) { case "url_includes": return `url includes "${typeof pred.text === "string" ? pred.text : ""}"`; case "title_includes": return `title includes "${typeof pred.text === "string" ? pred.text : ""}"`; case "text_visible": return `text visible "${pred.text}"`; case "text_absent": return `text absent "${pred.text}"`; case "ref_exists": return `ref ${pred.ref} exists`; case "control_exists": return `control exists ${pred.role ?? "any"} "${pred.name ?? ""}"`; case "control_absent": return `control absent ${pred.role ?? "any"} "${pred.name ?? ""}"`; case "value_equals": return `"${pred.name}" equals "${pred.text}"`; case "value_includes": return `"${pred.name}" includes "${pred.text}"`; case "no_console_error": return "no console error"; case "dialog_open": return pred.open ? "a dialog is open" : "no dialog is open"; case "all": return `all of (${pred.of.map(describePredicate).join(", ")})`; case "any": return `any of (${pred.of.map(describePredicate).join(", ")})`; case "not": return `not (${describePredicate(pred.of)})`; default: { const kind = (pred as { kind?: unknown }).kind; return `unknown predicate "${String(kind ?? "")}"`; } } } /** Evaluate one predicate. Returns the outcome plus human-readable actuals. */ export function evaluatePredicate(pred: Predicate, facts: PageFacts): CheckResult { const kind = (pred as { kind?: unknown })?.kind; if (typeof kind !== "string" || !PREDICATE_KINDS.has(kind)) { return { passed: false, detail: `unknown predicate kind ${JSON.stringify(kind ?? null)}`, predicate: typeof kind === "string" && kind ? kind : "unknown", }; } const description = describePredicate(pred); const result = (passed: boolean, detail: string): CheckResult => ({ passed, detail, predicate: description, }); switch (pred.kind) { case "url_includes": return result( typeof pred.text === "string" && facts.url.includes(pred.text), `url=${facts.url}`, ); case "title_includes": return result( typeof pred.text === "string" && includesInsensitive(facts.title, pred.text), `title=${facts.title}`, ); case "text_visible": { if (typeof pred.text !== "string") { return result(false, "missing text"); } const text = readableText(facts); const seen = includesInsensitive(text, pred.text); return result(seen, describeTextMatch(text, pred.text, seen)); } case "text_absent": { if (typeof pred.text !== "string") { return result(false, "missing text"); } const text = readableText(facts); const seen = includesInsensitive(text, pred.text); return result(!seen, describeTextMatch(text, pred.text, seen)); } case "ref_exists": { // The whole ref table used to be the detail, which cost 365 bytes to answer yes. const live = facts.observation.controls; const present = live.some((c) => c.ref === pred.ref); return result(present, present ? "present" : `absent, ${live.length} refs live`); } case "control_exists": { const match = findControl(facts, pred.role, pred.name); return result(Boolean(match), match ? describeControl(match) : summarizeControls(facts)); } case "control_absent": { const match = findControl(facts, pred.role, pred.name); return result(!match, match ? describeControl(match) : "absent"); } case "value_equals": { const control = findByName(facts, pred.name); const actual = control?.value ?? ""; return result(actual === pred.text, `actual="${actual}"`); } case "value_includes": { const control = findByName(facts, pred.name); const actual = control?.value ?? ""; return result(includesInsensitive(actual, pred.text), `actual="${actual}"`); } case "no_console_error": return result( facts.observation.consoleErrors.length === 0, facts.observation.consoleErrors.join("; ") || "none", ); case "dialog_open": { const open = facts.observation.dialogs.length > 0; return result(open === pred.open, open ? facts.observation.dialogs.join("; ") : "none open"); } case "all": { const parts = pred.of.map((p) => evaluatePredicate(p, facts)); const failed = parts.filter((p) => !p.passed); return result(failed.length === 0, failed.map((f) => f.detail).join(" | ") || "all passed"); } case "any": { const parts = pred.of.map((p) => evaluatePredicate(p, facts)); const passed = parts.some((p) => p.passed); return result(passed, passed ? "one matched" : parts.map((p) => p.detail).join(" | ")); } case "not": { const inner = evaluatePredicate(pred.of, facts); return result(!inner.passed, inner.detail); } default: return result(false, `unknown predicate kind ${JSON.stringify(kind)}`); } } function summarizeControls(facts: PageFacts): string { return facts.observation.controls .slice(0, 12) .map((c) => `${c.role}:${c.name}`) .join(", "); } /** Evaluate a list of predicates as one verification. */ export function verify(predicates: Predicate[], facts: PageFacts): Verification { const checks = predicates.map((pred) => evaluatePredicate(pred, facts)); return { status: checks.every((check) => check && check.passed) ? "passed" : "failed", checks, }; } /** * Reject unknown kinds and script-shaped input before evaluation. Follows the * closed-verb pattern the old page-plan validator used: anything not in the set * is refused rather than best-guessed. */ export function validatePredicate(value: unknown, path = "predicate"): string[] { if (value === null || typeof value !== "object" || Array.isArray(value)) { return [`${path}: expected an object`]; } const pred = value as { kind?: unknown; of?: unknown; [key: string]: unknown }; if (typeof pred.kind !== "string") { return [`${path}: missing "kind" (allowed: ${allowedPredicateKinds()})`]; } if (!PREDICATE_KINDS.has(pred.kind)) { return [`${path}: unknown predicate kind "${pred.kind}" (allowed: ${allowedPredicateKinds()})`]; } const errors: string[] = []; const needsText = ["url_includes", "title_includes", "text_visible", "text_absent"]; if (needsText.includes(pred.kind) && typeof pred.text !== "string") { errors.push(`${path}: "${pred.kind}" needs a string "text"`); } if (pred.kind === "ref_exists" && typeof pred.ref !== "string") { errors.push(`${path}: "ref_exists" needs a string "ref"`); } if ( (pred.kind === "value_equals" || pred.kind === "value_includes") && (typeof pred.name !== "string" || typeof pred.text !== "string") ) { errors.push(`${path}: "${pred.kind}" needs string "name" and "text"`); } if (pred.kind === "dialog_open" && typeof pred.open !== "boolean") { errors.push(`${path}: "dialog_open" needs a boolean "open"`); } if (pred.kind === "all" || pred.kind === "any") { if (!Array.isArray(pred.of) || pred.of.length === 0) { errors.push(`${path}: "${pred.kind}" needs a non-empty "of" array`); } else { pred.of.forEach((child, index) => { errors.push(...validatePredicate(child, `${path}.of[${index}]`)); }); } } if (pred.kind === "not") { errors.push(...validatePredicate(pred.of, `${path}.of`)); } for (const key of ["script", "code", "evaluate", "fn", "expression"]) { if (key in pred) { errors.push(`${path}: "${key}" is not allowed; predicates are data, not code`); } } return errors; } export function parsePredicate(value: unknown): Predicate { const errors = validatePredicate(value); if (errors.length > 0) { throw new Error(errors.join("; ")); } return value as Predicate; } /** * A predicate the caller offered, or nothing. * * Used at the tool boundary. Evaluating a missing `text` is how `url includes * "undefined"` appeared: `String.prototype.includes` stringifies the hole. */ export function optionalPredicate(value: unknown): Predicate | undefined { if (value === undefined || value === null) return undefined; return validatePredicate(value).length === 0 ? (value as Predicate) : undefined; }