/** * faf bench — the AI-grounding benchmark (P1, agent-native). * * Measures the thing FAF sells, falsifiably: AI works better and faster WITH * structured context. Two numbers, one harness — grounding ACCURACY (N * project questions, cold vs with-faf) and grounding COST (tokens to get * grounded, as reported by the runner). * * The unfair advantage: the .faf IS the answer key. Questions derive from the * populated active slots; grading is mechanical (normalize + versioned alias * map + significant-token containment — no judge, no rubric drift). The * question-set hash rides a ✪ receipt, parity-hash discipline. * * DOCTRINE (wolfejam 2026-06-11): * - "A low score is an ALARM BELL — you are hemorrhaging tokens and the AI * is pretty much clueless about what you are doing, even trying to do." * Headline: "Trouble ahead, expensive trouble." * - NEVER render a low score alone — always the pair (cold → with-faf), or * the alarm framing with headroom. The delta is the product; the cold * number belongs to the ABSENCE of context ("without context"), never to * FAF. Output always ends in a prescription, never a verdict. * - The 6Ws are UNDERIVABLE from code: cold exploration can dig out a stack * (you pay for the dig); it can never find intent. */ /** Bump when phrasing, aliases, or grading rules change — part of the qset hash. */ export declare const BENCH_VERSION = "faf-bench/1"; export interface BenchQuestion { n: number; path: string; question: string; } /** * Versioned alias groups — mechanical equivalences, no fuzzy scoring. * Members are compared post-normalization. Extend deliberately; every change * bumps the qset hash via BENCH_VERSION. */ export declare const ALIAS_GROUPS: string[][]; export declare function normalizeAnswer(s: string): string; /** * Mechanical match: exact normalized equality, alias-group equality, or * every significant token of the EXPECTED value present in the answer * (deterministic set containment — handles sentence-shaped 6W answers * without a judge). A miss is a miss. */ export declare function answersMatch(expected: string, given: string): boolean; export interface QuestionSet { version: string; qsetHash: string; questions: BenchQuestion[]; /** Internal answer key — NEVER printed by `questions`; used by `grade`. */ answers: Record; } /** Derive the question set + answer key from a project.faf (active populated slots only). */ export declare function deriveQuestionSet(yaml: string): QuestionSet; /** * The answer-key-safe projection of a QuestionSet — version + qsetHash + * questions, NEVER `answers`. Any "give me the questions" surface (MCP tools, * UIs) MUST hand out THIS, not the raw QuestionSet: a tool that prints the * answer key makes the benchmark a lie. The CLI's `bench questions` follows * the same rule. */ export declare function publicQuestions(qset: QuestionSet): { version: string; qsetHash: string; questions: BenchQuestion[]; }; export interface GradeResult { correct: number; total: number; misses: BenchQuestion[]; perQuestion: { n: number; path: string; ok: boolean; }[]; } export declare function gradeAnswers(qset: QuestionSet, given: Record): GradeResult; export interface RunRecord { score: number; total: number; tokens?: number; model?: string; } export interface BenchState { version: string; qsetHash: string; protocol: 'in-session'; cold?: RunRecord; faf?: RunRecord; } /** ✪ receipt — sha256 over the canonical projection; third-party verifiable. * Includes `repo` so the receipt attests WHICH project was benched — two repos * that score identically on the same qset get distinct receipts (bench-only; * independent of the parity/trust score hash). */ export declare function buildReceipt(state: BenchState, repo?: string): { projection: string; hash: string; }; export interface BenchOptions { json?: boolean; cold?: boolean; faf?: boolean; tokens?: string; model?: string; file?: string; submit?: boolean; endpoint?: string; } export declare function benchCommand(action?: string, answersFile?: string, options?: BenchOptions): void;