/** * `createScorer` and a batteries-included set of built-in scorers. * * A scorer is a 4-step pipeline (preprocess → analyze → generateScore → * generateReason). `createScorer` is a thin identity-with-validation factory: * it enforces the one hard contract (`generateScore` is required) and returns * a fully-typed `Scorer`. Built-in scorers below show both flavors: * * - `exactMatch` / `contains` — pure-JS analyze, no model. * - `llmJudge` — analyze runs an LLM judge through the resolved engine * (provider-agnostic; the model is whatever the runner resolved). */ import type { AgentRunOutput, Scorer, ScorerDefinition } from "./types.js"; /** * Create a scorer from a 4-step pipeline definition. * * `generateScore` is the only required step. `preprocess`/`analyze` default to * identity (the scorer sees the raw `AgentRunOutput`), and `generateReason` is * optional. */ export declare function createScorer
(def: ScorerDefinition): Scorer; /** Clamp any number into [0, 1]; coerce non-finite to 0. */ export declare function clamp01(n: number): number; /** * `exactMatch` — 1.0 when the agent's (trimmed, case-insensitive by default) * text equals `expected`, else 0.0. Pure JS, no model. */ export declare function exactMatch(expected: string, opts?: { caseSensitive?: boolean; }): Scorer; /** * `contains` — 1.0 when the agent's text contains every required substring * (case-insensitive by default). Score is the fraction matched, so a partial * hit still surfaces signal. Pure JS, no model. */ export declare function contains(needles: string | string[], opts?: { caseSensitive?: boolean; }): Scorer ; /** * `usesTool` — 1.0 when the agent invoked the named tool/action at least once. * Useful as a behavioral gate ("the agent must call send-email"). Pure JS. */ export declare function usesTool(toolName: string): Scorer ; interface JudgeVerdict { score: number; reasoning: string; } export interface LlmJudgeOptions { /** Scorer name (defaults to `llm_judge`). */ name?: string; /** What is being judged, e.g. "helpfulness". */ criteria: string; /** A rubric describing what 0.0 vs 1.0 means. */ rubric?: string; /** * The score scale the judge is told to use. Output is normalized to [0,1]. * Defaults to a 0..1 scale. */ scoreRange?: { min: number; max: number; }; } /** * `llmJudge` — an LLM-as-judge scorer. The analyze step asks the resolved * engine to score the agent output against a natural-language rubric and emit * `{ "score": , "reasoning": " " }`. The model is whatever the runner * resolved from the engine registry — this scorer NEVER hardcodes a provider * or model, so evals stay provider-agnostic. */ export declare function llmJudge(opts: LlmJudgeOptions): Scorer ; export {}; //# sourceMappingURL=scorer.d.ts.map