/**
 * Validate-by-Reproduce drift-detection canary (Phase 2 A5).
 *
 * The A/B safety gate guards prompt *quality* — it catches a new prompt version
 * that scores worse. But an agent can drift WITHOUT its quality score moving: a
 * model update, an MCP tool breaking, or a prompt evolution's side-effect can
 * change *how* the agent gets to the answer (different tools, more turns, more
 * errors) while the final score stays flat. Vadim Nicolai's production write-up
 * captures it: "the agent still sounds fluent, but the trajectory is longer and
 * the tool-calls more repetitive."
 *
 * This module compares an agent's recent execution *trajectories* against a
 * frozen baseline and flags behavioural drift. It is the read side of the
 * trajectory capture shipped in A1 (`ExecutionTrace`).
 *
 * ── Method (grounded in 2026 agent-eval practice) ──
 * Exact-hash equivalence is the wrong tool — LLM runs are non-deterministic, so
 * an exact trajectory hash drifts every single run. Instead we use the same
 * deterministic, tolerance-based metrics the field settled on (LangChain's
 * `agentevals`, Vertex AI trajectory metrics, Galileo):
 *
 *   1. **Unordered tool-set Jaccard** — `|A∩B| / |A∪B|` over tool *names*.
 *      Catches new/missing tools regardless of order. Primary signal.
 *   2. **Ordered tool-sequence similarity** — normalised Levenshtein over the
 *      tool-name sequence. Catches re-ordering / repetition.
 *   3. **Turn-count ratio** — `candidate.turns / baseline.turns`. A ratio > 1.5
 *      is the canonical "agent is grinding" symptom.
 *   4. **Error-rate delta** — error spikes are the most reliable tool-break tell.
 *
 * We deliberately do NOT compare tool *arguments* (LLM-variable → noise) or
 * output length (verbosity swings without drift). We keep zero hard deps: these
 * metrics are ~a dozen lines each, so `agentevals` stays an inspiration, not a
 * dependency — Darwin's whole point is a pure, injectable core.
 *
 * ── Sampling ──
 * A single failing run is noise, not drift. The canary needs a *pattern*:
 * evaluate several same-version runs and alert only when ≥ N of them drift
 * (default 2). Three candidate runs per week is the field's rule of thumb.
 *
 * ── Baseline staleness (the load-bearing footgun) ──
 * A baseline is only meaningful against runs of the SAME prompt version. When a
 * prompt evolves, the new version *should* behave differently — comparing it to
 * the old baseline would fire a false alarm on every intentional improvement.
 * `runCanaryOverExperiments` therefore baselines strictly within the active
 * prompt version: after an evolution the canary reports `insufficient-data`
 * (re-baseline in progress) rather than screaming drift.
 *
 * Everything here is pure and deterministic. Side-effects (loading experiments,
 * printing, alerting) live in the CLI command and in private cron wiring.
 */
import type { DarwinExperiment, ExecutionTrace } from '../types.js';
/** The ordered sequence of tool names exactly as they fired. */
export declare function toolSequence(trace: ExecutionTrace): string[];
/** The set of distinct tool names used (order-independent). */
export declare function toolSet(trace: ExecutionTrace): Set<string>;
/**
 * Error rate of a run: (tool-call errors + turn-level errors) normalised by the
 * turn count. Uses `max(1, turnCount)` so a zero-turn trace can't divide by zero.
 */
export declare function errorRate(trace: ExecutionTrace): number;
/**
 * Jaccard similarity of two string sets: `|A∩B| / |A∪B|`.
 * 1 = identical, 0 = disjoint. Two empty sets are defined as 1 (both used no
 * tools → identical behaviour, not drift).
 */
export declare function jaccard(a: Set<string>, b: Set<string>): number;
/**
 * Normalised Levenshtein similarity over two token sequences:
 * `1 - editDistance / max(len)`. 1 = identical order, 0 = fully different.
 * Two empty sequences are defined as 1.
 */
export declare function sequenceSimilarity(a: string[], b: string[]): number;
export interface CanaryThresholds {
    /** Min unordered tool-set Jaccard before flagging drift. Default 0.7. */
    minToolJaccard?: number;
    /** Min ordered tool-sequence similarity before flagging drift. Default 0.6. */
    minSequenceSimilarity?: number;
    /** Max turn-count ratio (candidate/baseline) before flagging drift. Default 1.5. */
    maxTurnRatio?: number;
    /** Max absolute increase in error rate before flagging drift. Default 0.2. */
    maxErrorRateIncrease?: number;
}
export interface TrajectoryComparison {
    /** Unordered tool-set Jaccard [0..1]. */
    toolJaccard: number;
    /** Ordered tool-sequence similarity [0..1]. */
    sequenceSimilarity: number;
    /** candidate.turnCount / max(1, baseline.turnCount). */
    turnRatio: number;
    /** candidate error-rate minus baseline error-rate. */
    errorRateDelta: number;
    /** True if any threshold tripped. */
    drifted: boolean;
    /** Human-readable list of the dimensions that tripped (empty when stable). */
    reasons: string[];
}
/**
 * Compare one candidate trajectory against a baseline across all four
 * dimensions. Pure. Missing thresholds fall back to {@link DEFAULT_THRESHOLDS}.
 */
export declare function compareTrajectory(candidate: ExecutionTrace, baseline: ExecutionTrace, thresholds?: CanaryThresholds): TrajectoryComparison;
export interface CanaryOptions {
    thresholds?: CanaryThresholds;
    /**
     * How many drifted runs in the candidate set trigger an alert. Default 2 —
     * a single failing run is noise; drift is a pattern.
     */
    minDriftedToAlert?: number;
}
export interface CanaryResult {
    /** Number of candidate runs evaluated against the baseline. */
    total: number;
    /** How many candidates drifted on at least one dimension. */
    driftedCount: number;
    /** True when driftedCount >= minDriftedToAlert. */
    alert: boolean;
    /** Median tool-set Jaccard across candidates. */
    medianToolJaccard: number;
    /** Median turn ratio across candidates. */
    medianTurnRatio: number;
    /** Per-candidate comparisons, input order preserved. */
    comparisons: TrajectoryComparison[];
    /** De-duplicated union of every trip reason across drifted candidates. */
    reasons: string[];
}
/**
 * Evaluate a set of candidate trajectories against one baseline. Pure.
 * An empty candidate set yields `total: 0`, `alert: false`.
 */
export declare function evaluateCanary(candidates: ExecutionTrace[], baseline: ExecutionTrace, opts?: CanaryOptions): CanaryResult;
export type CanaryStatus = 'ok' | 'drift' | 'insufficient-data' | 'no-trajectory';
export interface CanaryReport {
    agentName: string;
    /** The prompt version the canary baselined against (the active one), or null. */
    promptVersion: string | null;
    /** Experiment id used as the frozen baseline, or null. */
    baselineId: string | null;
    /** Candidate runs evaluated (excludes the baseline). */
    evaluated: number;
    /** Detailed result, or null when there wasn't enough data to run. */
    result: CanaryResult | null;
    status: CanaryStatus;
    /** One-line operator-facing summary. */
    message: string;
}
export interface RunCanaryOptions extends CanaryOptions {
    /**
     * Minimum candidate runs (excluding the baseline) required before the canary
     * produces a verdict. Below this it reports `insufficient-data`. Default 3.
     */
    minCandidates?: number;
}
/**
 * Run a canary over an agent's stored experiments. Pure: takes the experiments
 * and the active prompt version, returns a report. The caller supplies the data
 * (e.g. `memory.loadExperiments` + `memory.getActivePrompt`).
 *
 * Baseline policy: among experiments carrying a trajectory for the ACTIVE prompt
 * version, the earliest (by `startedAt`) is the frozen reference; every later
 * same-version run is a candidate. Restricting to the active version is what
 * prevents false alarms right after an intentional prompt evolution.
 */
export declare function runCanaryOverExperiments(agentName: string, experiments: DarwinExperiment[], activeVersion: string | null, opts?: RunCanaryOptions): CanaryReport;
//# sourceMappingURL=canary.d.ts.map