/**
 * Automation rules: when a score falls below a threshold over enough samples, act. Open review items
 * for people, send a webhook, or mark a deployment's revision unhealthy, which rolls a canary back.
 *
 * Rules read scores from a shared source, such as a deployment's run records, so every replica judges
 * the same scores, and a firing is claimed in a shared store, so it acts once however many replicas
 * see it.
 */
import type { OnlineScored } from './online.js';
import type { Store } from '../types/store.js';
import type { RunQuery, TraceStore } from '../types/tracing.js';
/** One score, as automation rules judge them. */
export interface ScoreSample {
    /** The score's key. */
    key: string;
    /** The score. */
    score: number;
    /** Whether it passed, when its evaluator judged. */
    passed?: boolean;
    /** The run it was given to. */
    runId: string;
    /** ISO-8601 time it was given. */
    at: string;
    /** Who gave it. */
    source?: string;
    /** The assistant that ran. */
    assistant?: string;
    /** The revision that served the run. */
    revision?: string;
    /** The run's tenant. */
    tenantId?: string;
    /** What the run received, for a person reviewing it. */
    inputs?: unknown;
    /** What the run produced. */
    output?: unknown;
}
/** Where rules read scores: any replica reading the same source judges the same scores. */
export interface ScoreSource {
    /** The most recent scores of one key, newest first, at most `limit`, and none older than `since`. */
    samples(query: {
        key: string;
        since?: string;
        limit: number;
    }): Promise<ScoreSample[]> | ScoreSample[];
}
/** The scores of one assistant's runs, from its deployment's run records: shared by every replica. */
export declare function deploymentScores(deployments: {
    scores(assistant: string, options: {
        key?: string;
        since?: string;
        limit?: number;
    }): Promise<Array<Omit<ScoreSample, 'passed' | 'inputs'>>>;
}, assistant: string): ScoreSource;
/**
 * The scores recorded on a trace store's runs, as an online evaluation writes them: shared by every
 * replica that reads the store. `query` narrows the runs, such as `{ kind: 'image' }`.
 */
export declare function traceScores(traces: TraceStore, query?: RunQuery): ScoreSource;
/** Scores held in this process, fed by `observe()` or `record()`: for one process, or a test. */
export interface MemoryScoreSource extends ScoreSource {
    /** Keeps one score. */
    record(sample: ScoreSample): void;
    /** Keeps a run's scores as an online evaluation's `onScored` hears them. */
    observe(scored: OnlineScored): void;
}
/** A score source in this process, keeping the latest `max` scores (10,000). */
export declare function memoryScores(options?: {
    max?: number;
}): MemoryScoreSource;
/** What a rule judges a scope's samples by. */
export type RuleStatistic = 'mean' | 'pass-rate';
/** A rule: when `key` falls below `below` over enough samples, act. */
export interface AutomationRule {
    /** Names the rule in events, in the shared state, and as `rule:<name>` in a deployment's history. */
    name: string;
    /** The score key judged, such as `banner` from a media judge. */
    key: string;
    /** Fires when the statistic is below this. */
    below: number;
    /** The mean score, the default, or the share that passed: `passed`, or a score of at least 0.5. */
    statistic?: RuleStatistic;
    /** Samples a scope needs before it is judged, each run counted once. Defaults to 20. */
    minSamples?: number;
    /** The most recent runs judged per scope. Defaults to 100. */
    window?: number;
    /** Only samples this recent, in milliseconds. */
    windowMs?: number;
    /**
     * Judges by an interval instead of the bare statistic: fires only when its upper bound at this
     * confidence, such as 0.95, is below the threshold, so a few unlucky samples never fire it.
     */
    confidence?: number;
    /** Only scores from this source. */
    source?: string;
    /** Narrows the samples further. */
    where?: (sample: ScoreSample) => boolean;
    /**
     * What is judged on its own: each revision of each assistant, the default, so a canary is judged
     * apart from the live revision; each assistant; everything as one; or a scope of your own.
     */
    by?: 'revision' | 'assistant' | 'all' | ((sample: ScoreSample) => string);
    /** What it does when it fires, in order. */
    actions: AutomationAction[];
    /**
     * Fires again after this long while a scope stays below, in milliseconds. Defaults to never: once
     * per fall, again only after it recovers, which takes `minSamples` runs back at the threshold or
     * above.
     */
    repeatAfterMs?: number;
}
/** A rule that fired, or recovered, for one scope. */
export interface AutomationEvent {
    /** `fired` when it fell below, `recovered` when it came back. */
    type: 'fired' | 'recovered';
    /** The rule. */
    rule: string;
    /** The scope, such as `designer@v2`. */
    scope: string;
    /** The score key. */
    key: string;
    /** How it was judged. */
    statistic: RuleStatistic;
    /** The statistic's value. */
    value: number;
    /** The interval at the rule's confidence, when it has one. */
    interval?: [number, number];
    /** The threshold. */
    below: number;
    /** Runs judged. */
    samples: number;
    /** The assistant, when the scope has one. */
    assistant?: string;
    /** The revision, when the scope has one. */
    revision?: string;
    /** The lowest samples, at most five, so an event points at runs to read. */
    worst: ScoreSample[];
    /** ISO-8601 time it was judged. */
    at: string;
    /** A sentence saying what happened. */
    summary: string;
}
/** What a rule does when it fires. A failure goes to `onError`, and the rule's other actions still run. */
export type AutomationAction = (event: AutomationEvent) => Promise<void> | void;
/** Where a rule judged a scope, from `AutomationRules.status()`. */
export interface AutomationStatus {
    /** The rule. */
    rule: string;
    /** The scope. */
    scope: string;
    /** The statistic's value. */
    value: number;
    /** Runs judged. */
    samples: number;
    /** Whether it is below with enough samples. */
    breached: boolean;
    /** ISO-8601 time it last fired for this scope, while it stays below. */
    firingSince?: string;
}
/** Options for `automationRules()`. */
export interface AutomationRulesOptions {
    /** The rules. */
    rules: AutomationRule[];
    /** Where scores are read. Defaults to `memoryScores()`, fed by `observe()`. */
    scores?: ScoreSource;
    /**
     * Where firings are claimed, shared by every replica, so a rule acts once for a fall however many
     * replicas judge it: a long-term store with `putIfVersion()`. Defaults to memory, for one process.
     */
    state?: Store;
    /** The namespace firings are kept under. Defaults to `['nexus', 'automation']`. */
    namespace?: readonly string[];
    /** Most recent samples read per rule, across its scopes. Defaults to 5,000. */
    scanLimit?: number;
    /** Hears every event. */
    onEvent?: (event: AutomationEvent) => void;
    /** Hears failures of actions and reads, which never stop the rules. */
    onError?: (error: unknown, event?: AutomationEvent) => void;
    /** Replaces the system clock, for tests. */
    now?: () => Date;
}
/** Running automation rules. */
export interface AutomationRules {
    /** Judges every rule now, acting on what fell, and resolves to the events. */
    check(): Promise<AutomationEvent[]>;
    /**
     * Hears a run's scores, as an online evaluation's `onScored`: kept by a memory source, and judged
     * at once by the rules on their keys.
     */
    observe(scored: OnlineScored): Promise<void>;
    /** Where each rule stands for each scope. */
    status(): Promise<AutomationStatus[]>;
    /** Judges every `everyMs` (one minute) until `stop()`. */
    start(options?: {
        everyMs?: number;
    }): void;
    /** Stops `start()`. */
    stop(): void;
}
/**
 * Automation rules over scores: each rule judges the most recent runs of each scope (each revision by
 * default) and acts when the statistic falls below its threshold over at least `minSamples` runs.
 * A firing is claimed in `state` before it acts, so it acts once for a fall across every replica,
 * and again only after the scope recovers, or after `repeatAfterMs`.
 *
 * ```ts
 * const rules = automationRules({
 *   scores: deploymentScores(deployments, 'designer'),
 *   state: store,
 *   rules: [{ name: 'banner quality', key: 'banner', below: 0.6, minSamples: 20,
 *             actions: [markUnhealthy({ deployments }), openReview(queue), sendWebhook({ url })] }],
 * });
 * const worker = deployments.evaluationWorker('designer', { evaluators, operations, onScored: rules.observe });
 * ```
 */
export declare function automationRules(options: AutomationRulesOptions): AutomationRules;
/** Anything review items are opened in, such as an `AnnotationQueue`. */
export interface ReviewOpener {
    /** Opens one item. */
    enqueue(subject: {
        inputs?: unknown;
        output?: unknown;
        runId?: string;
    }, metadata?: Record<string, unknown>): unknown;
}
/** Opens a review item for each of the lowest-scoring runs, at most `limit` (5), when a rule fires. */
export declare function openReview(queue: ReviewOpener, options?: {
    limit?: number;
}): AutomationAction;
/** Options for `sendWebhook()`. */
export interface AutomationWebhookOptions {
    /** Where the event is posted, as JSON. */
    url: string;
    /** Headers on the request, such as an authorization. */
    headers?: Record<string, string>;
    /**
     * Signs the body with HMAC-SHA256 under this secret, in `x-nexus-signature: sha256=<hex>`, so the
     * receiver knows it came from here.
     */
    secret?: string;
    /** Replaces the global `fetch`. */
    fetch?: typeof globalThis.fetch;
}
/** Posts the event to a webhook when a rule fires. A refusal rejects, which `onError` hears. */
export declare function sendWebhook(options: AutomationWebhookOptions): AutomationAction;
/** Marks the revision a rule fired for unhealthy, which rolls back a canary, as `Deployments.markUnhealthy()` does. */
export declare function markUnhealthy(options: {
    deployments: {
        markUnhealthy(assistant: string, revision: string, options: {
            reason: string;
            by?: string;
            rollback?: boolean;
        }): Promise<unknown>;
    };
    /** Rolls a canary back at once. Defaults to true; false leaves it to a canary guard. */
    rollback?: boolean;
}): AutomationAction;