import type { AnnotationQueue } from './review.js';
import type { EvaluationScore, Evaluator } from '../types/evaluate.js';
import type { BudgetLedger } from '../types/lifecycle.js';
import type { OperationStore } from '../types/operations.js';
import type { Run, RunQuery, TraceStore } from '../types/tracing.js';
/** Options for `onlineEvaluationWorker()`. */
export interface OnlineEvaluationWorkerOptions {
    /**
     * Names the evaluation. A run is scored once per source; its scores are recorded with it as their
     * `source`, and its operations are of kind `online-evaluation:<source>`.
     */
    source: string;
    /** Where the runs are, and where their scores go. It needs `addFeedback`. */
    traces: TraceStore;
    /** The evaluators each sampled run is scored with. */
    evaluators: Evaluator[];
    /** The durable queue the workers share: any operation store, such as Redis or Postgres. */
    operations: OperationStore;
    /** Which runs to score. Defaults to finished model runs. */
    query?: RunQuery;
    /**
     * The share of runs scored, from 0 to 1. A run is in or out by a hash of its id and the source, so
     * every worker, and every pass after a restart, samples the same runs. Defaults to 1.
     */
    sampleRate?: number;
    /** Runs read per scheduling pass. Defaults to 1,000. */
    scanLimit?: number;
    /** Stops queuing while this many of this source's operations are queued or running. Defaults to 1,000. */
    maxQueued?: number;
    /** Evaluations one worker runs at once. Defaults to 4. */
    concurrency?: number;
    /** How long a worker's claim on an evaluation lasts without a heartbeat. Defaults to 30 seconds. */
    leaseMs?: number;
    /** How long an idle worker waits before looking again. Defaults to 1 second. */
    pollMs?: number;
    /** The spend budget evaluations draw on, such as `budgetLedger()`. */
    budget?: BudgetLedger;
    /** US dollars reserved before each evaluation. Defaults to 0. */
    estimate?: number;
    /** What an evaluation actually cost, reconciled against its reservation. Defaults to the estimate. */
    costOf?: (scores: EvaluationScore[], run: Run) => number;
    /** The tenant the budget charges, when the ledger keys budgets by tenant. */
    tenantId?: string;
    /** Sends an uncertain result to people instead of scoring it automatically. */
    reviewQueue?: AnnotationQueue;
    /** Decides whether a run's scores are uncertain enough to send it to people. */
    reviewWhen?: (scores: EvaluationScore[], run: Run) => boolean;
    /**
     * Gives each evaluator what the run did, read from its trace, as `context.trace`: the run and
     * everything beneath it, with its tool calls, retrievals, model calls, and media. Costs one
     * `tree()` read per evaluation. Defaults to false.
     */
    withTrace?: boolean;
    /**
     * Hears each run's scores once they are written, on the worker that wrote them, such as
     * `automationRules().observe` acting when a score falls. A failure goes to `onError` and never
     * fails the evaluation, whose scores are already written.
     */
    onScored?: (scored: OnlineScored) => Promise<void> | void;
    /** Receives failures of `onScored`. */
    onError?: (error: unknown) => void;
    /** Identifies this worker in leases. Defaults to a generated id. */
    owner?: string;
    /** Replaces the clock, for tests. */
    now?: () => Date;
}
/** One run's scores, as `onScored` hears them. */
export interface OnlineScored {
    /** The run scored. */
    run: Run;
    /** Its scores, from every evaluator. */
    scores: EvaluationScore[];
    /** The source they were recorded under. */
    source: string;
    /** The evaluation operation that wrote them. */
    operationId: string;
}
/** What one scheduling pass did. */
export interface OnlineSchedulePass {
    /** Runs read. */
    scanned: number;
    /** Runs in the sample. */
    sampled: number;
    /** Operations this pass queued. */
    queued: number;
    /** True when the pass stopped because the queue was full. */
    throttled: boolean;
    /** True when the pass read back to the oldest matching run, so the next one starts at the newest. */
    exhausted: boolean;
    /**
     * How the pass paged: by cursor, or by offset on a trace store that does not honour
     * `RunQuery.after`, where runs that start in the same millisecond may be read twice but never skipped.
     */
    paging: 'cursor' | 'offset';
}
/** What a worker has done since it was created. */
export interface OnlineEvaluationWorkerReport {
    /** Runs this worker queued. */
    queued: number;
    /** Runs this worker scored. */
    scored: number;
    /** Operations it ran whose scores were already written, as after a crash. */
    alreadyScored: number;
    /** Operations whose run was gone. */
    missing: number;
    /** Evaluations that failed and will be tried again, or were dead-lettered. */
    failed: number;
    /** Why the worker last stopped: the queue ran dry, the budget ran out, or it was told to stop. */
    stoppedBy?: 'idle' | 'budget' | 'stopped';
}
/** A worker of an online evaluation. Run several, on as many processes, over one operation store. */
export interface OnlineEvaluationWorker {
    /** Identifies this worker in leases. */
    readonly id: string;
    /** Samples runs and queues what is not queued yet, until the queue is full. */
    schedule(): Promise<OnlineSchedulePass>;
    /**
     * Scores one batch: takes back evaluations whose worker stopped, claims up to `concurrency`
     * queued ones, and waits for them. Resolves to how many it took on.
     */
    work(): Promise<number>;
    /**
     * Schedules and scores until nothing is left to queue or score, or the budget runs out. Waits out
     * the lease of an evaluation a stopped worker held, then takes it back.
     */
    drain(options?: {
        signal?: AbortSignal;
    }): Promise<OnlineEvaluationWorkerReport>;
    /** Drains again and again, waiting `intervalMs` when idle, until `stop()`. */
    start(options?: {
        intervalMs?: number;
    }): void;
    /** Stops `start()` after the batch in flight. Resolves when it has stopped. */
    stop(): Promise<void>;
    /** What this worker has done. */
    report(): OnlineEvaluationWorkerReport;
}
/**
 * A worker of an online evaluation. Any number of them, in any number of processes, share one
 * `operations` store: each samples, queues, and scores, and between them every sampled run is scored
 * exactly once for the source. A worker that crashes leaves its claims to lapse, and another takes
 * them back; a score it had already written is not written again.
 */
export declare function onlineEvaluationWorker(options: OnlineEvaluationWorkerOptions): OnlineEvaluationWorker;