import { type OnlineEvaluationWorker, type OnlineEvaluationWorkerOptions } from '../evaluate/online.js'; import { type Regression, type RegressionMetric, type RunSample } from '../insights/regressions.js'; import type { EvaluationScore, Evaluator } from '../types/evaluate.js'; import type { OperationStore } from '../types/operations.js'; import type { TraceStore } from '../types/tracing.js'; import type { BucketStrategy, DeploymentChange, DeploymentChangeRecord, DeploymentRecord, ReplicaReport, RevisionChoice, RevisionHealthMark, RevisionRequest, RunRecord, ServerAssistant, ServerDeployments, ServerStateStore } from '../types/server.js'; export type { BucketStrategy, DeploymentChange, DeploymentChangeRecord, DeploymentRecord, ReplicaReport, RevisionChoice, RevisionHealthMark, RevisionReason, RevisionRequest, RunRevision, } from '../types/server.js'; /** Options for `Deployments`. */ export interface DeploymentsOptions { /** * Where deployments and replica heartbeats are recorded. Give it the server's own state store, so * every replica — and a studio pointed at the same store — sees one set. Defaults to memory, for one * process. */ state?: ServerStateStore; /** How often a replica heartbeats, in milliseconds. Defaults to 10 seconds. */ heartbeatMs?: number; /** How long a heartbeat counts as fresh, in milliseconds. Defaults to three heartbeats. */ replicaTtlMs?: number; /** * How long a replica reuses a deployment it read when routing, in milliseconds, so starting a run * costs no read. A change made elsewhere reaches every replica within this time; one made through * this object applies here at once. Defaults to 2 seconds. */ cacheMs?: number; /** History entries kept per deployment. Defaults to 50. */ historyLimit?: number; /** * How long this replica gathers the runs it finished before writing them to its rollups, in * milliseconds. Defaults to 1 second. Reading rollups through this object writes its own first. */ rollupMs?: number; /** Receives heartbeat and rollup write failures, which never stop the server. */ onError?: (error: unknown) => void; /** Replaces the system clock, for tests. */ now?: () => Date; } /** Options for `Deployments.assistant()`. */ export interface RevisionedAssistantOptions { /** * The revision that takes the traffic until a deployment is recorded. Defaults to the last one * declared. Once anything is recorded, the stored deployment decides, so a new revision in new code * takes no traffic until it is given some. */ live?: string; /** What the assistant is, reported by the assistants endpoint. */ description?: string; /** * How the deployment places threads, when it is first recorded. Defaults to `hash-v2`, and under * 2.x's defaults, `useDefaults('2.x')`, to `fnv1a-v1`. A deployment already recorded keeps its own; * `Deployments.bucketing()` changes it. */ bucketStrategy?: BucketStrategy; } /** One revision's finished runs, as a replica counts them for one deployment version. */ export interface RevisionRollup { /** Runs counted: succeeded, failed, and cancelled. */ runs: number; /** Runs that succeeded or are awaiting input. */ succeeded: number; /** Runs that failed or expired. */ failed: number; /** Runs that were cancelled, which count neither way. */ cancelled: number; /** * Durations of succeeded and failed runs, as counts by bucket. A bucket's key is its index; each is * 5% wider than the one before, so a percentile read from them is within 2.5% of the true one. */ latency: Record; /** Total recorded cost of succeeded and failed runs, in US dollars. */ cost: number; /** How many of those runs recorded a cost. */ costed: number; } /** What one replica counted under one deployment version, as stored in the state store. */ export interface DeploymentRollup { /** The assistant. */ assistant: string; /** The deployment version the runs were routed under. */ deployment: number; /** The replica that counted them. */ replica: string; /** ISO-8601 time it was last written. */ updatedAt: string; /** The counts, per revision. */ revisions: Record; } /** What one revision's runs did, from `Deployments.stats()`. */ export interface RevisionStats { /** The revision. */ revision: string; /** Runs recorded for it: finished or not from run records, and finished or cancelled from rollups. */ runs: number; /** Runs that succeeded or are awaiting input. */ succeeded: number; /** Runs that failed or expired. */ failed: number; /** Runs that were cancelled, which count neither way. */ cancelled: number; /** Failed runs as a share of the finished ones, or 0 before any finished. */ errorRate: number; /** Median duration of finished runs, in milliseconds. */ p50Ms?: number; /** 95th-percentile duration of finished runs, in milliseconds. */ p95Ms?: number; /** Mean recorded cost of finished runs, in US dollars. */ meanCost?: number; } /** Options for `Deployments.evaluate()`. */ export interface RunEvaluationOptions extends Pick { /** What each run is scored by: evaluators from `nexus-ai-pro/evaluate`, such as an LLM judge. */ evaluators: ReadonlyArray; /** * Where the server's traces are: the store of the `tracer` given to the server. Each evaluator * then sees the run's input as `context.example.inputs`, and what it did as `context.trace`: its * tool calls, retrievals, model calls, and media. Without it, evaluators see the run's output. */ traces?: TraceStore; /** * The operation store evaluations are queued in, one operation per run and source. Share one * between replicas, and a run is scored once between them. Defaults to memory, for one process. */ operations?: OperationStore; /** How long a drain waits for an evaluation another worker holds. Defaults to 50 milliseconds. */ pollMs?: number; /** Only runs created from this time, such as the start of a canary. */ since?: string; /** Only this revision's runs. Defaults to every revision's. */ revision?: string; /** The share of runs scored, from 0 to 1, chosen by run id so every replica picks the same ones. Defaults to 1. */ sampleRate?: number; /** How many recent runs are read. Defaults to 2,000. */ limit?: number; /** Recorded as each score's source; a run already scored by this source is skipped. Defaults to `online-evaluation`. */ source?: string; } /** What `Deployments.evaluate()` did. */ export interface RunEvaluationReport { /** Runs scored. */ evaluated: number; /** Runs passed over because this source had already scored them. */ skipped: number; /** Evaluations that failed, each tried again on a later call. */ failed: number; /** Every score given. */ scores: EvaluationScore[]; } /** One score a run was given, as `Deployments.scores()` reads it. */ export interface RunScore { /** The run. */ runId: string; /** Its assistant. */ assistant: string; /** The revision that served it. */ revision?: string; /** Its tenant. */ tenantId?: string; /** The score's key. */ key: string; /** The score. */ score: number; /** Who gave it: an online evaluation's source, or a person's. */ source?: string; /** The comment that came with it. */ comment?: string; /** ISO-8601 time it was given. */ at: string; /** What the run produced, for a person reviewing it. */ output?: unknown; } /** Options for `Deployments.evaluationWorker()`. */ export interface DeploymentEvaluationWorkerOptions extends RunEvaluationOptions { /** The durable queue every replica's worker shares, such as the Redis or Postgres operation store. */ operations: OperationStore; /** Stops queuing while this many evaluations are queued or running. Defaults to 1,000. */ maxQueued?: number; } /** * Revisions, traffic splits, and replica health for the agent server, recorded in a shared store. * * An assistant with revisions is served through `assistant()`, which the server routes every run * through: a run on a thread keeps the revision the thread started on while that revision takes * traffic, and a new thread or a stateless run is split by a stable hash of its id. Raising a * canary's share therefore only ever moves new threads onto it, and rolling it back moves its * threads home. Every run records the revision, its share, the reason it was chosen, and the * deployment version, so what served a request is always answerable. * * The same object, built on the same store without any assistants, is what a studio or a script uses * to read and change deployments. A change is a short history entry, never a redeploy: the code for * every revision ships in the image, and the split decides which one runs. */ export declare class Deployments implements ServerDeployments { private readonly options; private readonly state; /** What fresh workers carry, per assistant, read at most every `cacheMs`. */ private readonly fleet; /** The change in progress per deployment, which the next one waits for. */ private readonly changing; private readonly now; private readonly registrations; private readonly cache; private heartbeat?; /** Runs finished here and not yet written, per assistant and deployment version. */ private readonly pending; /** This replica's rollups as last written, so a write needs no read. */ private readonly written; private rollupTimer?; private flushing; /** Names this replica's rollups before the server attaches it with its own id. */ private readonly unattached; constructor(options?: DeploymentsOptions); /** * Serves an assistant through its revisions, by id: `{ '2026-09-01': v1, '2026-09-30': v2 }`. * * The result is an assistant like any other, so it goes straight into `createAgentServer()`'s * `assistants`. It can do what its live revision can: resume, report state, and roll back. */ assistant(id: string, revisions: Record, options?: RevisionedAssistantOptions): ServerAssistant; /** Chooses the revision for a run. The server calls it through the assistant; tests may call it directly. */ route(assistant: string, request: RevisionRequest): Promise; /** * Whether changes are atomic across replicas: true when the state store has `putIfVersion()`. * Otherwise, changes on this replica apply in order, but two replicas changing one deployment in * the same instant can lose one change. */ get atomicChanges(): boolean; /** Every deployment: those recorded, and the default of every assistant registered here that has none. */ list(): Promise; /** One assistant's deployment, as recorded, or its default when this process registered it. */ get(assistant: string): Promise; /** A deployment's changes, newest first. */ history(assistant: string): Promise; /** * Applies a change, records it in the history, and returns the deployment after it. * * - `split` sets each revision's share. Shares must add up to at most 1, and the live revision * takes whatever is left. * - `canary` gives one revision a share, from 0 to 1, and the live revision the rest. * - `promote` makes a revision live, with all the traffic. * - `rollback` sends all traffic back to the live revision when a canary has any; otherwise it * makes the previous live revision live again. `to` names the revision instead. * - `bucketing` changes how new threads are placed in the split, keeping the split itself. * * `expectedVersion` applies a change only to the version it was decided on, and throws * `DEPLOYMENT_CONFLICT` otherwise, so two people — or a person and a canary guard — cannot undo each * other without noticing. * * Changes to one deployment through this object apply one at a time, so two made at once on one * replica both land, in order. Across replicas, a state store with `putIfVersion()` makes each * change atomic: a replica that loses a race decides again on the winner's record. Without it, two * replicas changing one deployment in the same instant can lose one change: make changes from one * place, or pass `expectedVersion` and retry a conflict. */ change(assistant: string, change: DeploymentChange & { expectedVersion?: number; }, by?: string): Promise; private applyChange; /** The record a change makes of the stored one, or of the default when none is stored. */ private decide; /** * Marks a revision unhealthy, as an automation rule does when a score falls, recorded in the * deployment's history. A canary marked unhealthy is rolled back at once, unless `rollback` is * false; a canary guard also rolls back any canary marked since it began. The live revision is * only marked: there is nothing to roll it back to without a decision. */ markUnhealthy(assistant: string, revision: string, options: { reason: string; by?: string; rollback?: boolean; }): Promise; /** Clears a revision's unhealthy mark. */ markHealthy(assistant: string, revision: string, options?: { reason?: string; by?: string; }): Promise; /** Sets each revision's share of new runs. */ split(assistant: string, traffic: Record, options?: { by?: string; reason?: string; }): Promise; /** Gives one revision a share of new runs, from 0 to 1, and the live revision the rest. */ canary(assistant: string, revision: string, weight: number, options?: { by?: string; reason?: string; }): Promise; /** Makes a revision live, with all the traffic. */ promote(assistant: string, revision: string, options?: { by?: string; reason?: string; }): Promise; /** Pulls a canary, or undoes the last promotion when there is none; `to` names the revision instead. */ rollback(assistant: string, options?: { to?: string; by?: string; reason?: string; }): Promise; /** * Changes how new threads and stateless runs are placed in the split. A thread already on a revision * that takes traffic stays there; new threads are placed by the new strategy. */ bucketing(assistant: string, strategy: BucketStrategy, options?: { by?: string; reason?: string; }): Promise; /** Replicas whose heartbeat is fresh, newest first. */ replicas(): Promise; /** * Starts heartbeating a replica's report, and records the default deployment of every assistant * registered here that has none yet, so a studio or script can change it before anything else has. * The server calls it from `start()`. */ attach(report: () => ReplicaReport): Promise; /** * Writes the runs this replica counted, stops heartbeating, and removes this replica's report. The * server calls it from `stop()`. */ detach(): Promise; /** The revisions this process registered, and those fresh replicas report, for an assistant. */ revisions(assistant: string): Promise; /** * What each revision's runs did: counts, error rate, latency percentiles, and mean cost. * * With `deployment`, a version, they come from the rollups of every run routed under it, on every * replica: the whole window a canary guard judges, in one read. Percentiles are then within 2.5%. * Runs in flight are counted when they finish. A version no replica counted, such as one served * only by replicas older than 2.5, falls back to run records. * * Otherwise they come from the server's run records in the same store. `since` limits them to runs * created from that time, such as the start of a canary; `limit` bounds how many recent runs are * read, 2,000 by default. */ stats(assistant: string, options?: { since?: string; limit?: number; deployment?: number; }): Promise; /** * Every revision's rollup under one deployment version, summed over the replicas that counted its * runs. This replica's runs not yet written are written first. */ rollups(assistant: string, deployment: number): Promise>; /** * Counts a finished run in this replica's rollup for the deployment version it was routed under. * The server calls it as each run finishes. Counts are written every `rollupMs`, each replica to its * own record, so counting never contends with another replica. */ recordRun(run: RunRecord): void; /** Writes the runs this replica counted and has not written yet. Failures go to `onError` and are retried. */ flushRollups(): Promise; private writeRollups; /** * One revision's finished runs since a time, as samples `compareRuns()` reads: a failed or * expired run is an error, a cancelled one is left out. */ samples(assistant: string, revision: string, since?: string, limit?: number): Promise; /** * Online evaluation of the server's own runs: scores each finished run with evaluators from * `nexus-ai-pro/evaluate`, and records the scores as the run's feedback. A canary guard given * those keys in `feedback` then compares quality between revisions, beside errors, latency, and * cost. * * It runs through an online-evaluation worker, as `evaluationWorker()` does, drained once: each * run is one operation in `operations`, so replicas sharing that store score each run once between * them, within a budget, and send uncertain ones to people. Given the server's `traces`, evaluators * see what each run received and did: its input, tool calls, retrievals, model calls, and media. * * A run is scored once per `source`, so calling this on a schedule never counts a run twice. * Sampling is stable: the same run is always in or out of a sample, on every replica. */ evaluate(assistant: string, options: RunEvaluationOptions): Promise; /** * A worker that scores this assistant's finished runs as they come, for as long as it runs: the * online-evaluation worker of `nexus-ai-pro/evaluate/online`, reading the server's runs instead of * a trace store's. Run one on every replica, over one shared `operations` store, and between them * each sampled run is scored once per source, across restarts, within the budget. * * ```ts * const worker = deployments.evaluationWorker('support', { * evaluators: [helpfulnessJudge], operations, traces: traceStore, sampleRate: 0.2, * }); * worker.start(); * ``` */ evaluationWorker(assistant: string, options: DeploymentEvaluationWorkerOptions): OnlineEvaluationWorker; /** * The scores this assistant's runs were given, by people or by online evaluation, from the most * recent runs: what automation rules read through `deploymentScores()`, so every replica judges the * same scores. Narrowed to one `key`, one `source`, runs created from `since`, and `limit` runs read * (2,000). */ scores(assistant: string, options?: { key?: string; source?: string; since?: string; limit?: number; }): Promise; /** * This assistant's finished runs, as a trace store the online-evaluation worker reads: each run in * scope is a run of kind `agent`, with its input and its trace from `traces`, and feedback written * to it lands on the run record. `skipped` collects the runs passed over as already scored. */ private evaluationRuns; /** The deployment the router uses, reusing a recent read. */ private current; /** The revisions fresh workers that claim queued runs report carrying, read at most every `cacheMs`. */ private workerRevisions; /** The deployment of a registered assistant before anything is recorded: its live revision, all traffic. */ private initial; private knownRevisions; private runs; } /** Options for `watchCanaries()`. */ export interface CanaryGuardOptions { /** The deployments to watch, and where their runs are read. */ deployments: Deployments; /** Only these assistants. Defaults to every deployment with a canary. */ assistants?: readonly string[]; /** How often to judge, in milliseconds. Defaults to a minute. `check()` also judges on demand. */ everyMs?: number; /** Finished runs each side needs before a canary is judged. Defaults to 20. */ minRuns?: number; /** What is compared. Defaults to the error rate and p95 latency. */ metrics?: readonly RegressionMetric[]; /** A rise in p95 latency, relative, that counts. Defaults to 0.25. */ latencyIncrease?: number; /** * The smallest rise in p95 latency, in milliseconds, that counts. Defaults to 50, so a few * milliseconds of noise on fast runs never rolls a canary back. */ minLatencyChangeMs?: number; /** A rise in mean cost, relative, that counts. Defaults to 0.25. */ costIncrease?: number; /** * The shares a canary moves through while it holds up, such as `[0.25, 0.5]`, before it is * promoted. Without steps, a canary that holds up stays where it is until a person promotes it. */ steps?: readonly number[]; /** * What a canary is judged on. * * - `runs`, the default, reads run samples: the server's run records since the split last changed, * the latest 2,000, or those `samples` returns. * - `rollups` reads the counts every replica keeps under the deployment's current version: every run * since the split last changed, however many, in one read. It judges the error rate, p95 latency, * and mean cost, not feedback, so it cannot be combined with `samples` or `feedback`. */ from?: 'runs' | 'rollups'; /** * Where a revision's runs come from. Defaults to the server's run records. Give it runs from a * trace store to judge on feedback scores too, with `feedback` naming the keys. */ samples?: (request: { assistant: string; revision: string; since: string; }) => Promise; /** * Feedback keys whose mean score is compared: the scores `Deployments.evaluate()` records from * online evaluation, or any a person left through the feedback route. A canary that is worse on one * is rolled back like one that fails more. */ feedback?: readonly string[]; /** * Judges each metric as an evaluation comparison does, at this confidence, such as 0.95. A bootstrap * interval must not reach zero, and the change must be at least the metric's minimum effect: * `minErrorRateIncrease`, `latencyIncrease` with `minLatencyChangeMs`, `costIncrease`, and * `minFeedbackDrop`. A real regression rolls back, and noise or a trivial change does not. Without * it, the error rate uses a z-test at 95%, and latency and cost use their margins alone. * * From rollups, the error rate is judged by a z-test at this confidence, with * `minErrorRateIncrease`, and latency and cost by their margins. */ confidence?: number; /** The smallest rise in the error rate, absolute, that counts under `confidence`. Defaults to 0.02. */ minErrorRateIncrease?: number; /** The smallest fall in a feedback score's mean, absolute, that counts under `confidence`. Defaults to 0.05. */ minFeedbackDrop?: number; /** Bootstrap resamples under `confidence`. Defaults to 1,000. */ resamples?: number; /** Recorded as who made a change. Defaults to `guard`. */ by?: string; /** Hears every decision, including holds. */ onDecision?: (decision: CanaryDecision) => void; /** Receives failures, which never stop the guard. */ onError?: (error: unknown) => void; } /** What the guard decided about one canary. */ export interface CanaryDecision { /** The assistant. */ assistant: string; /** The canary revision. */ canary: string; /** The live revision it was compared with. */ live: string; /** * `rollback` when it regressed, `advance` to the next step, `promote` after the last, and `hold` * while either side lacks the runs to judge, or while it holds up without steps. */ action: 'rollback' | 'advance' | 'promote' | 'hold'; /** What got worse, when it rolled back. */ regressions: Regression[]; /** Runs judged on each side. */ samples: { live: number; canary: number; }; /** The canary's share after the decision. */ weight: number; /** The mark it was rolled back on, when it was marked unhealthy rather than measured worse. */ unhealthy?: RevisionHealthMark; } /** A running canary guard. */ export interface CanaryGuard { /** Judges every watched canary now, and acts on what it finds. */ check(): Promise; /** Stops the periodic checks. */ stop(): void; } /** * Watches canaries and acts on what their runs show, using the same statistics as the insights' * regression detection. * * Every check compares each canary with the live revision over the same window — the runs since the * split last changed — so both sides saw the same traffic and time of day. A canary whose error rate, * latency, or cost regressed is rolled back at once, with the regression as the reason in the * deployment's history. One that holds up moves to its next step, and is promoted after the last. * * Changes are made against the version that was judged, so a guard on every replica is safe: the * first to act wins, and the rest find the deployment already moved. */ export declare function watchCanaries(options: CanaryGuardOptions): CanaryGuard; /** * A stable position in [0, 1) for a key, so the same thread always lands in the same place. The * strategy defaults to `fnv1a-v1`, whose positions never change; `hash-v2` spreads sequential keys * evenly too. */ export declare function bucket(key: string, strategy?: BucketStrategy): number;