import type { OperationRunnerConfig } from '../types/operations.js'; import type { Tracer } from '../tracing/tracer.js'; import type { ResidencyPolicy } from '../types/residency.js'; import type { RunFeedback } from '../types/tracing.js'; import type { Principal, ReplicaReport, RunEventLog, RunRecord, ScalingSnapshot, ServerAssistant, ServerDeployments, ServerStateStore, TenantGate, ThreadBusyPolicy, ThreadRecord } from '../types/server.js'; import { type ListPage } from './list-page.js'; import { ServerMetrics } from './metrics.js'; /** What a run needs to start. */ export interface StartRunOptions { /** The assistant to run. */ assistant: string; /** What it runs on. */ input?: unknown; /** The thread it belongs to. Without one the run is stateless. */ threadId?: string; /** Answers an interrupt instead of starting a new input. */ resume?: unknown; /** Who asked. */ principal?: Principal; /** Replays the existing run when a run with this key was already accepted. */ idempotencyKey?: string; /** Application data recorded on the run. */ metadata?: Record; /** What to do when the thread is already running something. Defaults to the server's policy. */ onBusy?: ThreadBusyPolicy; /** Runs this revision of the assistant, instead of letting the traffic split choose. */ revision?: string; } /** * Runs in a queue that any replica's workers claim, instead of on the replica that accepted them. * * With a queue, a replica runs at most `concurrency` runs at once. It starts a new run itself while * it has room, and leaves the rest queued in the operation store for whichever worker frees up * first. That queue is what an autoscaler watches: `GET /scaling` and `GET /metrics` report it. */ export interface RunQueueOptions { /** Runs this replica executes at once. Defaults to 10. */ concurrency?: number; /** * Whether this replica claims queued runs. `false` makes it an API replica that only accepts them, * in front of a separate pool of workers. Defaults to true. */ claim?: boolean; /** How often an idle worker looks for queued runs, in milliseconds. Defaults to 1 second. */ pollMs?: number; } /** Options for the run manager. */ export interface RunManagerOptions { /** The assistants that can be run, by id. */ assistants: Record; /** Where threads, runs, and cron jobs are recorded. Defaults to memory. */ state?: ServerStateStore; /** Where run events are kept for resumable streaming. Defaults to memory. */ events?: RunEventLog; /** Configures the operation runner underneath: store, dispatcher, retries, leases, webhooks. */ operations?: OperationRunnerConfig; /** Runs in a shared queue with a per-replica cap. Without it, a run executes where it was accepted. */ queue?: RunQueueOptions; /** * How often this replica takes over runs whose worker stopped, in milliseconds. Defaults to 30 * seconds with a queue. Without a queue it is off unless set, and only `start()` recovers. */ recoverEveryMs?: number; /** Per-tenant limits, enforced when a run is accepted. See `tenantLimits()` in `nexus-ai-pro/server/tenancy`. */ tenants?: TenantGate; /** The deployments registry, which counts each finished run in this replica's rollups. */ deployments?: Pick; /** * Traces every run: each attempt is the root of a trace, named after the assistant, holding its * input and output, and everything the assistant does through the same tracer nests beneath it, * such as a model call through `traceModelClient()`. The run records the trace's id as `traceId`, * which `Deployments.evaluate()` reads. Give it the tracer the assistants' clients use. */ tracer?: Pick; /** * The region this server runs in. Its threads and runs record it, and their ids start with it, as * `eu-west~thread-…`, so `regionalGateway()` sends every later request for one to the region that * holds its data. */ region?: string; /** * Which regions each tenant may reach: a thread or run for a tenant that may not reach this * server's region is refused with a `ResidencyError`, as a 403, and nothing is recorded. */ residency?: ResidencyPolicy; /** What to do when a thread is already running something. Defaults to `reject`. */ onBusy?: ThreadBusyPolicy; /** How long a run may take before it expires, in milliseconds. */ runTimeoutMs?: number; /** How long `enqueue` waits for the run in flight, in milliseconds. Defaults to 30 seconds. */ queueTimeoutMs?: number; /** Receives errors that must not fail a request, such as a failed event append. */ onError?: (error: unknown, context: { runId?: string; threadId?: string; }) => void; /** Replaces the system clock, for tests. */ now?: () => Date; } /** What `drain()` did with the runs this replica was executing. */ export interface DrainResult { /** Runs that finished before the timeout. */ finished: string[]; /** Runs handed back to the queue for another worker, because the timeout passed first. */ released: string[]; } /** * Runs assistants, records threads and runs, and keeps the event log a client streams from. * * Every run is a durable operation, so a run that outlives the request that started it, a worker * that dies mid-run, a duplicate submission, and a cancellation from another replica are all handled * by the operation runner rather than by anything here. What this adds is the thread: which run owns * it, what happens when a second arrives, and where the events go. With a `queue`, it also decides * where a run executes: here while there is room, otherwise on the next free worker. */ export declare class RunManager { private readonly options; private readonly runner; private readonly store; /** Handles of runs this worker is executing, so a cancellation aborts them at once. */ private readonly local; /** Runs holding one of this replica's execution slots. */ private readonly slots; /** The drain switch of each run this replica is executing, so `drain()` can reach graph runs. */ private readonly controls; /** Runs handed back to the queue during a drain, as each graph run stopped at a boundary. */ private readonly drainedRuns; private readonly state; private readonly events; private readonly now; private readonly startedAt; /** This replica's run counters and latency histograms, rendered by `GET /metrics`. */ readonly metrics: ServerMetrics; private claiming; private draining; private working; private ticking; private timer?; private lastRecovery; constructor(options: RunManagerOptions); /** The event log, which the event-stream route reads. */ get eventLog(): RunEventLog; /** The id this worker writes into operation leases. */ get workerId(): string; /** The assistants that can be run. */ get assistants(): Record; /** Runs this replica is executing now. */ get inFlight(): number; /** Runs this replica executes at once, when it runs a queue. */ get capacity(): number | undefined; /** Whether this replica is draining, which `GET /health` reports so a load balancer stops sending. */ get isDraining(): boolean; /** Looks an assistant up, or refuses the request. */ assistant(id: string): ServerAssistant; /** Creates a thread for an assistant. */ createThread(options: { assistant: string; threadId?: string; principal?: Principal; metadata?: Record; }): Promise; /** A new id, starting with this server's region when it has one, so a gateway can route by it. */ private regionalId; /** Refuses work for a tenant that may not reach this server's region. */ private checkResidency; /** The region this server runs in, when it has one. */ get region(): string | undefined; /** Reads a thread, refusing one that belongs to another tenant. */ thread(threadId: string, principal?: Principal): Promise; /** Threads of a tenant, newest written first. */ threads(principal?: Principal, limit?: number): Promise; /** * A page of a tenant's threads, newest written first, 50 unless `limit` says otherwise. `cursor` is * a page's `nextCursor`, and asks for the page after it. */ listThreads(principal?: Principal, options?: { limit?: number; cursor?: string; }): Promise>; /** * Reads a namespace, newest written first, until a page of the records `keep` takes is full or the * namespace ends, so another tenant's newer records never crowd a tenant's page out. A page deep in * a long list reads every record before it. */ private page; /** Deletes a thread's record. The assistant's own checkpoints are left alone. */ deleteThread(threadId: string, principal?: Principal): Promise; /** The assistant's state for a thread, from the revision the thread last ran on. */ threadState(threadId: string, principal?: Principal): Promise; /** Reads a run, refusing one that belongs to another tenant. */ run(runId: string, principal?: Principal): Promise; /** Runs of a tenant, newest written first, optionally for one thread. */ runs(principal?: Principal, options?: { threadId?: string; limit?: number; }): Promise; /** * A page of a tenant's runs, newest written first, optionally for one thread, 50 unless `limit` says * otherwise. `cursor` is a page's `nextCursor`, and asks for the page after it. */ listRuns(principal?: Principal, options?: { threadId?: string; limit?: number; cursor?: string; }): Promise>; /** * Accepts a run and starts it in the background. * * Returns as soon as the run is recorded, so the caller can stream its events or hand back its id. * A thread already running something is handled by the busy policy before anything is submitted, * the assistant's traffic split picks the revision, and the tenant's limits admit the run or refuse * it with a `TenantLimitError`. */ start(options: StartRunOptions): Promise; /** The rest of `start()`, once the thread, if any, is this run's. */ private startClaimed; /** * Leaves feedback on a run: a person's rating, or an online evaluator's score. A canary guard * compares it between revisions as their quality. Another tenant's run is not found, and a run * keeps its latest 100 pieces of feedback. */ addFeedback(runId: string, feedback: Omit & { createdAt?: string; }, principal?: Principal): Promise; /** Cancels a run, including one another replica is executing. */ cancel(runId: string, principal?: Principal, reason?: string): Promise; /** * Re-runs whatever this worker can claim from the store, after a restart or another worker's * crash. The operation runner decides what is claimable; this only supplies the executor. */ recover(limit?: number): Promise; /** * Claims queued runs up to this replica's free capacity and starts them. The worker loop calls it; * a test or a custom scheduler can call it directly. Resolves to the ids it claimed. */ claim(): Promise; /** * The kinds of run this replica can execute: each assistant's runs without a revision, and those * of every revision it carries. A run routed to a revision this image lacks is left for a worker * that has it. */ private claimableKinds; /** * Starts the worker loop: claiming queued runs while there is room, and taking over runs whose * worker stopped. Does nothing without a queue or `recoverEveryMs`. `stopWorking()` ends it. */ startWorking(): void; /** Stops the worker loop. Runs in flight carry on. */ stopWorking(): void; /** * Stops taking new work and waits for the runs in flight, before this replica shuts down. * * New runs accepted here go to the queue instead of starting, and the worker loop stops claiming. * Runs still going when `timeoutMs` passes are handed back to the queue for another worker to * continue, so a scale-down neither waits for a long run nor loses it. Without a queue nothing can * claim a hand-off, so they are left to their leases, which another replica's recovery takes over. */ drain(options?: { timeoutMs?: number; }): Promise; /** This replica, as it reports itself to heartbeats and the scaling endpoint. */ replica(metadata?: Record): ReplicaReport; /** The queue as the operation store counts it, across every replica, with this replica's report. */ scaling(metadata?: Record): Promise; /** Every metric in Prometheus text: the deployment-wide queue, this replica, and its run counters. */ prometheus(extra?: (lines: string[]) => void | Promise): Promise; /** Records a run and submits it: to run here while there is room, or to the queue otherwise. */ private accept; /** Runs a run from what its record and submission say: a claimed queued run, or a recovered one. */ private executeStored; private execute; /** The assistant for one revision, or the assistant itself when it has none by that id. */ private revisionOf; /** Whether a new run starts on this replica rather than waiting in the queue. */ private runsHere; private freeSlots; /** Keeps a run's handle while it executes here, and frees its slot when it settles. */ private track; private freeSlot; private schedule; private tick; /** * Applies the busy policy, returning the thread once this run holds it. The thread is claimed * atomically, so of several replicas starting runs on one thread at once, exactly one proceeds and * the rest meet the policy; a thread still busy after the policy has run its course is refused. */ private settleBusyThread; /** * Claims a thread for a run. The claim is a record in the operation store under a unique * idempotency key, moved from run to run by compare-and-set, so two replicas cannot both take a free * thread: creating it races on the key, and taking it over races on the sequence. Resolves * undefined once this run holds the thread, or the id of the unfinished run that does. * * A store without `findByIdempotencyKey` falls back to the thread record's `activeRunId`, which is * what every release before 2.2 used, and which two replicas racing can both see empty. */ private claimThread; /** * Whether a thread's holder no longer needs it: its run finished, or it was never recorded and its * claim is old enough that the replica starting it must have died. */ private runIsFree; /** Frees a thread this run holds. Losing the compare-and-set means another run took it over, which is fine. */ private releaseClaim; /** Waits for a run to finish, for the `enqueue` policy. */ private waitForRun; private releaseThread; private releaseTenant; private countFinished; private record; private settle; private report; }