import type { ContextBundle, ContextBundleDefinition, ContextHub } from '../context-hub/hub.js'; import type { CompareOptions, MetricComparison } from '../evaluate/compare.js'; import type { PromptRegistry } from '../prompts/registry.js'; import type { Experiment } from '../types/evaluate.js'; import type { CompletionRequest } from '../types/messages.js'; import type { PromptDefinition, PromptVersion } from '../types/prompts.js'; import type { Run } from '../types/tracing.js'; import type { Issue } from './clusters.js'; /** What a fix changes: a prompt in a registry, or a context bundle in a hub, served by a label. */ export type FixSubject = { kind: 'prompt'; registry: PromptRegistry; name: string; label: string; } | { kind: 'context'; hub: ContextHub; name: string; label: string; }; /** What a fix proposer is given: the issue, the version the label serves, and some runs that show it. */ export interface FixRequest { /** The issue to fix. */ issue: Issue; /** Whether a prompt or a bundle is changed, and which. */ kind: 'prompt' | 'context'; /** The version the label serves now. */ current: PromptVersion | ContextBundle; /** Up to five of the issue's runs. */ examples: Run[]; } /** A proposed new definition, and why it should fix the issue. */ export interface FixCandidate { /** The new prompt or bundle definition, under the same name. */ definition: PromptDefinition | ContextBundleDefinition; /** Why it should help, shown to the person who decides. */ rationale?: string; } /** Proposes a fix, or `undefined` when it has none: `modelFixProposer()`, or your own code. */ export type FixProposer = (request: FixRequest) => Promise | FixCandidate | undefined; /** Opens a pull request. Inject a client for your forge — GitHub, GitLab, or another — so none is a dependency. */ export interface PullRequestClient { /** Opens a pull request that writes `files` on a new `branch`, and returns its address. */ createPullRequest(request: { title: string; body: string; branch: string; base?: string; files: Record; }): Promise<{ url: string; }>; } /** A proposed fix: the candidate, how it scored against the current version, and what a person decided. */ export interface FixProposal { /** The proposal's id. */ id: string; /** The issue it answers. */ issueId: string; /** The issue's summary, as it read when the fix was proposed. */ issue: string; /** Whether a prompt or a bundle is changed. */ kind: 'prompt' | 'context'; /** The prompt or bundle. */ name: string; /** The label the fix would be promoted to. */ label: string; /** The version the label served, and the experiment that scored it. */ baseline: { version: string; experiment: string; }; /** The proposed version, committed without a label, and the experiment that scored it. */ candidate: { version: string; experiment: string; }; /** Why the proposer thinks it helps. */ rationale?: string; /** `improved` when a metric got better and none got worse, beyond noise. */ verdict: 'improved' | 'unchanged' | 'regressed'; /** Every metric both experiments scored, with its verdict. */ metrics: MetricComparison[]; /** `pending` waits for a person; `discarded` did not improve; `promoted` and `rejected` were decided. */ status: 'pending' | 'promoted' | 'rejected' | 'discarded'; /** ISO-8601 time it was proposed. */ createdAt: string; /** ISO-8601 time it was decided. */ decidedAt?: string; /** Who decided. */ decidedBy?: string; /** Why, as the person who decided wrote it. */ note?: string; /** The pull request opened for it, when one was. */ pullRequest?: { url: string; }; } /** Where proposals are kept. */ export interface ProposalStore { /** Stores a proposal, replacing one with the same id. */ save(proposal: FixProposal): Promise | void; /** Reads a proposal. */ get(id: string): Promise | FixProposal | undefined; /** Proposals, newest first, optionally with one status. */ list(filter?: { status?: FixProposal['status']; limit?: number; }): Promise | FixProposal[]; } /** Proposals in process memory. */ export declare class MemoryProposalStore implements ProposalStore { private readonly proposals; /** Stores a copy of a proposal. */ save(proposal: FixProposal): void; /** A copy of a proposal. */ get(id: string): FixProposal | undefined; /** Proposals, newest first. */ list(filter?: { status?: FixProposal['status']; limit?: number; }): FixProposal[]; } /** Proposals as one JSON file each in a directory, so a shared studio keeps them across restarts. */ export declare class FileProposalStore implements ProposalStore { private readonly directory; constructor(directory: string); /** Writes a proposal through a temporary file and a rename. */ save(proposal: FixProposal): Promise; /** Reads a proposal. */ get(id: string): Promise; /** Proposals, newest first. */ list(filter?: { status?: FixProposal['status']; limit?: number; }): Promise; private fileOf; } /** Options for `proposeFix()`. */ export interface ProposeFixOptions { /** The issue to fix. */ issue: Issue; /** The prompt or bundle to change, and the label that serves it. */ subject: FixSubject; /** Proposes the new definition. */ propose: FixProposer; /** * Scores a version over your dataset: `evaluatePrompt()` or `evaluateContext()` with your client * and evaluators. Store the experiments, so the promotion's experiment gate finds the candidate's. */ evaluate: (version: PromptVersion | ContextBundle) => Promise; /** Passed to `compareExperiments()`. */ compare?: CompareOptions; /** Where the proposal is kept, for the studio's inbox. */ store?: ProposalStore; /** Opens a pull request for an improving fix, with the files `files` writes — by default the candidate as JSON. */ pullRequest?: { client: PullRequestClient; base?: string; files?: (proposal: FixProposal, candidate: PromptVersion | ContextBundle) => Record; }; /** Recorded as the candidate's author. Defaults to `insights`. */ by?: string; /** Replaces the system clock, for tests. */ now?: () => Date; } /** * Proposes a fix for an issue and measures it before anyone is asked: the proposer writes a new * definition, it is committed without a label, the current version and the candidate are both scored * over your dataset, and the comparison decides. An improvement is kept as a `pending` proposal for a * person to promote — through the label's gates, as any promotion — and anything else is `discarded`, * with the evidence. Resolves `undefined` when the proposer had nothing, or proposed what is served. */ export declare function proposeFix(options: ProposeFixOptions): Promise; /** Options for a `ProposalInbox`. */ export interface ProposalInboxOptions { /** Where proposals are kept. */ store: ProposalStore; /** The prompt registry prompt fixes are promoted in. */ prompts?: PromptRegistry; /** The context hub bundle fixes are promoted in. */ contexts?: ContextHub; /** Replaces the system clock, for tests. */ now?: () => Date; } /** * The proposals waiting for a person, and the two decisions they can make. Promoting goes through the * label's gates like any promotion; the studio's inbox is this, behind roles. */ export declare class ProposalInbox { private readonly options; constructor(options: ProposalInboxOptions); /** Proposals, newest first, optionally with one status. Defaults to `pending`. */ list(status?: FixProposal['status'] | 'all'): Promise; /** One proposal. */ get(id: string): Promise; /** Promotes a pending proposal's candidate to its label, through the label's gates, and records who did. */ promote(id: string, options: { by: string; note?: string; }): Promise; /** Rejects a pending proposal. The candidate version stays committed, unlabelled. */ reject(id: string, options: { by: string; note?: string; }): Promise; private pending; private decide; } /** A client that runs one completion — a `NexusAI` instance, or anything with the same `complete()`. */ export interface FixModelClient { /** Runs one completion and returns its text. */ complete(request: CompletionRequest): Promise<{ content: string; }>; } /** Options for `modelFixProposer()`. */ export interface ModelFixProposerOptions { /** The model that writes fixes. */ model: string; /** Characters of each example run's inputs and error shown to the model. Defaults to 800. */ maxExampleCharacters?: number; } /** * A fix proposer over a chat model: it is shown the issue, the current prompt's messages or the * bundle's instructions, and example runs, and asked for replacements as JSON. A reply it cannot read * proposes nothing. Every proposal is still evaluated before anyone sees it. */ export declare function modelFixProposer(client: FixModelClient, options: ModelFixProposerOptions): FixProposer;