/** * Public API type declarations for @browserstack/ai-sdk * * This is the ONLY types file that TypeScript consumers see. * Only symbols declared here appear in IDE autocomplete. * Everything else is hidden from suggestions but still callable at runtime. * * @module @browserstack/ai-sdk */ // --------------------------------------------------------------------------- // Enums // --------------------------------------------------------------------------- declare enum TraceAttribute { SESSION_ID = "session.id", USER_ID = "user.id", TRACE_INPUT = "AIOPS_INTERNAL.trace.input", TRACE_OUTPUT = "AIOPS_INTERNAL.trace.output", TRACE_RETRIEVED_CONTEXTS = "AIOPS_INTERNAL.trace.retrieved_contexts", } declare enum ExperimentRunStatus { PENDING = "PENDING", RUNNING = "RUNNING", COMPLETED = "COMPLETED", FAILED = "FAILED", CANCELLED = "CANCELLED", } declare enum EvaluatorParamDataType { STRING = "string", INTEGER = "integer", FLOAT = "float", BOOLEAN = "boolean", STRING_ARRAY = "string[]", INTEGER_ARRAY = "integer[]", FLOAT_ARRAY = "float[]", BOOLEAN_ARRAY = "boolean[]", OBJECT = "object", } declare enum ComparisonDeltaDirection { IMPROVEMENT = "improvement", REGRESSION = "regression", NEUTRAL = "neutral", } declare enum ComparisonThresholdStatus { PASS = "pass", FAIL = "fail", MISSING_SCORE = "missing_score", } declare enum ComparisonStatus { READY = "ready", PENDING = "pending", NO_BASELINE = "no_baseline", } declare enum ComparisonVerdict { PASS = "PASS", REGRESSION = "REGRESSION", } // --------------------------------------------------------------------------- // Comparison Verdict types (AIOPS-2424) // --------------------------------------------------------------------------- /** * Status of a stored comparison verdict between two experiment runs. * * NOT_GENERATED — no verdict has been computed yet. * READY — verdict is current. * STALE — new scores arrived since the verdict was generated. * DECLINED — the engine had insufficient data on last generation. */ export type ComparisonVerdictStatus = | "NOT_GENERATED" | "READY" | "STALE" | "DECLINED"; /** * Response from GET /api/public/experiment-comparisons/verdict. * * status=NOT_GENERATED is a normal 200 response, not an error. * Callers may poll until the status transitions to READY or DECLINED. */ export interface ComparisonVerdictResponse { status: ComparisonVerdictStatus; baseRunId: string; targetRunId: string; /** "BASE", "TARGET", or "TIE" — null when NOT_GENERATED or DECLINED. */ winner: string | null; /** Confidence level: "HIGH" | "MEDIUM" | "LOW", null when not available. */ confidence: string | null; /** Quality-bucket outcome: "BASE" | "TARGET" | "TIE", null when not available. */ qualityCall: string | null; /** Efficiency-bucket outcome: "BASE" | "TARGET" | "TIE", null when not available. */ efficiencyCall: string | null; /** Human-readable explanation, null when not yet generated. */ reasoning: string | null; /** Per-evaluator driver breakdown, null when not yet generated. */ drivers: unknown | null; /** ISO timestamp when scores were computed, null when NOT_GENERATED. */ generatedAt: string | null; } // --------------------------------------------------------------------------- // Config // --------------------------------------------------------------------------- /** * Customer-supplied mask function. Receives the value of a PII-bearing field * (input / output / metadata on traces and spans; PII-bearing OTel attributes * for auto-tracing) and must return the masked replacement. * * The SDK does NOT walk or transform the value before/after calling this — the * customer owns recursion, regex sets, and any external PII-detection library. * * Must run synchronously. If it throws, the SDK replaces the value with * `"[MASKED_ERROR]"` (fail-closed) so a buggy mask cannot leak data. */ export type CustomScrubFn = (data: unknown) => unknown; // --------------------------------------------------------------------------- // Guardrails (init-level binding) // --------------------------------------------------------------------------- /** * Thrown to the caller when a guardrail with action `error_4xx` blocks a call, * when a blocking action on a streaming request has no provider-shaped * substitute to synthesize, or when a `reask` rule's retry actually ran and * still failed: a retry reply that trips the same rule again throws this * (action `'reask'`, carrying the rule's resolved authored text) — a failure of * the retry's own call propagates the provider's error instead, not this. Catch * it to handle blocked calls: `if (e instanceof GuardrailBlockedError) ...` */ /** HTTP 422 Unprocessable Entity — the fixed 4xx marker a guardrail block * surfaces as (named so it is not a bare literal). */ export type GuardrailBlockedStatus = 422; export declare class GuardrailBlockedError extends Error { readonly name: 'GuardrailBlockedError'; /** 4xx-class marker so HTTP-ish error handlers classify it as a client error. */ readonly status: GuardrailBlockedStatus; readonly guardrailName: string; readonly action: 'default_response' | 'redact' | 'error_4xx' | 'reask'; /** Substitute body to return in place of the blocked call (manual-enforce). */ readonly defaultResponse?: string; /** * The guarded input as it stood when the BLOCKING rule judged it — i.e. after any * earlier redact in the same chain. The shape is seam-local: a rewritten request * for the model seam, a rewritten arguments value for the tool seam. Absent means * no rewrite occurred, so the original is what was judged. * * Note for error handlers: on a plain block this carries the ORIGINAL guarded * input, so treat it as sensitive — do not forward it to a log sink or error * tracker without redacting it yourself. */ readonly judgedValue?: unknown; /** * A redact did rewrite the text before the block, but the rewrite could not be * expressed in the seam's own shape. `judgedValue` is then absent, because the * original still contains what the redact removed. */ readonly judgedRewriteUnavailable?: boolean; constructor( guardrailName: string, action: 'default_response' | 'redact' | 'error_4xx' | 'reask', defaultResponse?: string, judgedValue?: unknown, judgedRewriteUnavailable?: boolean, ); } /** Guardrails init config (grouped under `AISDKConfig.guardrails`). Every field * is optional and has an `AISDK_GUARDRAILS_*` env fallback. */ export interface GuardrailsConfig { /** * Guardrail binding — names of LIVE guardrails enforced on every auto-traced * model call. A `string[]` binds globally; the agent-wise map * `{ : [...], "*": [...global] }` binds per agent (the seam applies * the active agent's set ∪ global). Rules are fetched by name (60s cache, * ETag) and enforced in-process; any failure fails open. */ rules?: string[] | Record; /** Trace environment sent on enforce requests (score partition/dedup). */ environment?: string; /** * Global on/off for ALL guardrails (auto seam + manual check/enforce). * Default true. Also settable via `AISDK_GUARDRAILS_ENABLED=false`. Flip at * runtime with `Observe.setGuardrailsStatus(status)`. */ enabled?: boolean; /** * Max input length (characters) scanned by in-process deterministic guardrail * rules; larger inputs fail open (`payload_too_large`). Default 256K. Also * settable via `AISDK_GUARDRAILS_MAX_PAYLOAD_CHARS`. LLM-judge rules unaffected. */ maxPayloadChars?: number; /** * Per-rule fail-open timeout (ms). When set it OVERRIDES the server-supplied * per-rule timeout for every guardrail; a rule still pending at the deadline * fails open. Default: the server's per-rule value, else 2000. Also settable * via `AISDK_GUARDRAILS_TIMEOUT_MS`. */ timeoutMs?: number; /** * Wall-clock budget (ms) per hook point per call across ALL rules. Rules still * pending when it is spent fail open (`budget_breach`). Default 10000. Also * settable via `AISDK_GUARDRAILS_SEAM_BUDGET_MS`. */ seamBudgetMs?: number; } /** * Configuration for the AISDK client. * Provide either `publicKey` + `secretKey` for full access, * or `writeOnlyKey` alone for write-only (untrusted environment) access. */ export interface AISDKConfig { /** Public key for authentication. Falls back to `AISDK_PUBLIC_KEY` env var. Required unless using writeOnlyKey. */ publicKey?: string; /** Secret key for authentication. Falls back to `AISDK_SECRET_KEY` env var. Required unless using writeOnlyKey. */ secretKey?: string; /** Write-only key for untrusted environments. Mutually exclusive with publicKey/secretKey. */ writeOnlyKey?: string; /** Enable or disable the SDK. Defaults to true. */ enabled?: boolean; /** Environment tag (e.g. 'production', 'staging'). */ environment?: string; /** Enable file-based debug logging. Actual file logging only activates when both this flag and the remote config flag are true. Default: false. */ fileLog?: boolean; /** Prefer IPv4 in DNS and bump autoSelectFamily timeout to 2000ms; mitigates VS Code PacProxyAgent IPv4-abort. Sets process-wide Node defaults. Falls back to `AISDK_RELAX_FAMILY_SELECTION` / `TESTOPS_RELAX_FAMILY_SELECTION` (`true`, case-insensitive). Default: false. */ relaxFamilySelection?: boolean; mask?: CustomScrubFn; /** * Guardrails configuration — binding + enforcement tuning grouped under one * object (guardrails is a first-class user-facing feature). Every field is * optional and has an `AISDK_GUARDRAILS_*` env fallback. */ guardrails?: GuardrailsConfig; } /** * @deprecated Use `AISDKConfig` with `AISDK` instead. */ export interface TestOpsConfig { /** Public key for authentication. Falls back to `TESTOPS_PUBLIC_KEY` env var. */ publicKey?: string; /** Secret key for authentication. Falls back to `TESTOPS_SECRET_KEY` env var. */ secretKey?: string; /** Write-only key for untrusted environments. Mutually exclusive with publicKey/secretKey. */ writeOnlyKey?: string; /** Enable or disable the SDK. Defaults to true. */ enabled?: boolean; /** Environment tag (e.g. 'production', 'staging'). */ environment?: string; /** Enable file-based debug logging. Actual file logging only activates when both this flag and the remote config flag are true. Default: false. */ fileLog?: boolean; /** Prefer IPv4 in DNS and bump autoSelectFamily timeout to 2000ms; mitigates VS Code PacProxyAgent IPv4-abort. Sets process-wide Node defaults. Falls back to `AISDK_RELAX_FAMILY_SELECTION` / `TESTOPS_RELAX_FAMILY_SELECTION` (`true`, case-insensitive). Default: false. */ relaxFamilySelection?: boolean; mask?: CustomScrubFn; /** * Guardrails configuration — binding + enforcement tuning grouped under one * object (guardrails is a first-class user-facing feature). Every field is * optional and has an `AISDK_GUARDRAILS_*` env fallback. */ guardrails?: GuardrailsConfig; } // --------------------------------------------------------------------------- // Body types // --------------------------------------------------------------------------- export interface TraceBody { id?: string | null; timestamp?: Date | null; name?: string | null; userId?: string | null; input?: any; output?: any; retrievedContexts?: any; sessionId?: string | null; release?: string | null; version?: string | null; metadata?: any; tags?: string[] | null; environment?: string | null; public?: boolean | null; } export interface SpanBody { endTime?: Date | null; id?: string | null; traceId?: string | null; name?: string | null; startTime?: Date | null; metadata?: any; input?: any; output?: any; level?: "DEBUG" | "DEFAULT" | "WARNING" | "ERROR"; statusMessage?: string | null; parentObservationId?: string | null; version?: string | null; environment?: string | null; } export interface EventBody { id?: string | null; traceId?: string | null; name?: string | null; startTime?: Date | null; metadata?: any; input?: any; output?: any; level?: "DEBUG" | "DEFAULT" | "WARNING" | "ERROR"; statusMessage?: string | null; parentObservationId?: string | null; version?: string | null; environment?: string | null; } export interface GenerationBody extends SpanBody { completionStartTime?: Date | null; model?: string | null; modelParameters?: Record | null; usage?: | { input?: number | null; output?: number | null; total?: number | null; unit?: "CHARACTERS" | "TOKENS" | "MILLISECONDS" | "SECONDS" | "IMAGES" | "REQUESTS"; inputCost?: number | null; outputCost?: number | null; totalCost?: number | null; } | { promptTokens?: number | null; completionTokens?: number | null; totalTokens?: number | null; }; usageDetails?: Record | null; costDetails?: Record | null; promptName?: string | null; promptVersion?: number | null; } export interface ScoreBody { id?: string | null; traceId?: string | null; sessionId?: string | null; observationId?: string | null; datasetRunId?: string | null; name: string; environment?: string | null; value: string | number; comment?: string | null; metadata?: any; dataType?: "NUMERIC" | "BOOLEAN" | "CATEGORICAL"; configId?: string | null; } // --------------------------------------------------------------------------- // Prompt types // --------------------------------------------------------------------------- export interface ChatMessage { role: string; content: string; } export interface TextPromptClient { readonly name: string; readonly version: number; readonly prompt: string; readonly labels: string[]; readonly tags: string[]; readonly config: Record; readonly type: "text"; readonly isFallback: boolean; readonly commitMessage?: string; readonly promptResponse: any; readonly tools: ToolList; readonly mcpServers: PromptMcpServerRef[]; compile(variables?: Record): string; getLangchainPrompt(): string; } export interface ChatPromptClient { readonly name: string; readonly version: number; readonly prompt: ChatMessage[]; readonly labels: string[]; readonly tags: string[]; readonly config: Record; readonly type: "chat"; readonly isFallback: boolean; readonly commitMessage?: string; readonly promptResponse: any; readonly normalizePrompt: any; readonly tools: ToolList; readonly structuredOutput?: StructuredOutputSnapshot | null; readonly mcpServers: PromptMcpServerRef[]; compile(variables?: Record): ChatMessage[]; getLangchainPrompt(): string; } /** * Structured-output snapshot persisted per prompt version. Mirrors the * server-side `StructuredOutputSnapshotSchema` (webapp/packages/shared * /src/features/prompts/structuredOutputSnapshot.ts). * * - `text` — plain completion (no JSON enforcement). * - `json_object` — OpenAI `response_format: { type: "json_object" }`. * - `json_schema` — full JSON Schema enforcement; `strict` toggles * OpenAI/Azure native strict mode (silently dropped on providers that * fall back to tool-calling). */ export type StructuredOutputSnapshot = | { output_type: "text" } | { output_type: "json_object" } | { output_type: "json_schema"; name: string; description?: string; schema: Record; strict: boolean; source: "schema" | "from_data"; data_path?: string; schema_id: string | null; }; export interface CreateTextPromptBody { type: "text"; name: string; prompt: string; labels?: string[]; config?: Record; modelParams?: Record; toolList?: Record[]; tags?: string[]; isActive?: boolean; commitMessage?: string; } export interface CreateChatPromptBody { type: "chat"; name: string; prompt: ChatMessage[]; labels?: string[]; config?: Record; modelParams?: Record; toolList?: Record[]; tags?: string[]; isActive?: boolean; commitMessage?: string; /** * Optional structured-output snapshot to attach to the new prompt version. * Pass `null` to clear an inherited snapshot from a previous version. */ structuredOutput?: StructuredOutputSnapshot | null; } // --------------------------------------------------------------------------- // Wrapper interfaces (returned by AISDK methods) // --------------------------------------------------------------------------- export interface TraceWrapper { readonly id: string; readonly traceId: string; readonly customId?: string; update(body: Record): TraceWrapper; span(body: SpanBody): SpanWrapper; generation(body: GenerationBody): GenerationWrapper; event(body: EventBody): EventWrapper; score(body: ScoreBody): string; getTraceUrl(): string; } export interface SpanWrapper { readonly id: string; readonly traceId: string; readonly parentId?: string; update(body: Record): SpanWrapper; end(body?: Record): SpanWrapper; span(body: SpanBody): SpanWrapper; generation(body: GenerationBody): GenerationWrapper; event(body: EventBody): EventWrapper; score(body: ScoreBody): string; getTraceUrl(): string; } export interface GenerationWrapper { readonly id: string; readonly traceId: string; readonly parentId?: string; update(body: Record): GenerationWrapper; end(body?: Record): GenerationWrapper; span(body: SpanBody): SpanWrapper; generation(body: GenerationBody): GenerationWrapper; event(body: EventBody): EventWrapper; score(body: ScoreBody): string; getTraceUrl(): string; } export interface EventWrapper { readonly id: string; readonly traceId: string; readonly parentId?: string; span(body: SpanBody): SpanWrapper; generation(body: GenerationBody): GenerationWrapper; event(body: EventBody): EventWrapper; score(body: ScoreBody): string; getTraceUrl(): string; } // --------------------------------------------------------------------------- // AISDK class // --------------------------------------------------------------------------- export declare class AISDK { /** Pass keys explicitly or set `AISDK_PUBLIC_KEY` / `AISDK_SECRET_KEY` env vars. */ constructor(config?: AISDKConfig); // --- Core observability methods --- trace(body: TraceBody): TraceWrapper; generateTraceId(customId: string): string; generation(body: GenerationBody): GenerationWrapper; span(body: SpanBody): SpanWrapper; event(body: EventBody): EventWrapper; score(body: ScoreBody): void; // --- Lifecycle methods --- flush(): Promise; flushAsync(): Promise; shutdown(): Promise; shutdownAsync(): Promise; // --- Dataset runs --- readonly datasetRuns: DatasetRuns; // --- Datasets --- readonly datasets: DatasetsClient; // --- Evals --- readonly evals: EvaluationExecution; // --- Evaluator lists --- readonly evalsList: EvaluatorLists; // --- Evaluators --- readonly evaluators: Evaluators; // --- Experiment runs --- readonly experimentRuns: ExperimentRunsAPI; // --- Experiments --- readonly experiments: ExperimentsAPI; // --- Personas --- readonly personas: Personas; // --- Online Eval --- readonly onlineEval: OnlineEval; // --- Alert --- readonly alert: Alert; // --- Agent (persisted HTTP / Voice agent config; internally `ApiRecord`) --- readonly agent: Agent; // --- Default LLM Model (per-project singleton) --- readonly defaultLlmModel: DefaultLlmModel; // --- Traces --- readonly traces: Traces; // --- Automation Rules --- readonly automationRules: AutomationRule; // --- Scenario datasets --- readonly scenarioDatasets: ScenarioDatasets; // --- Scenario items --- readonly scenarioItems: ScenarioItems; // --- Scenario runs --- readonly scenarioRuns: ScenarioRuns; // --- Tools --- readonly tools: Tool; // --- MCP servers --- readonly mcpServers: McpServers; // --- Prompt namespace --- readonly prompt: Prompt; // --- Evaluate namespace --- readonly evaluate: Evaluate; // --- Static members --- static observe: { init: (config?: AISDKConfig) => Promise; }; } // --------------------------------------------------------------------------- // Type aliases // --------------------------------------------------------------------------- /** * @deprecated Use `AISDK` instead. `AISDKClient` will be removed in a future major version. */ export declare class AISDKClient extends AISDK {} /** * @deprecated Use `AISDK` instead. `TestOps` will be removed in a future major version. */ export declare class TestOps extends AISDK { /** @deprecated Use `new AISDK(AISDKConfig)` with `AISDK_PUBLIC_KEY` / `AISDK_SECRET_KEY` env vars instead. */ constructor(config?: TestOpsConfig); } /** * @deprecated Use `AISDK` instead. `TestOpsClient` will be removed in a future major version. */ export declare class TestOpsClient extends AISDK { /** @deprecated Use `new AISDK(AISDKConfig)` with `AISDK_PUBLIC_KEY` / `AISDK_SECRET_KEY` env vars instead. */ constructor(config?: TestOpsConfig); } /** * @deprecated Use `Media` instead. `TestOpsMedia` will be removed in a future major version. */ export declare class TestOpsMedia extends Media {} // --------------------------------------------------------------------------- // Observe namespace // --------------------------------------------------------------------------- type StringOnlyTraceAttribute = TraceAttribute.SESSION_ID | TraceAttribute.USER_ID; type TraceAttributeValue = string | number | boolean | Record | unknown[]; export type ContextHandle = { span: any; previousContext: unknown; }; export interface ObserveMiddlewareOptions { allowHosts?: string[]; denyHosts?: string[]; injectW3C?: boolean; injectXTraceContext?: boolean; } export declare const Observe: { init: (config?: AISDKConfig) => Promise; /** Force-flush buffered auto-traced spans. Call before a serverless handler returns. */ flush: () => Promise; setAttribute: { (key: TraceAttribute, value: TraceAttributeValue): void; (key: K extends StringOnlyTraceAttribute ? never : K, value: TraceAttributeValue): void; }; TraceAttribute: typeof TraceAttribute; getBaggage: () => string; setBaggage: (baggageString: string) => void; withTrace: (name: string, baggageString: string | null, fn: () => T | Promise) => T | Promise; startContext: (name: string, baggageString?: string | null) => ContextHandle; endContext: (handle?: ContextHandle) => void; middleware: (options?: ObserveMiddlewareOptions) => (req: any, res: any, next: () => void) => void; propagation: { inject: (context: any, carrier: any, setter?: any) => void; extract: (context: any, carrier: any, getter?: any) => any; getBaggage: (context: any) => any; }; context: { active: () => any; with: (context: any, fn: () => T) => T; }; trace: { getTracer: (name: string, version?: string) => any; getActiveSpan: () => any; setSpan: (context: any, span: any) => any; setSpanContext: (context: any, spanContext: any) => any; }; defaultTextMapGetter: any; defaultTextMapSetter: any; getCurrentSpan: () => SpanHandle; getRootSpan: () => SpanHandle; getLatestObservation: () => ObservationHandle; updateLatestObservation: (options: UpdateObservationOptions) => void; updateObservation: (traceId: string, spanId: string, options: UpdateObservationOptions) => Promise; /** * Runtime kill-switch for ALL guardrails (auto seam + manual check/enforce). * Takes effect immediately, no re-init. Overrides `guardrailsEnabled` config * until the next `Observe.init`. Use for incident response. */ setGuardrailsStatus: (status: boolean) => void; /** * Current global guardrails on/off state — the value last set by * `guardrailsEnabled` config or `setGuardrailsStatus`. Pairs with * `setGuardrailsStatus` for parity with the Python SDK. */ isGuardrailsEnabled: () => boolean; }; export interface ObservationHandle { readonly traceId: string | null; readonly spanId: string | null; update(options: UpdateObservationOptions): void; } export type LogLevel = 'DEBUG' | 'INFO' | 'WARN' | 'ERROR'; export interface UsageDetails { promptTokens?: number; completionTokens?: number; totalTokens?: number; cacheReadInputTokens?: number; cacheCreationInputTokens?: number; reasoningTokens?: number; } export interface CostDetails { inputCostUsd?: number; outputCostUsd?: number; totalCostUsd?: number; } export interface UpdateObservationOptions { // Span-shaped fields metadata?: Record; tags?: string[]; input?: unknown; output?: unknown; level?: LogLevel; statusMessage?: string; version?: string; userId?: string; sessionId?: string; // Generation-shaped fields model?: string; usageDetails?: UsageDetails; costDetails?: CostDetails; completionStartTime?: number; promptName?: string; promptVersion?: string; } // SpanHandle and update() accept the same field set as UpdateObservationOptions. export type UpdateOptions = UpdateObservationOptions; export interface SpanHandle { readonly traceId: string | null; readonly spanId: string | null; readonly isRecording: boolean; readonly permalink: string | null; update(options: UpdateOptions): void; recordException(err: Error | string): void; } // --------------------------------------------------------------------------- // Top-level function exports // --------------------------------------------------------------------------- export declare function wrapVercelAI(aiModule: T): T; // --------------------------------------------------------------------------- // SSR framework helpers // --------------------------------------------------------------------------- /** * Next.js config helper. Adds the SDK's auto-instrumented packages to * `serverExternalPackages` so SSR bundlers don't inline them — required * for OpenTelemetry's require/import hooks to see the real modules at * runtime. * * Usage in `next.config.ts`: * * import { withAISDKNext } from "@browserstack/ai-sdk"; * export default withAISDKNext({}); */ export declare function withAISDKNext( userConfig?: T ): T; /** * Vite plugin (SvelteKit / Astro / any Vite-based SSR). Adds the SDK's * auto-instrumented packages to `ssr.external` so Vite/Rollup don't inline * them — required for OpenTelemetry's require/import hooks to see the real * modules at runtime. * * Usage in `vite.config.ts`: * * import { withAISDKVite } from "@browserstack/ai-sdk"; * export default defineConfig({ plugins: [withAISDKVite(), sveltekit()] }); */ export declare function withAISDKVite(): { name: string; enforce?: "pre" | "post"; config: (config: any) => any; }; // --------------------------------------------------------------------------- // Decorators, factory functions, and utilities are available at runtime // but intentionally excluded from this type declaration file. // They are not part of the public API surface. // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- // Class declarations // --------------------------------------------------------------------------- export interface DatasetRuns { create(datasetName: string, options?: { name?: string; description?: string; metadata?: string; tag?: string }): Promise; create(params: CreateWithPromptRequest): Promise; /** @deprecated Use `create(params: CreateWithPromptRequest)` instead. */ createWithPrompt(params: CreateWithPromptRequest): Promise; createItems(datasetName: string, datasetRunId: string, items: CreateDatasetRunItemRequest[]): Promise; getRunByTag(datasetName: string, tagName: string): Promise; getRunTags(datasetName: string, datasetRunId: string, page?: number, limit?: number): Promise; list(datasetName: string, page?: number, limit?: number, name?: string, type?: string): Promise; listItems(datasetName: string, datasetRunId: string, page?: number, limit?: number): Promise; getByName(datasetName: string, runName: string): Promise; update(datasetName: string, runName: string, fields: { description?: string; metadata?: Record }): Promise; delete(datasetName: string, runName: string): Promise; } export interface Datasets {} export interface UpdateDatasetItemFields { input?: unknown; expectedOutput?: unknown; output?: unknown; expectedToolCalls?: unknown; context?: unknown; retrievedContexts?: unknown; referenceContexts?: unknown; metadata?: unknown; sourceTraceId?: string | null; sourceObservationId?: string | null; status?: string | null; } export interface UpdateDatasetItemRequest extends UpdateDatasetItemFields { id: string; } export interface UpdateItemsRequest { datasetName: string; items?: UpdateDatasetItemRequest[]; fileUrl?: string; } export interface UpdateItemsOptions { batchSize?: number; } export interface UpdateDatasetItemsResponse { success: boolean; updatedCount: number; } export interface DeleteDatasetResponse { message: string; } export interface DatasetItemResponse { id: string; datasetId: string; datasetName: string; input?: unknown; expectedOutput?: unknown; output?: unknown; expectedToolCalls?: unknown; context?: unknown; retrievedContexts?: unknown; referenceContexts?: unknown; metadata?: unknown; sourceTraceId?: string | null; sourceObservationId?: string | null; status?: string; createdAt: string; updatedAt: string; } export interface PaginationMeta { page: number; limit: number; totalItems: number; totalPages: number; } export interface GetDatasetResponse { id: string; name: string; description?: string; metadata?: Record; projectId: string; kind?: DatasetKind; currentVersion?: number; createdAt: string; updatedAt: string; items: DatasetItemResponse[]; runs: string[]; meta: PaginationMeta; } export declare class UpdateItemsChunkError extends Error { readonly failedChunkIndex: number; readonly lastSuccessfulRowIndex: number; readonly cause: Error; constructor(message: string, failedChunkIndex: number, lastSuccessfulRowIndex: number, cause: Error); } export interface DatasetsClient { clone(name: string, newName: string): Promise; create(name: string, description?: string, metadata?: Record, kind?: DatasetKind): Promise; createItems(params: CreateItemsRequest, options?: CreateItemsOptions): Promise; delete(name: string): Promise; deleteItem(id: string): Promise<{ message: string }>; get(datasetName: string, page?: number, limit?: number): Promise; getItem(id: string): Promise; getTags(datasetName: string): Promise; list(page?: number, limit?: number, name?: string, kind?: DatasetKind): Promise; generateItems(params: GenerateItemsParams): Promise; generateMoreItems(params: GenerateMoreItemsParams): Promise; update(name: string, fields: { description?: string; metadata?: Record }): Promise; updateItems(params: UpdateItemsRequest, options?: UpdateItemsOptions): Promise; } export interface RunEvaluationExperimentRequest { name: string; datasetId: string; evaluatorListId: string; description?: string; concurrency?: number; datasetVersion?: number | null; } export interface EvaluationDatasets { create(name: string, description?: string, metadata?: Record): Promise; list(page?: number, limit?: number, name?: string): Promise; createItems(params: CreateItemsRequest, options?: CreateItemsOptions): Promise; runExperiment(params: RunEvaluationExperimentRequest): Promise; } export interface EvaluationDatasetsClient { create(name: string, description?: string, metadata?: Record): Promise; list(page?: number, limit?: number, name?: string): Promise; createItems(params: CreateItemsRequest, options?: CreateItemsOptions): Promise; runExperiment(params: RunEvaluationExperimentRequest): Promise; } export declare class Evaluate { /** Pass keys explicitly or set `AISDK_PUBLIC_KEY` / `AISDK_SECRET_KEY` env vars. */ constructor(publicKey?: string, secretKey?: string); readonly dataset: { create(name: string, description?: string, metadata?: Record, kind?: DatasetKind): Promise; list(page?: number, limit?: number, name?: string, kind?: DatasetKind): Promise; getTags(datasetName: string): Promise; createItems(params: CreateItemsRequest, options?: CreateItemsOptions): Promise; }; readonly evaluationDataset: { create(name: string, description?: string, metadata?: Record): Promise; list(page?: number, limit?: number, name?: string): Promise; createItems(params: CreateItemsRequest, options?: CreateItemsOptions): Promise; runExperiment(params: RunEvaluationExperimentRequest): Promise; }; readonly datasetRun: { create(datasetName: string, options?: { name?: string; description?: string; metadata?: string; tag?: string }): Promise; create(params: CreateWithPromptRequest): Promise; /** @deprecated Use `create(params: CreateWithPromptRequest)` instead. */ createWithPrompt(params: CreateWithPromptRequest): Promise; list(datasetName: string, page?: number, limit?: number, name?: string, type?: string): Promise; listItems(datasetName: string, datasetRunId: string, page?: number, limit?: number): Promise; createItems(datasetName: string, datasetRunId: string, items: CreateDatasetRunItemRequest[]): Promise; getRunByTag(datasetName: string, tagName: string): Promise; getRunTags(datasetName: string, datasetRunId: string, page?: number, limit?: number): Promise; getByName(datasetName: string, runName: string): Promise; update(datasetName: string, runName: string, fields: { description?: string; metadata?: Record }): Promise; delete(datasetName: string, runName: string): Promise; }; readonly evaluationExecution: { evaluate(requestData: EvaluationExecutionRequest): Promise; }; readonly evaluator: { create(options: CreateEvaluatorOptions): Promise; get(evaluatorId: string): Promise; list(options?: ListEvaluatorsOptions): Promise; update(evaluatorId: string, options: UpdateEvaluatorOptions): Promise; listVersions(evaluatorId: string, options: ListVersionsOptions): Promise; setLabels(evaluatorId: string, labels: string[]): Promise; }; readonly evaluatorList: { create(requestData: CreateEvaluatorListRequest): Promise; list(limit?: number, page?: number, orderBy?: { column?: string; order?: "ASC" | "DESC" }): Promise; get(evaluatorListId: string): Promise; delete(evaluatorListId: string): Promise; }; /** * Full experiment API — same contract as the standalone `Experiments` * singleton: define-only local (`LocalExperimentCreateOptions` → * `LocalExperimentHandle`) plus the server overloads. The one-step * task-in-create form is intentionally absent (use `experimentRun.create`). */ readonly experiment: ExperimentsAPI; /** * Full experiment-run API — same contract as the standalone `ExperimentRuns` * singleton: local two-step `create(handle, { task })` plus the server * create/subscribe/compare/summarize overloads. */ readonly experimentRun: ExperimentRunsAPI; /** * Guardrails — CRUD (guardrails-as-code) AND runtime enforcement on ONE * namespace. `create` / `update` only ever produce drafts (promote in the UI * to enforce); `check` / `enforce` run the same chain as the auto seams. * Available as a static (`Evaluate.guardrail.*`) and on an instance * (`client.evaluate.guardrail.*`). */ readonly guardrail: Guardrails; /** * Static form of {@link Evaluate.guardrail} — `Evaluate.guardrail.create(...)` * etc. work without constructing an instance (keys resolved from * `AISDK_PUBLIC_KEY` / `AISDK_SECRET_KEY`). */ static guardrail: Guardrails; } export interface EvaluationExecution { evaluate(requestData: EvaluationExecutionRequest): Promise; } export interface EvaluationExecutionError extends Error { readonly statusCode?: number; readonly response?: any; } export interface EvaluatorLists { create(requestData: CreateEvaluatorListRequest): Promise; delete(evaluatorListId: string): Promise; get(evaluatorListId: string): Promise; list(limit?: number, page?: number, orderBy?: { column?: string; order?: "ASC" | "DESC" }): Promise; } export interface Evaluators { create(options: CreateEvaluatorOptions): Promise; get(evaluatorId: string): Promise; list(options?: ListEvaluatorsOptions): Promise; listVersions(evaluatorId: string, options: ListVersionsOptions): Promise; setLabels(evaluatorId: string, labels: string[]): Promise; update(evaluatorId: string, options: UpdateEvaluatorOptions): Promise; } // --------------------------------------------------------------------------- // Guardrails CRUD (guardrails-as-code) // --------------------------------------------------------------------------- export type GuardrailHookPoint = | 'before_model' | 'after_model' // Dispatched by the agent-framework entry seams (agent-seam) for wrapped agent // frameworks (e.g. claude-agent-sdk, google-adk); a rule bound here fires on the // whole agent run's input/output. Rollout across the remaining agentic // frameworks is in progress. | 'before_agent' | 'after_agent' | 'before_tool_call' | 'after_tool_call'; export type GuardrailActionType = 'default_response' | 'redact' | 'error_4xx' | 'reask'; /** Deterministic (in-process) rule spec — regex is RE2-pinned (no lookbehind/backrefs). */ export type GuardrailDeterministicRule = | { mode: 'regex'; pattern: string; caseInsensitive: boolean } | { mode: 'keyword'; keywords: string[]; caseInsensitive: boolean } | { mode: 'json_schema'; schema: Record } | { mode: 'pii'; entities: Array<'email' | 'credit_card' | 'ip_address' | 'mac_address' | 'url'> }; export interface GuardrailUpsertOptions { hookPoint: GuardrailHookPoint; action: GuardrailActionType; defaultResponseText?: string | null; /** Provide this OR `judgePrompt` — exactly one mechanism per guardrail. */ ruleSpec?: GuardrailDeterministicRule | null; judgePrompt?: string | null; provider?: string | null; model?: string | null; } export interface CreateGuardrailOptions extends GuardrailUpsertOptions { name: string; } /** Create / update responses — the touched version is always a draft. */ export interface GuardrailDraftResponse { name: string; state: 'draft'; } export interface GuardrailListItem { name: string; version: number; state: 'live' | 'draft' | 'disabled'; hookPoint: GuardrailHookPoint | null; action: GuardrailActionType | null; defaultResponseText: string | null; mechanism: 'deterministic' | 'llm'; ruleSpec: Record | null; judgePrompt: string | null; } export interface ListGuardrailsResponse { guardrails: GuardrailListItem[]; } export interface DeleteGuardrailResponse { name: string; archived: true; } /** The rule payload the SDK enforces at runtime (the wire contract). */ export interface GuardrailLiveRule { evaluatorId: string; name: string; hookPoint: GuardrailHookPoint; action: GuardrailActionType; defaultResponseText?: string | null; shortCircuit: boolean; logic: | { kind: 'deterministic'; spec: GuardrailDeterministicRule } | { kind: 'llm'; spec: { judgePrompt: string; provider: string; model: string; modelParams?: Record | null; timeoutMs: number; }; }; } /** GET by name: the live definition, or `unresolved` when no version is live. */ export type GuardrailByNameResult = | { status: 'live'; contractVersion: '1'; name: string; etag: string; rule: GuardrailLiveRule } | { status: 'unresolved'; contractVersion: '1'; name: string; etag: string; reason: 'no_live_version' }; /** * Guardrails API — ONE surface (`Evaluate.guardrail`) for both CRUD * (guardrails-as-code) and runtime enforcement. Create/update only ever produce * DRAFTs — the SDK cannot promote; promotion stays a validated UI action. * check/enforce run the same chain as auto-instrumentation. */ export interface Guardrails { /** Create a new guardrail as a DRAFT (version 1). 409s if the name exists. */ create(options: CreateGuardrailOptions): Promise; /** Fetch a guardrail's LIVE definition by name (the enforcement payload). */ get(name: string): Promise; /** List the project's guardrails (latest version of each). */ list(): Promise; /** Edit — forks a new DRAFT and auto-demotes any live version of the name. */ update(name: string, options: GuardrailUpsertOptions): Promise; /** Archive all versions (soft delete). */ delete(name: string): Promise; /** * Manually enforce the named guardrail(s) over `content` (manual tracing). * Runs the same chain as auto-instrumentation (deterministic in-process + * `llm` via the server judge). Returns a verdict and NEVER throws. * * Both args are optional under auto-tracing: omit `options.guardrails` to run * the init-bound set for the active agent, and omit `content` to guard the * active generation — each rule reads its hook-point side (before_* → input, * after_* → output) from the recorded generation I/O; a missing side fails * open (`missing_content`). To name guardrail(s) without content, pass the * options directly: `check({ guardrails: [...] })`. */ check(options?: ManualGuardrailOptions): Promise; check(content?: string, options?: ManualGuardrailOptions): Promise; /** * Like `check`, but throws `GuardrailBlockedError` when a rule stops the call * and otherwise returns the safe (possibly redacted) text. Both args optional * under auto-tracing (see `check`); the options-only form * `enforce({ guardrails: [...] })` names guardrail(s) without content. */ enforce(options?: ManualGuardrailOptions): Promise; enforce(content?: string, options?: ManualGuardrailOptions): Promise; } /** Options for the manual-enforce API (`Evaluate.guardrail.check` / `enforce`). */ export interface ManualGuardrailOptions { /** * Guardrail name(s) to run for this call (resolved on demand). Optional under * auto-tracing — omit to run the init-bound guardrails for the active agent. */ guardrails?: readonly string[]; /** Agent name for the server-side attachment merge; else the active agent span. */ agentName?: string; /** Explicit trace identity for the emitted decision score; else the active span. */ traceId?: string; observationId?: string; } /** Per-guardrail outcome — one entry per resolved rule, in `GuardrailCheckResult.details`. */ export interface GuardrailDetail { /** The guardrail (rule) name. */ guardrailName: string; /** This rule's own decision. */ decision: 'passed' | 'blocked' | 'redacted' | 'failed_open'; /** This rule's on-match action, when it fired. */ action?: 'default_response' | 'redact' | 'error_4xx' | 'reask'; /** True when this rule's action stops the call. */ blocked: boolean; /** The hook point this rule is bound to. */ hookPoint?: GuardrailHookPoint; /** Wall-clock time this rule's evaluation took. */ latencyMs: number; /** Why this rule failed open, when it did. */ failOpenReason?: string; } /** Verdict returned by `guardrail.check`. */ export interface GuardrailCheckResult { /** passed | blocked | redacted | failed_open. */ decision: 'passed' | 'blocked' | 'redacted' | 'failed_open'; /** The matched rule's on-match action, when a rule fired. */ action?: 'default_response' | 'redact' | 'error_4xx' | 'reask'; /** True when the action stops the call (the caller must not proceed as-is). */ blocked: boolean; /** Text to carry forward: redacted on `redact`, the substitute body on a block, else the original. */ content: string; guardrailName?: string; failOpenReason?: string; /** Whether a decision score was emitted (only when trace identity resolved). */ emitted: boolean; /** Per-guardrail breakdown — one entry per resolved rule, in request order. */ details: GuardrailDetail[]; } export interface ExperimentRunsAPI { /** Local execution — handle from Experiments.create() → runs dataset → task → evaluators locally */ create(handle: LocalExperimentHandle, options: LocalExperimentRunOptions): Promise; /** * Object-form server-side create. Preferred over the positional form — * carries typed `runConfig` and the `wait` sugar. When `wait` is set, * the return resolves the polled subscription result. */ create(options: CreateExperimentRunOptions): Promise; /** * Positional server-side create. * * @deprecated The `llmColumnName` positional param is stripped by the * server (see `webapp/web/src/features/experiments/server` §9.4) — pass * the object form instead so future additions (`runConfig.apiCallTimeoutMs`, * `wait`) are typed. */ create(experimentId: string, llmColumnName?: string, concurrency?: number, runConfig?: Record, metadata?: Record): Promise; delete(experimentRunId: string): Promise; find(experimentRunId: string): Promise; list(experimentId?: string, page?: number, limit?: number): Promise; subscribe(experimentRunId: string, timeout?: number, pollInterval?: number): Promise; update(experimentRunId: string, updateData: UpdateExperimentRunRequest): Promise; /** * Fetch a server-computed "compare this run vs its baseline" payload. * The backend resolves the baseline (most-recent COMPLETED run for the * same experiment, excluding this one) and computes per-evaluator * deltas, threshold pass/fail, and a pre-rendered PR-comment markdown. * Returns status='pending' when scores are still aggregating; retry. */ compare(experimentRunId: string): Promise; /** * Fetch a full summary for an experiment run in a single call. * * Returns scores (with per-evaluator distributions, threshold pass/fail, * and diffs vs the baseline), aggregate metrics (latency, token counts, * cost), an overall PASS/REGRESSION verdict, and UI deep-links to the * TestAI evaluation dashboard. Equivalent to Braintrust's * `experiment.summarize()`. * * `opts.compareToRunId` — pin the comparison to a specific baseline run. * When omitted, the server uses the experiment's UI-pinned baseline (set * via "Set as baseline"). If neither is provided, no comparison is * performed and every score's `diff` is null. * * Deep-link shortcuts: `summary.urls.scoresTab`, `summary.urls.analyticsTab`. */ summarize(experimentRunId: string, opts?: { compareToRunId?: string }): Promise; /** * Read the stored comparison verdict for a (base, target) run pair. * * This is a read-only call — it does not trigger verdict generation. * Returns status=NOT_GENERATED (HTTP 200) when no verdict has been computed * yet for the pair; the caller may poll until status=READY. * * @param experimentId The experiment both runs belong to. * @param baseRunId The baseline run ID. * @param targetRunId The run ID to compare against the baseline. */ comparisonVerdict(experimentId: string, baseRunId: string, targetRunId: string): Promise; } /** Standalone ExperimentRuns object — reads credentials from env vars. */ export declare const ExperimentRuns: ExperimentRunsAPI; export interface ExperimentsAPI { /** Two-step Step 1 — local config fields → creates experiment handle */ create(options: LocalExperimentCreateOptions): Promise; /** Server-side — no task → creates experiment via platform API */ create(requestData: CreateExperimentWithTagRequest): Promise; create(requestData: CreateExperimentWithPromptRequest): Promise; delete(experimentId: string): Promise; /** * Find an experiment by id (legacy: throws on 404) or by name lookup * (returns null when no match). Pass a string for id, or an object * with `id` or `name`. */ find(experimentId: string): Promise; find(opts: { id?: string; name?: string }): Promise; list(limit?: number, page?: number): Promise; } export interface Personas { create(request: CreatePersonaRequest): Promise; list(): Promise; get(personaId: string): Promise; delete(personaId: string): Promise; } export type EventSchedulerType = "TRACE" | "SESSION"; export type DatasetMappingType = "EXPECTED_OUTPUT" | "LLM_OUTPUT"; export type DatasetExtractionMode = "SINGLE_IO" | "FULL_CONVERSATION"; export type OnlineEvalFilterField = | "name" | "input" | "output" | "user_id" | "session_id" | "release" | "version" | "environment" | "level" | "total_cost" | "user_generated_id" | "tags" | "trace_count" | "observationType" | "observationName"; export interface OnlineEvalFilter { /** * One of the known fields in `OnlineEvalFilterField` (autocomplete on the * 15 known values) OR a free-form `metadata.` path. Server validates * against the allowlist matching the rule's `sourceType` — TRACE rules * accept trace + observation fields, SESSION rules accept a tighter subset * including `trace_count` but not observation fields. */ fieldName: OnlineEvalFilterField | (string & {}); operator: string; value: string; logicalOperator?: string; } export interface OnlineEvalEvaluatorList { id: string; name: string; /** ExperimentEvaluator.type — `EVALUATOR` | `HUMAN_REVIEW` | `MIXED`. */ type: string; } export interface OnlineEvalConfig { id: string; projectId: string; name: string; description: string | null; /** Immutable after creation. */ sourceType: EventSchedulerType; /** At least one of `onlineEvalEnabled` / `datasetEnabled` is always true. */ onlineEvalEnabled: boolean; datasetEnabled: boolean; datasetId: string | null; /** Values match `DatasetMappingType` (response side is un-enum'd on the wire). */ datasetMappingType: string | null; /** Values match `DatasetExtractionMode` (response side is un-enum'd on the wire). */ datasetExtractionMode: string | null; outputObservationName: string | null; datasetTag: string | null; samplingRate: number; delay: number; /** * SESSION rules only — fire as soon as the session accumulates this many * traces, bypassing the idle `delay`. 0 = disabled. Always 0 on TRACE rules. */ triggerTraceCount: number; status: boolean; filters: OnlineEvalFilter[]; evaluatorLists: OnlineEvalEvaluatorList[]; createdBy: string | null; createdAt: string; updatedAt: string; /** * Distinct number of traces this rule has evaluated. Only present on list * responses (omitted on single-rule GET/POST/PATCH). */ traceCount?: number; } export interface CreateOnlineEvalBody { name: string; description?: string | null; /** Defaults to TRACE if omitted. Immutable after creation. */ sourceType?: EventSchedulerType; /** * Presence toggles the dataset-addition action on. When set, * `datasetMappingType = LLM_OUTPUT` additionally requires `datasetTag`. */ datasetId?: string; datasetMappingType?: DatasetMappingType; datasetExtractionMode?: DatasetExtractionMode; outputObservationName?: string | null; /** Required when `datasetMappingType: LLM_OUTPUT`. */ datasetTag?: string | null; samplingRate?: number; delay?: number; /** SESSION rules only. Sending a non-zero value with TRACE returns 400. */ triggerTraceCount?: number; status?: boolean; filters?: OnlineEvalFilter[]; /** * Non-empty array toggles the online-evaluation action on. At least one * of the two actions (online eval / dataset) must be enabled — i.e. at * least one of `evaluatorListIds` (non-empty) or `datasetId` (set) must * be present. */ evaluatorListIds?: string[]; } export interface UpdateOnlineEvalBody { name?: string; description?: string | null; // sourceType is intentionally absent — immutable after creation. /** * Explicit pause/resume for the online-evaluation action. When omitted, * the server derives from `evaluatorListIds` if that field is present. */ onlineEvalEnabled?: boolean; /** * Explicit pause/resume for the dataset-addition action. When omitted, * the server derives from `datasetId` if that field is present. */ datasetEnabled?: boolean; datasetId?: string; datasetMappingType?: DatasetMappingType; datasetExtractionMode?: DatasetExtractionMode; outputObservationName?: string | null; datasetTag?: string | null; samplingRate?: number; delay?: number; triggerTraceCount?: number; status?: boolean; filters?: OnlineEvalFilter[]; evaluatorListIds?: string[]; } export interface ListOnlineEvalsParams { page?: number; limit?: number; search?: string; } export interface ListOnlineEvalsResponse { /** Each row has `traceCount` populated. */ data: OnlineEvalConfig[]; meta: { page: number; limit: number; totalItems: number; totalPages: number }; } export interface DeleteOnlineEvalResponse { success: boolean; } // ── OnlineEval analytics (4 methods added in AIOPS-2541) ─────────────────── export type Granularity = 'minute' | 'hour' | 'day' | 'week' | 'month'; export type TraceFilterColumn = | 'bookmarked' | 'id' | 'name' | 'timestamp' | 'userId' | 'sessionId' | 'userGeneratedId' | 'experimentRunId' | 'metadata' | 'classifier_data' | 'version' | 'release' | 'environment' | 'tags' | 'level' | 'observationType' | 'observationName' | 'model' | 'warningCount' | 'errorCount' | 'defaultCount' | 'debugCount' | 'inputTokens' | 'outputTokens' | 'totalTokens' | 'tokens' | 'latency' | 'agentLatency' | 'inputCost' | 'outputCost' | 'totalCost' | 'scores'; export type FilterOperator = | 'EQUALS' | 'NOT_EQUALS' | 'CONTAINS' | 'NOT_CONTAINS' | 'STARTS_WITH' | 'ENDS_WITH' | 'IN' | 'NOT_IN' | 'GREATER_THAN' | 'LESS_THAN' | 'GREATER_THAN_OR_EQUAL' | 'LESS_THAN_OR_EQUAL' | 'IS_EMPTY' | 'IS_NOT_EMPTY'; export interface OnlineEvalTraceFilterEntry { /** TraceFilterColumn, OR a dot-path `.` for metadata/classifier_data/scores. */ fieldName: string; operator: FilterOperator; /** Omit for IS_EMPTY / IS_NOT_EMPTY. */ value?: string | number | boolean | string[] | number[]; } export interface OnlineEvalTraceFilterSet { /** Every entry must match (AND-combined). */ must?: OnlineEvalTraceFilterEntry[]; /** At least one entry must match (OR-combined). */ any?: OnlineEvalTraceFilterEntry[]; } export interface EvaluatorScoresParams { /** Inclusive lower bound. Defaults to `toTimestamp − 24h`. */ fromTimestamp?: Date | string; /** Inclusive upper bound. Defaults to server clock. */ toTimestamp?: Date | string; granularity?: Granularity; } export interface NumericEvaluatorSeries { evaluatorId: string; evaluatorName: string; dataType: 'NUMERIC'; source: 'EVAL' | 'ANNOTATION'; scores: { timestamp: string; score: number }[]; } export interface CategoricalEvaluatorSeries { evaluatorId: string; evaluatorName: string; dataType: 'CATEGORICAL'; source: 'EVAL' | 'ANNOTATION'; buckets: { timestamp: string; counts: Record }[]; ignoreCategories?: string[]; } export interface EvaluatorScoresResponse { evaluators: (NumericEvaluatorSeries | CategoricalEvaluatorSeries)[]; } export interface ObservationMetricsParams { fromTimestamp?: Date | string; toTimestamp?: Date | string; granularity?: Granularity; } export interface ObservationMetricsResponse { costChartData: Record[]; totalCost: number; costAdapters: { name: string; color: string }[]; tokenChartData: Record[]; totalTokens: number; tokenAdapters: { name: string; color: string }[]; providerDistribution: { name: string; percentage: number; color: string; count: number }[]; modelDistribution: { name: string; percentage: number; color: string; count: number }[]; } export interface TraceListParams { page?: number; limit?: number; searchQuery?: string; searchType?: ('id' | 'content')[]; filter?: OnlineEvalTraceFilterSet; orderBy?: { column: TraceFilterColumn; order: 'ASC' | 'DESC' }; } export interface OnlineEvalRuleTrace { id: string; projectId: string; timestamp: string; name: string | null; userId: string | null; sessionId: string | null; release: string | null; version: string | null; environment: string | null; tags: string[]; bookmarked: boolean; public: boolean; } export interface ListOnlineEvalTracesResponse { data: OnlineEvalRuleTrace[]; meta: { page: number; limit: number; totalItems: number; totalPages: number }; } export interface OnlineEval { // docs:params-start(online-eval-rules.create) create(body: CreateOnlineEvalBody): Promise; // docs:params-end(online-eval-rules.create) // docs:params-start(online-eval-rules.list) list(params?: ListOnlineEvalsParams): Promise; // docs:params-end(online-eval-rules.list) // docs:params-start(online-eval-rules.get) get(ruleId: string): Promise; // docs:params-end(online-eval-rules.get) // docs:params-start(online-eval-rules.update) update(ruleId: string, body: UpdateOnlineEvalBody): Promise; // docs:params-end(online-eval-rules.update) // docs:params-start(online-eval-rules.delete) delete(ruleId: string): Promise; // docs:params-end(online-eval-rules.delete) /** Time-bucketed evaluator score series for a rule. Defaults to last 24h. */ // docs:params-start(online-eval-rules.scores) scores(ruleId: string, params?: EvaluatorScoresParams): Promise; // docs:params-end(online-eval-rules.scores) /** Cost + token usage timeseries + distributions for traces this rule scored. */ // docs:params-start(online-eval-rules.metrics) metrics(ruleId: string, params?: ObservationMetricsParams): Promise; // docs:params-end(online-eval-rules.metrics) /** Paginated, filtered list of traces this rule has produced executions for. */ // docs:params-start(online-eval-rules.listTraces) listTraces(ruleId: string, params?: TraceListParams): Promise; // docs:params-end(online-eval-rules.listTraces) /** Fetch a single trace with observations + scores (facade over /api/public/traces/{traceId}). */ // docs:params-start(online-eval-rules.trace) trace(traceId: string): Promise; // docs:params-end(online-eval-rules.trace) } // --- Alert (codegen-emitted; see src/clients/Alert) --- export interface NotificationTarget { channel: 'webhook' | 'email' | 'slack'; enabled: boolean; config: Record; } export interface AlertConfig { id: string; projectId: string; name: string; description: string | null; scope: 'TRACE' | 'SESSION'; aggregation: string; metric: string; operator: string; threshold: number; unit: string | null; windowMinutes: number; cooldownMinutes: number; environments: string[]; filter: Record | null; notifications: NotificationTarget[]; status: boolean; evaluationState: string; lastFilterError: string | null; lastFiredAt: string | null; lastEvaluatedAt: string | null; createdBy: string | null; createdAt: string; updatedAt: string; } export interface CreateAlertBody { name: string; description?: string | null; scope?: 'TRACE' | 'SESSION'; aggregation?: string; metric?: string; operator?: string; threshold?: number; unit?: string | null; windowMinutes: number; cooldownMinutes?: number; environments?: string[]; notifications: NotificationTarget[]; filter?: Record | null; } export interface UpdateAlertBody { name?: string; description?: string | null; scope?: 'TRACE' | 'SESSION'; aggregation?: string; metric?: string; operator?: string; threshold?: number; unit?: string | null; windowMinutes?: number; cooldownMinutes?: number; environments?: string[]; notifications?: NotificationTarget[]; filter?: Record | null; } export interface ToggleAlertBody { status: boolean; } export interface ListAlertsParams { page?: number; limit?: number; search?: string; status?: 'active' | 'paused'; scope?: 'TRACE' | 'SESSION'; metric?: string; } export interface ListAlertsResponse { data: AlertConfig[]; meta: { page: number; limit: number; totalItems: number; totalPages: number }; } export interface DeleteAlertResponse { success: boolean; } export interface Alert { // docs:params-start(alerts.create) create(body: CreateAlertBody): Promise; // docs:params-end(alerts.create) // docs:params-start(alerts.list) list(params?: ListAlertsParams): Promise; // docs:params-end(alerts.list) // docs:params-start(alerts.get) get(alertId: string): Promise; // docs:params-end(alerts.get) // docs:params-start(alerts.update) update(alertId: string, body: UpdateAlertBody): Promise; // docs:params-end(alerts.update) // docs:params-start(alerts.delete) delete(alertId: string): Promise; // docs:params-end(alerts.delete) // SDK-CONVENIENCE — preserve on regen. // Note: toggleAlert(alertId, body) exists at runtime but is intentionally // not declared here. Use pause()/resume() for status flips. // docs:params-start(alerts.pause) /** * Pause an alert without deleting its config. Convenience wrapper for * `toggleAlert(alertId, false)`. * @param alertId The alert's id (returned from `create()`). */ pause(alertId: string): Promise; // docs:params-end(alerts.pause) // docs:params-start(alerts.resume) /** * Re-enable a previously-paused alert. Convenience wrapper for * `toggleAlert(alertId, true)`. * @param alertId The alert's id (returned from `create()`). */ resume(alertId: string): Promise; // docs:params-end(alerts.resume) } // --- Agent (codegen-emitted; see src/clients/Agent) --- // // Public rename of the internal `ApiRecord` Prisma model to match the // surface customers see in the product. Wire field names are unchanged // (camelCase); only the class / route names change. See // `testops-sdk/specs/agents.http.yml` for the contract. export type HttpMethod = | "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS"; export type AgentPurpose = "DATASET_RUN" | "S3_SIGNING"; export type AgentChannel = "HTTP" | "VOICE"; export type AgentDirection = "OUTBOUND" | "INBOUND"; export type AgentAuthStatus = "ACTIVE" | "NEEDS_REAUTH" | "BROKEN"; // Key/value collection stored as an array of single-entry objects — matches // the composer's on-wire shape (`[{ "X-Foo": "bar" }, ...]`). Values are // strings; the server rejects non-string values. export type KeyValueList = Array>; // Async completion callback. Presence of ≥ 1 webhook classifies the agent // as `api_async`; the completion endpoint requires a row named // `completionWebhook`. Write-only in v1 — accepted on POST / PATCH, omitted // from GET responses. export interface AgentWebhookInput { name: string; webhookUrl?: string; method?: HttpMethod; contentType?: string; headers?: KeyValueList; queryParams?: KeyValueList; pathVariables?: KeyValueList; requestBody?: Record | null; responses?: Record | null; } // The persisted agent record. Corresponds to the internal `ApiRecord` // Prisma model. Nine JSON blobs (`headers`, `displayHeaders`, `queryParams`, // `pathVariables`, `body`, `bodyContainer`, `response`, `variableConfig`, // `callbackConfig`) are typed as passthrough on this surface; the tRPC // procedure runs the full field-level refinement. export interface AgentConfig { id: string; projectId: string; name: string; description?: string | null; type: string; method?: HttpMethod | null; url?: string | null; contentType?: string | null; includeCredentials: boolean; headers: KeyValueList; displayHeaders: KeyValueList; queryParams: KeyValueList; pathVariables: KeyValueList; body?: Record | null; bodyContainer?: Record | null; response?: Record | null; variableConfig?: Record | null; callbackConfig?: Record | null; purpose: AgentPurpose; channel: AgentChannel; direction?: AgentDirection | null; simulatorNumber?: string | null; apiEnvironmentId?: string | null; groundingDocMediaId?: string | null; authType?: string | null; authStatus?: AgentAuthStatus | null; createdAt: string; updatedAt: string; } export interface CreateAgentBody { name: string; description?: string; type: string; method: HttpMethod; url: string; contentType?: string; includeCredentials?: boolean; headers?: KeyValueList; displayHeaders?: KeyValueList; queryParams?: KeyValueList; pathVariables?: KeyValueList; body?: Record; bodyContainer?: Record; response?: Record; variableConfig?: Record | null; callbackConfig?: Record | null; apiWebhooks?: AgentWebhookInput[]; purpose?: AgentPurpose; channel?: AgentChannel; direction?: AgentDirection; simulatorNumber?: string | null; // Nullable so a `GET → POST` round-trip works — the response types this as // nullable, and PATCH already accepts `null` (H1 fix). apiEnvironmentId?: string | null; } // Partial update — every field optional EXCEPT `method` and `url`, which // the internal Zod schema still requires on every update call. Nullability // semantics on write: // - `description: null` — clears the field // - `apiEnvironmentId: null` — unbinds the environment // - `simulatorNumber: null` — clears the voice number // - `variableConfig: null` — clears the config // - `callbackConfig: null` — clears the mapper // - `apiWebhooks: []` — removes every async callback export interface UpdateAgentBody { name?: string; description?: string | null; type?: string; method: HttpMethod; url: string; contentType?: string; includeCredentials?: boolean; headers?: KeyValueList; displayHeaders?: KeyValueList; queryParams?: KeyValueList; pathVariables?: KeyValueList; body?: Record; bodyContainer?: Record; response?: Record; variableConfig?: Record | null; callbackConfig?: Record | null; apiWebhooks?: AgentWebhookInput[]; purpose?: AgentPurpose; channel?: AgentChannel; direction?: AgentDirection; simulatorNumber?: string | null; apiEnvironmentId?: string | null; } // List is UN-paginated — returns `AgentConfig[]` directly, NOT a // `{ data, meta }` wrapper. Only optional `apiEnvironmentId` filter. export interface ListAgentsParams { apiEnvironmentId?: string; } export interface DeleteAgentResponse { success: boolean; } export interface Agent { // docs:params-start(agents.create) create(body: CreateAgentBody): Promise; // docs:params-end(agents.create) // docs:params-start(agents.list) list(params?: ListAgentsParams): Promise; // docs:params-end(agents.list) // docs:params-start(agents.get) get(agentId: string): Promise; // docs:params-end(agents.get) // docs:params-start(agents.update) update(agentId: string, body: UpdateAgentBody): Promise; // docs:params-end(agents.update) // docs:params-start(agents.delete) delete(agentId: string): Promise; // docs:params-end(agents.delete) } // --- Default LLM Model --- // // Per-project singleton — no id in the URL, methods are `get()`, // `set(body)`, `delete()`. Codegen-emitted from // `testops-sdk/specs/default-llm-model.{http,sdk}.yml`. export type LlmAdapter = | "anthropic" | "openai" | "azure" | "bedrock" | "google-vertex-ai" | "google-ai-studio" | "groq" | "mistral" | "fireworks" | "databricks"; export interface DefaultLlmModelParams { max_tokens?: number; temperature?: number; top_p?: number; top_k?: number; stop_sequences?: string[]; tool_choice?: "none" | "auto" | "required"; reasoning_enabled?: boolean | "disabled" | "enabled" | "adaptive"; reasoning_budget?: number; reasoning_effort?: "low" | "medium" | "high"; use_cache?: boolean; frequency_penalty?: number; presence_penalty?: number; } export interface DefaultLlmModelConfig { projectId: string; llmApiKeyId: string; provider: string; adapter: LlmAdapter; model: string; modelParams: DefaultLlmModelParams | null; createdAt: string; updatedAt: string; } export interface SetDefaultLlmModelInput { provider: string; adapter: LlmAdapter; model: string; modelParams?: DefaultLlmModelParams | null; } export interface DeleteDefaultLlmModelResponse { success: boolean; alreadyDeleted: boolean; } export interface DefaultLlmModel { // docs:params-start(default-llm-model.get) get(): Promise; // docs:params-end(default-llm-model.get) // docs:params-start(default-llm-model.set) set(body: SetDefaultLlmModelInput): Promise; // docs:params-end(default-llm-model.set) // docs:params-start(default-llm-model.delete) delete(): Promise; // docs:params-end(default-llm-model.delete) } // --- Traces (codegen-emitted; see src/clients/Traces) --- export interface TraceUsage { promptTokens: number; completionTokens: number; totalTokens: number; } export interface Trace { id: string; externalId: string | null; timestamp: string; name: string | null; userId: string | null; metadata: unknown; release: string | null; version: string | null; projectId: string; environment: string; isPublic: boolean; bookmarked: boolean; tags: string[]; input: unknown; output: unknown; sessionId: string | null; customId: string | null; userGeneratedId: string | null; experimentRunId: string | null; expectedOutput: unknown; context: unknown; retrievedContexts: unknown; referenceContexts: unknown; multiResponses: unknown; rubrics: unknown; usage?: TraceUsage; createdAt: string; updatedAt: string; } // LIST-shape trace returned by `client.traces.list()`. Runtime: // src/clients/Traces/types.ts. export interface ExtendedTrace extends Trace { observations: string[] | null; scores: string[] | null; totalCost: number | null; latency: number | null; htmlPath: string; } export type TraceObservationLevel = 'DEBUG' | 'DEFAULT' | 'WARNING' | 'ERROR'; export interface TraceObservation { id: string; projectId: string; traceId: string | null; parentObservationId: string | null; name: string | null; type: string; environment: string; startTime: string; endTime: string | null; input: unknown; output: unknown; metadata: unknown; level: TraceObservationLevel; statusMessage: string | null; model: string | null; modelParameters: unknown; completionStartTime: string | null; promptId: string | null; promptName: string | null; promptVersion: number | null; usageDetails: Record; costDetails: Record; calculatedInputCost: number | null; calculatedOutputCost: number | null; calculatedTotalCost: number | null; latency: number | null; timeToFirstToken: number | null; version: string | null; createdAt: string; updatedAt: string; } export type TraceScoreDataType = 'NUMERIC' | 'CATEGORICAL' | 'BOOLEAN' | 'TEXT'; export interface TraceScore { id: string; traceId: string; observationId: string | null; name: string; value: number | string | null; stringValue: string | null; dataType: TraceScoreDataType; source: string; comment: string | null; metadata: unknown; configId: string | null; environment: string; timestamp: string; createdAt: string; updatedAt: string; authorUserId: string | null; queueId: string | null; tracePath: string | null; } /** Full hydrated trace returned by `client.traces.get(traceId)`. */ export interface TraceDetail extends Trace { observations: TraceObservation[]; scores: TraceScore[]; totalCost: number; latency: number; htmlPath: string; } export interface ListTracesResponse { data: ExtendedTrace[]; meta: PaginationMeta; } export interface ListTracesParams { page?: number; limit?: number; userId?: string; name?: string; tags?: string; environment?: string; sessionId?: string; version?: string; release?: string; fromTimestamp?: string; toTimestamp?: string; orderBy?: string; searchQuery?: string; searchType?: string; filter?: string; } export type TraceFilterOperator = | 'EQUALS' | 'NOT_EQUALS' | 'CONTAINS' | 'NOT_CONTAINS' | 'STARTS_WITH' | 'ENDS_WITH' | 'IN' | 'NOT_IN' | 'GREATER_THAN' | 'LESS_THAN' | 'GREATER_THAN_OR_EQUAL' | 'LESS_THAN_OR_EQUAL' | 'IS_EMPTY' | 'IS_NOT_EMPTY'; // `TraceFilterColumn` is defined above (near line 1193) and reused here. export interface TraceFilter { fieldName: string; operator: TraceFilterOperator; value?: unknown; } export interface TraceFilterSet { must?: TraceFilter[]; any?: TraceFilter[]; } export interface PublishTraceBody { isPublic: boolean; } export interface UpdateTraceTagsBody { tags: string[]; } export type AddToDatasetBodyMappingType = 'EXPECTED_OUTPUT' | 'LLM_OUTPUT'; export type AddToDatasetBodyExtractionMode = 'SINGLE_IO' | 'FULL_CONVERSATION'; export interface AddToDatasetBody { datasetId: string; traceIds?: string[]; filter?: TraceFilterSet; datasetTag?: string | null; mappingType?: AddToDatasetBodyMappingType; extractionMode?: AddToDatasetBodyExtractionMode; outputObservationName?: string | null; } export interface AddToEvalDatasetBody { datasetId: string; traceIds?: string[]; filter?: TraceFilterSet; datasetTag?: string | null; mappingType?: AddToDatasetBodyMappingType; extractionMode?: AddToDatasetBodyExtractionMode; outputObservationName?: string | null; } export interface EnqueueDatasetItemsResponse { datasetId: string; datasetName: string; // Omitted on Path B (filter): matched-set size isn't known at request time. traceCount?: number; } export interface Traces { // docs:params-start(traces.list) list(params?: ListTracesParams): Promise; // docs:params-end(traces.list) // docs:params-start(traces.get) get(traceId: string): Promise; // docs:params-end(traces.get) // docs:params-start(traces.publish) publish(traceId: string, body: PublishTraceBody): Promise; // docs:params-end(traces.publish) // docs:params-start(traces.tag) tag(traceId: string, body: UpdateTraceTagsBody): Promise; // docs:params-end(traces.tag) // docs:params-start(traces.addToDataset) addToDataset(body: AddToDatasetBody): Promise; // docs:params-end(traces.addToDataset) // docs:params-start(traces.addToEvalDataset) addToEvalDataset(body: AddToEvalDatasetBody): Promise; // docs:params-end(traces.addToEvalDataset) } // --- AutomationRule (codegen-emitted; see src/clients/AutomationRule) --- export type AutomationRuleStatus = 'ACTIVE' | 'PAUSED'; export type AutomationRuleActionType = 'ONLINE_EVAL' | 'DATASET'; export type OnlineEvalActionInputType = 'ONLINE_EVAL'; export type DatasetActionInputType = 'DATASET'; export type TargetingFilterOperator = | 'EQUALS' | 'NOT_EQUALS' | 'CONTAINS' | 'NOT_CONTAINS' | 'STARTS_WITH' | 'ENDS_WITH' | 'REGEX' | 'IN' | 'NOT_IN' | 'GREATER_THAN' | 'LESS_THAN' | 'GREATER_THAN_OR_EQUAL' | 'LESS_THAN_OR_EQUAL'; export type TargetingFilterLogicalOperator = 'AND' | 'OR'; export interface TargetingFilter { fieldName: string; operator: TargetingFilterOperator; value: string; logicalOperator?: TargetingFilterLogicalOperator; } export interface OnlineEvalActionConfigInput { concurrency?: number; delaySeconds?: number | null; eventName?: string | null; } export interface OnlineEvalActionConfigResponse { concurrency: number; delaySeconds: number | null; eventName: string | null; } export interface OnlineEvalMetricInput { evaluatorListId: string; routeToHumanReview?: boolean; } export interface OnlineEvalMetricResponse { id: string; evaluatorListId: string; evaluatorListName: string | null; routeToHumanReview: boolean; annotationQueueId: string | null; } export interface MetricRoutingToggle { evaluatorListId: string; routeToHumanReview: boolean; } export interface DatasetActionConfigInput { datasetId: string; mappingType?: DatasetMappingType; extractionMode?: DatasetExtractionMode; datasetTag?: string | null; outputObservationName?: string | null; } export interface DatasetActionConfigResponse { datasetId: string; mappingType: DatasetMappingType; extractionMode: DatasetExtractionMode; datasetTag: string | null; outputObservationName: string | null; } export interface OnlineEvalActionInput { type: OnlineEvalActionInputType; config: OnlineEvalActionConfigInput; metrics: OnlineEvalMetricInput[]; } export interface OnlineEvalActionUpdateInput { type: OnlineEvalActionInputType; config: OnlineEvalActionConfigInput; metrics?: MetricRoutingToggle[]; } export interface DatasetActionInput { type: DatasetActionInputType; config: DatasetActionConfigInput; } export type AutomationRuleActionInput = OnlineEvalActionInput | DatasetActionInput; export type AutomationRuleActionUpdateInput = OnlineEvalActionUpdateInput | DatasetActionInput; export interface OnlineEvalAction { id: string; type: OnlineEvalActionInputType; config: OnlineEvalActionConfigResponse; metrics: OnlineEvalMetricResponse[]; createdAt: string; updatedAt: string; } export interface DatasetAction { id: string; type: DatasetActionInputType; config: DatasetActionConfigResponse; createdAt: string; updatedAt: string; } export type AutomationRuleAction = OnlineEvalAction | DatasetAction; export interface AutomationRuleConfig { id: string; projectId: string; name: string; description: string | null; sourceType: EventSchedulerType; samplingRate: number; status: AutomationRuleStatus; targetingFilters: TargetingFilter[]; actions: AutomationRuleAction[]; createdBy: string | null; createdAt: string; updatedAt: string; } export interface CreateAutomationRuleBody { name: string; description?: string | null; sourceType?: EventSchedulerType; samplingRate?: number; status?: AutomationRuleStatus; targetingFilters?: TargetingFilter[]; actions: AutomationRuleActionInput[]; } export interface UpdateAutomationRuleBody { name?: string; description?: string | null; samplingRate?: number; status?: AutomationRuleStatus; targetingFilters?: TargetingFilter[]; } export interface UpdateAutomationRuleStatusBody { status: AutomationRuleStatus; } export interface AddAutomationRuleActionBody { action: AutomationRuleActionInput; } export interface UpdateAutomationRuleActionBody { action: AutomationRuleActionUpdateInput; } export interface ListAutomationRulesParams { page?: number; limit?: number; search?: string; actionType?: AutomationRuleActionType; } export interface ListAutomationRulesResponse { data: AutomationRuleConfig[]; meta: PaginationMeta; } export interface DeleteAutomationRuleResponse { success: boolean; } export interface SuccessAck { success: boolean; } export interface AutomationRule { // docs:params-start(automation-rules.create) create(body: CreateAutomationRuleBody): Promise; // docs:params-end(automation-rules.create) // docs:params-start(automation-rules.list) list(params?: ListAutomationRulesParams): Promise; // docs:params-end(automation-rules.list) // docs:params-start(automation-rules.get) get(ruleId: string): Promise; // docs:params-end(automation-rules.get) // docs:params-start(automation-rules.update) update(ruleId: string, body: UpdateAutomationRuleBody): Promise; // docs:params-end(automation-rules.update) // docs:params-start(automation-rules.delete) delete(ruleId: string): Promise; // docs:params-end(automation-rules.delete) // docs:params-start(automation-rules.updateStatus) updateStatus(ruleId: string, body: UpdateAutomationRuleStatusBody): Promise; // docs:params-end(automation-rules.updateStatus) // docs:params-start(automation-rules.addAction) addAction(ruleId: string, body: AddAutomationRuleActionBody): Promise; // docs:params-end(automation-rules.addAction) // docs:params-start(automation-rules.updateAction) updateAction( ruleId: string, actionId: string, body: UpdateAutomationRuleActionBody, ): Promise; // docs:params-end(automation-rules.updateAction) // docs:params-start(automation-rules.deleteAction) deleteAction(ruleId: string, actionId: string): Promise; // docs:params-end(automation-rules.deleteAction) } export interface ScenarioDatasets { create(request: CreateScenarioDatasetRequest): Promise; list(page?: number, limit?: number): Promise>; hasAny(): Promise; get(name: string): Promise; delete(name: string): Promise; lastUsedAgent(name: string): Promise; listVersions(name: string, page?: number, limit?: number): Promise>; } export interface ScenarioItems { listForDataset(datasetName: string, page?: number, limit?: number): Promise>; create(datasetName: string, request: CreateScenarioItemRequest): Promise; createBatch(datasetName: string, request: BatchCreateScenarioItemsRequest): Promise; generatePreview(datasetName: string, request: GeneratePreviewRequest): Promise; get(itemId: string): Promise; update(itemId: string, patch: UpdateScenarioItemRequest): Promise; delete(itemId: string): Promise; listRuns(itemId: string, page?: number, limit?: number): Promise>; } export interface ScenarioRuns { trigger(datasetName: string, request: TriggerRunRequest): Promise; get(datasetName: string, runId: string): Promise; listItems(datasetName: string, runId: string, page?: number, limit?: number): Promise>; metrics(datasetName: string, runId: string): Promise; batchMetrics(datasetName: string, request: BatchMetricsRequest): Promise; sessionContext(sessionId: string): Promise; } /** Standalone Experiments object — reads credentials from env vars. */ export declare const Experiments: ExperimentsAPI; export type MediaContentType = "image/png" | "image/jpeg" | "image/jpg" | "image/webp" | "image/gif" | "image/svg+xml" | "image/tiff" | "image/bmp" | "audio/mpeg" | "audio/mp3" | "audio/wav" | "audio/ogg" | "audio/oga" | "audio/aac" | "audio/mp4" | "audio/flac" | "video/mp4" | "video/webm" | "text/plain" | "text/html" | "text/css" | "text/csv" | "application/pdf" | "application/msword" | "application/vnd.ms-excel" | "application/zip" | "application/json" | "application/xml" | "application/octet-stream"; export declare class Media { constructor(options?: { contentType?: MediaContentType; base64DataUri?: string; contentBytes?: Buffer | Uint8Array; filePath?: string; unsignedUrl?: string; data?: Buffer | Uint8Array; url?: string }); readonly contentLength?: number; readonly contentSha256Hash?: string; readonly obj?: object; readonly unsignedUrl?: string; toJSON(): string; static parseReferenceString(referenceString: string): { mediaId: string; source: string; contentType: string }; } export declare class Prompt { /** Pass keys explicitly or set `AISDK_PUBLIC_KEY` / `AISDK_SECRET_KEY` env vars. */ constructor(publicKey?: string, secretKey?: string); create(body: ExtendedCreateChatPromptBodyWithPlaceholders): Promise; create(body: ExtendedCreateTextPromptBody): Promise; create(body: ExtendedCreateChatPromptBody): Promise; create(body: ExtendedCreateChatPromptBodyMultimodal): Promise; get(name: string, version?: number, options?: { label?: string; cacheTtlSeconds?: number; fallback?: string; maxRetries?: number; type?: "text"; fetchTimeoutMs?: number }): Promise; get(name: string, version?: number, options?: { label?: string; cacheTtlSeconds?: number; fallback?: ChatMessage[]; maxRetries?: number; type: "chat"; fetchTimeoutMs?: number }): Promise; update(body: { name: string; version: number; newLabels: string[] }): Promise; } export interface ProviderToolResult { compile(options?: ToolCompileOptions): CompiledTool; toData(): Record; } export declare class Tool { /** Pass keys explicitly or set `AISDK_PUBLIC_KEY` / `AISDK_SECRET_KEY` env vars. */ constructor(publicKey?: string, secretKey?: string); readonly baseUrl: any; readonly publicKey: any; readonly secretKey: any; create(options: CreateToolOptions): Promise; get(name: string, provider?: string, options?: { version?: number; label?: string; cacheTtlSeconds?: number }): Promise; list(limit?: number, cursor?: string, label?: string): Promise; update(options: UpdateToolOptions): Promise; } export interface ToolInstance { readonly id: string; readonly name: string; readonly description?: string; readonly parameters: ToolParameters; readonly version: number; readonly labels: string[]; readonly commitMessage?: string; readonly createdAt: string; readonly updatedAt: string; readonly isRunnable?: boolean; readonly sampleOutput?: Record | null; compile(options?: ToolCompileOptions): CompiledTool; toData(): LlmTool; toString(): string; } export interface ToolList { readonly length: number; compile(options?: ToolCompileOptions): CompiledTool[]; } // --------------------------------------------------------------------------- // MCP Server Registry // --------------------------------------------------------------------------- export type McpAuthConfigInput = | { kind: 'none' } | { kind: 'bearer'; token: string } | { kind: 'header'; headers: Array<{ name: string; value: string }> } | { kind: 'query_param'; paramName: string; value: string } | { kind: 'oauth' }; export type McpServerStatus = 'ACTIVE' | 'NEEDS_REAUTH' | 'BROKEN'; export type McpAuthType = | 'NONE' | 'BEARER' | 'HEADER' | 'QUERY_PARAM' | 'OAUTH'; export interface McpServer { id: string; name: string; url: string; authType: McpAuthType; status: McpServerStatus; createdAt: string; updatedAt: string; authConfigRedacted?: unknown; oauthIdentity?: { authenticatedBy: string | null; authenticatedAt: string | null; } | null; lastHealthCheckAt?: string | null; lastHealthError?: string | null; } export interface CreateMcpServerOptions { name: string; url: string; authConfig: McpAuthConfigInput; } export interface UpdateMcpServerOptions { id: string; name?: string; url?: string; authConfig?: McpAuthConfigInput; } export interface ListMcpServersResponse { data: McpServer[]; } export type McpServerAttachmentInput = | { serverName: string; serverId?: string; disabledTools?: string[] } | { serverId: string; serverName?: string; disabledTools?: string[] }; export interface PromptMcpServerRef { serverId: string; serverName: string; url: string; disabledTools: string[]; } export declare class McpServers { /** Pass keys explicitly or set `AISDK_PUBLIC_KEY` / `AISDK_SECRET_KEY` env vars. */ constructor(publicKey?: string, secretKey?: string); list(): Promise; get(options: { id: string } | { name: string }): Promise; create(options: CreateMcpServerOptions): Promise; update(options: UpdateMcpServerOptions): Promise; delete(options: { id: string }): Promise; revokeOAuth(options: { id: string }): Promise; } // --------------------------------------------------------------------------- // SDK Types & Interfaces // --------------------------------------------------------------------------- export interface CreateCodeEvaluatorOptions { name: string; code: string; language?: string; [key: string]: any; } export interface CreateDatasetRunItemRequest { id?: string; input?: any; output?: any; expectedOutput?: any; metadata?: any; sourceTraceId?: string; sourceObservationId?: string; status?: string; traceId?: string; observationId?: string; } export interface CreateDatasetRunItemsResponse { success: boolean; itemCount: number; } export interface CreateEvaluatorListRequest { name: string; evaluators: any[]; [key: string]: any; } export interface CreateEvaluatorListResponse { id: string; [key: string]: any; } export interface CreateExperimentRunRequest { experimentId: string; [key: string]: any; } /** * Typed replacement for the previously-untyped `runConfig: Record` * on `experimentRuns.create`. Extensible via index signature so unknown * fields still round-trip. */ export interface RunConfig { provider?: string; adapter?: LLMAdapter | string; /** Per-run LLM override — overrides the experiment's default LLM config. */ model?: string; /** * Per-agent-call timeout in ms. Positive, must be ≤ server cap * (default 120000ms). */ apiCallTimeoutMs?: number; [key: string]: any; } /** * Object-form options for `experimentRuns.create`. Preferred over the * legacy positional signature — clearer, forward-compatible, and carries * the `wait` sugar for create-and-poll flows. */ export interface CreateExperimentRunOptions { experimentId: string; concurrency?: number; runConfig?: RunConfig; metadata?: Record; /** * Create then poll to a terminal status. `true` uses defaults (~15 min * total, 5s poll); an object tunes the timeout / interval. When set, * the return type is `ExperimentRunSubscriptionResult`. */ wait?: boolean | { timeoutMs?: number; pollIntervalMs?: number }; } export interface CreateExperimentWithPromptRequest { name: string; description?: string; promptId?: string; /** Wire name: `apiRecordId`. XOR with `promptId`. */ agentId?: string; /** * @deprecated Use `agentId`. Retained as a wire-compatible alias — the * SDK forwards it as-is if `agentId` is not set. */ apiRecordId?: string; datasetId: string; datasetVersion?: number | null; /** Pinned dataset label. Server resolves to a version. XOR with datasetVersion. */ datasetLabel?: string; evaluatorListId: string; concurrency?: number; /** * Required when the chosen dataset's `kind = SCENARIO`; rejected * otherwise. Public wire name: `simulatorConfig` — boundary-mapped * server-side to the internal `scenarioConfig` column. */ simulatorConfig?: SimulatorConfig; } export interface CreateExperimentWithTagRequest { name: string; description?: string; datasetRunTagId: string; evaluatorListId: string; concurrency?: number; } export interface CreateLLMCustomEvaluatorOptions { name: string; prompt: string; model?: string; /** * AIOPS-1881: per-evaluator opt-in to strict variable validation. * Default `false` (lenient — empty-string substitution on missing * `{{var}}` references). Only meaningful for `llm_custom`. */ strictVariables?: boolean; [key: string]: any; } export interface CreateToolOptions { name: string; description: string; parameters?: ToolParameters; strict?: boolean; sampleOutput?: Record | null; isRunnable?: boolean; labels?: string[]; commitMessage?: string | null; } export interface DatasetRunItemResponse { id: string; datasetRunId: string; datasetItemId: string; traceId: string; observationId?: string; [key: string]: any; } export interface DatasetRunResponse { id: string; name: string; datasetId: string; datasetName: string; [key: string]: any; } export interface EnrichedEvaluatorResponse { id: string; name: string; /** * AIOPS-1881: current strict-variables setting for USER_PROMPT * evaluators. Absent on partner / code / built-in templates. */ strictVariables?: boolean; [key: string]: any; } export interface EvaluationExecutionRequest { evaluators: Array<{ metricName: string; displayName?: string; family?: string; runtimeProvider?: string; internalName?: string; params?: Record; }>; provider?: string; model?: string; data: { input?: string; output?: string; context?: string[]; expectedOutput?: string; retrievedContexts?: string[]; referenceContexts?: string[]; rubrics?: Record; }; concurrency?: number; } export interface EvaluationExecutionResponse { success: boolean; results: Array<{ evaluator: string; success: boolean; score: number | null; reasoning: string | null; error: string | null; metadata?: Record; }>; jobId: string; timestamp: string; error?: string; } export interface EvaluatorConfigResponse { id: string; [key: string]: any; } export interface EvaluatorListResponse { id: string; name: string; evaluators: any[]; [key: string]: any; } export interface EvaluatorModelParams { provider: string; model: string; [key: string]: any; } export interface EvaluatorParamResponse { name: string; dataType: string; [key: string]: any; } export interface EvaluatorResponseBase { id: string; name: string; /** * AIOPS-1881: current strict-variables setting for USER_PROMPT * evaluators. Absent on partner / code / built-in templates. */ strictVariables?: boolean; [key: string]: any; } export interface EvaluatorSummary { id: string; name: string; [key: string]: any; } export interface EvaluatorThreshold { metric: string; value: number; [key: string]: any; } export interface ExperimentResponse { id: string; name: string; [key: string]: any; } export type PersonaCohort = 'cooperative' | 'challenging'; export interface CreatePersonaRequest { name: string; cohort: PersonaCohort; systemPrompt: string; description?: string; traits?: string[]; } export interface Persona { id: string; name: string; description: string | null; cohort: PersonaCohort; traits: string[]; systemPrompt: string; isGlobal: boolean; isDefault: boolean; createdAt: string; updatedAt: string; } export interface ListPersonasResponse { data: Persona[]; } export interface DeletePersonaResponse { id: string; } export interface PageMeta { page: number; limit: number; totalItems: number; totalPages: number; } export interface Paginated { data: T[]; meta: PageMeta; } export interface ScenarioDataset { id: string; name: string; description: string | null; currentVersion: number; itemCount: number; createdAt: string; updatedAt: string; } export interface CreateScenarioDatasetRequest { name: string; description?: string; } export interface CreateScenarioDatasetResponse { id: string; name: string; } export interface DeleteScenarioDatasetResponse { id: string; } export interface HasAnyResponse { hasAny: boolean; } export interface LastUsedAgentResponse { agentId: string | null; } export interface ScenarioVersionLabel { id: string; name: string; } export interface ScenarioVersionCreator { id: string; name: string | null; } export interface ScenarioVersion { id: string; version: number; datasetId: string; createdAt: string; labels: ScenarioVersionLabel[]; createdBy: ScenarioVersionCreator | null; } export type ScenarioItemStatus = 'ACTIVE' | 'ARCHIVED'; export type ScenarioAddedFrom = 'MANUAL' | 'SYNTHETIC_SEED'; export type ScenarioRunStatus = 'PENDING' | 'RUNNING' | 'COMPLETED' | 'FAILED' | 'STOPPED'; export type LLMAdapter = 'anthropic' | 'openai' | 'azure' | 'bedrock' | 'google-vertex-ai' | 'google-ai-studio' | 'groq' | 'mistral' | 'fireworks' | 'databricks'; export interface ScenarioConfig { personaRef: string; goal: string; openingMessage: string; maxTurns?: number; } export interface ScenarioItem { id: string; status: ScenarioItemStatus; scenarioConfig: ScenarioConfig; expectedOutput: string | null; addedFrom: ScenarioAddedFrom; currentVersion: number; createdAt: string; updatedAt: string; } export interface CreateScenarioItemRequest { scenarioConfig: ScenarioConfig; expectedOutput?: string; } export interface CreateScenarioItemResponse { id: string; } export interface BatchCreateScenarioItemsRequest { items: CreateScenarioItemRequest[]; } export interface BatchCreateScenarioItemsResponse { count: number; } export interface UpdateScenarioItemRequest { scenarioConfig?: ScenarioConfig; expectedOutput?: string; } export interface UpdateScenarioItemResponse { id: string; } export interface DeleteScenarioItemResponse { id: string; } export interface GeneratePreviewRequest { personaIds: string[]; agentId: string; provider: string; adapter: LLMAdapter | string; model: string; count: number; instructions?: string; } export interface GeneratedScenario { personaRef: string; goal: string; openingMessage: string; expectedOutput?: string; } export interface GeneratePreviewResponse { scenarios: GeneratedScenario[]; } export interface SimulatorConfig { provider: string; adapter: LLMAdapter | string; model: string; maxTurns: number; } export interface ScenarioRun { id: string; name: string; description: string | null; status: ScenarioRunStatus; datasetVersion: number | null; apiRecordId: string | null; createdAt: string; updatedAt: string; } export interface ScenarioRunItem { id: string; datasetItemId: string; datasetRunId: string; datasetRunName: string; datasetVersion: number | null; sessionId: string | null; traceId: string | null; status: ScenarioRunStatus; error: string | null; output: unknown; expectedOutput: unknown | null; persona: string | null; goal: string | null; openingMessage: string | null; completedTurns?: number; stoppedReason?: string; sessionLatencyMs: number | null; sessionCostUsd: number | null; sessionInputTokens: number | null; sessionOutputTokens: number | null; sessionTotalTokens: number | null; sessionTurnCount: number | null; createdAt: string; updatedAt: string; } export interface ScenarioMetrics { totalItems: number; completedItems: number; failedItems: number; sessionsMeasured: number; avgLatencyMs: number | null; avgCostUsd: number | null; totalCostUsd: number; avgTokens: number | null; totalTokens: number; } export interface ScenarioSessionPersona { id: string; name: string; cohort: 'cooperative' | 'challenging'; description: string | null; } export interface ScenarioSessionContext { persona: ScenarioSessionPersona | null; goal: string | null; openingMessage: string | null; } export interface TriggerRunRequest { agentId: string; simulatorConfig: SimulatorConfig; name?: string; description?: string; datasetVersion?: number; concurrency?: number; } export interface TriggerRunResponse { runId: string; itemCount: number; } export interface BatchMetricsRequest { runIds: string[]; } export interface BatchMetricsEntry { avgLatencyMs: number; avgCostUsd: number; totalCostUsd: number; } export interface BatchMetricsResponse { byRunId: Record; } export interface ExperimentRunResponse { id: string; experimentId: string; status: ExperimentRunStatus; /** * Canonical TestAI UI URL for this run. Provided by the backend so * clients don't have to construct it from projectId/experimentId/runId * (route changes won't break consumers). May be null when the backend * can't determine its public web URL. */ uiUrl?: string | null; [key: string]: any; } export interface ExperimentRunComparisonEvaluator { name: string; currentScore: number | null; baselineScore: number | null; delta: number | null; deltaDirection: ComparisonDeltaDirection | null; threshold: { value: number; criteria: string } | null; thresholdStatus: ComparisonThresholdStatus | null; } export interface ExperimentRunComparisonResponse { status: ComparisonStatus; markdown?: string; current: { runId: string; runUiUrl: string | null; rowCount: number; }; baseline: { runId: string; runUiUrl: string | null; rowCount: number; } | null; evaluators: ExperimentRunComparisonEvaluator[]; summary: { verdict: ComparisonVerdict; failureCount: number; improvedCount: number; regressedCount: number; }; } export interface NumericScoreSummary { name: string; dataType: 'NUMERIC'; score: number | null; minScore?: number; maxScore?: number; p90Score?: number; passCount: number; failCount: number; totalItems: number; threshold: number | null; criteria: string | null; thresholdStatus: 'pass' | 'fail' | null; diff: number | null; improvements: number; regressions: number; source: 'EVAL' | 'ANNOTATION'; version?: number; versionLabel?: string; } export interface CategoricalScoreSummary { name: string; dataType: 'CATEGORICAL'; score: null; categoryCounts: Record; passCategories: string[]; ignoreCategories: string[]; neutralCount?: number; totalItems: number; threshold: null; criteria: null; thresholdStatus: 'pass' | 'fail' | null; diff: null; improvements: number; regressions: number; source: 'EVAL' | 'ANNOTATION'; } export type ExperimentRunScoreSummary = NumericScoreSummary | CategoricalScoreSummary; export interface ExperimentRunMetricSummary { name: string; metric: number; unit: string; diff: number | null; } export interface ExperimentRunSummary { projectId: string; projectName: string; experimentId: string; experimentName: string; experimentRunId: string; experimentRunName: string; comparisonExperimentRunId: string | null; comparisonExperimentRunName: string | null; urls: { project: string | null; experiment: string | null; run: string | null; scoresTab: string | null; analyticsTab: string | null; }; scores: ExperimentRunScoreSummary[]; metrics: Record; summary: { verdict: 'PASS' | 'REGRESSION'; failureCount: number; improvedCount: number; regressedCount: number; }; } export interface ExperimentRunSubscriptionResult { finalStatus: 'COMPLETED' | 'FAILED' | 'CANCELLED' | 'TIMEOUT'; experimentRunData: ExperimentRunResponse; } export interface GetDatasetRunByTagResponse extends DatasetRunResponse {} export interface GetDatasetRunTagsResponse { data: Array>; meta: Record; } export interface DeleteDatasetRunResponse { message: string; } export interface GetDatasetTagsResponse { tags: string[]; [key: string]: any; } export interface ListDatasetRunItemsResponse { data: any[]; meta: { page: number; limit: number; totalItems: number; totalPages: number }; } export interface ListDatasetRunsResponse { data: any[]; meta: { page: number; limit: number; totalItems: number; totalPages: number }; } export interface ListEvaluatorListsQuery { limit?: number; page?: number; orderBy?: { column?: string; order?: "ASC" | "DESC" }; } export interface ListEvaluatorListsResponse { evaluators: EvaluatorListResponse[]; totalCount: number; } export interface ListEvaluatorsOptions { page?: number; limit?: number; [key: string]: any; } export interface ListEvaluatorsResponse { evaluators: EnrichedEvaluatorResponse[]; totalCount: number; } export interface ListExperimentRunsResponse { experimentRuns: ExperimentRunResponse[]; totalCount: number; } export interface ListExperimentsResponse { experiments: ExperimentResponse[]; totalCount: number; } export interface ListToolsResponse { data: LlmTool[]; meta: { nextCursor: string | null; totalCount: number; }; } export interface ListVersionsOptions { page?: number; limit?: number; [key: string]: any; } export interface ListVersionsResponse { versions: EnrichedEvaluatorResponse[]; nextCursor?: string; } export interface LlmTool { name: string; description?: string; parameters?: Record; [key: string]: any; } export interface LlmToolInput { name: string; [key: string]: any; } export interface ModelParams { provider: string; model: string; adapter: "anthropic" | "openai" | "azure" | "bedrock" | "google-vertex-ai" | "google-ai-studio" | "groq" | "mistral" | "fireworks" | "databricks"; temperature?: number; max_tokens?: number; max_completion_tokens?: number; top_p?: number; top_k?: number; stop_sequences?: string[]; tool_choice?: "auto" | "none" | "required"; reasoning_enabled?: boolean; use_cache?: boolean; frequency_penalty?: number; presence_penalty?: number; } export interface SetLabelsResponse { [key: string]: any; } // ChatPromptClient and TextPromptClient already include tools export interface SDKMetadata { resourceAttributes: { 'telemetry.sdk.version': string; 'telemetry.sdk.name': string; }; } export interface ToolCompileOptions { strings?: Record; objects?: Record; } export interface UpdateEvaluatorOptions { name?: string; /** * AIOPS-1881: toggle strict variable validation on the evaluator * template. Omit to leave unchanged; pass `true`/`false` to set. * Server ignores the flag for non-USER_PROMPT evaluators. */ strictVariables?: boolean; [key: string]: any; } export interface UpdateExperimentRunRequest { [key: string]: any; } export interface UpdateToolOptions { name: string; [key: string]: any; } // --------------------------------------------------------------------------- // Dataset types // --------------------------------------------------------------------------- export type DatasetKind = "REGULAR" | "EVALUATION"; export interface DatasetResponse { id: string; name: string; kind?: DatasetKind; [key: string]: any; } export interface ListDatasetsResponse { data: any[]; meta: { page: number; limit: number; totalItems: number; totalPages: number }; } export interface CreateItemsRequest { datasetName: string; items: any[]; [key: string]: any; } export interface CreateItemsOptions { [key: string]: any; } export interface CreateDatasetItemsResponse { items: any[]; [key: string]: any; } export type DatasetItemSource = "MANUAL" | "SYNTHETIC_SEED" | "SYNTHETIC_TRACE"; export interface GeneratedDatasetItem { input: string; expectedOutput: string; } export interface GenerateDatasetItemsResponse { items: GeneratedDatasetItem[]; } export interface GenerateModel { provider: string; adapter: string; model: string; } export interface GenerateItemsParams { taskDescription: string; count: number; model: GenerateModel; signal?: AbortSignal; } export interface GenerateMoreItemsParams { datasetName: string; count: number; model: GenerateModel; referenceItemIds?: string[]; refinementInstruction?: string; signal?: AbortSignal; } export interface ListDatasetItemsResponse { data: DatasetItemResponse[]; meta: PaginationMeta; } export interface CreateEvaluatorOptions { name: string; [key: string]: any; } export interface ArchiveEvaluatorResponse { [key: string]: any; } export interface CreateWithPromptRequest { name: string; promptName: string; promptVersion?: string; description?: string; metadata?: Record; } // --------------------------------------------------------------------------- // Decorator config type + with*/decorator declarations // --------------------------------------------------------------------------- export interface DecoratorConfig { name?: string; // Generation-only fields (ignored by other span kinds). Lets callers // stamp model + model parameters onto a @generation/withGeneration // span without going through the manual client.generation({...}) API. model?: string; modelParameters?: Record; [key: string]: any; } // HOF form — wrap any function so its invocation is captured as the // named span kind. Returns the function's result (or Promise of it). export function withWorkflow(config: DecoratorConfig, fn: (...args: any[]) => R, ...args: any[]): R; export function withTask(config: DecoratorConfig, fn: (...args: any[]) => R, ...args: any[]): R; export function withAgent(config: DecoratorConfig, fn: (...args: any[]) => R, ...args: any[]): R; export function withTool(config: DecoratorConfig, fn: (...args: any[]) => R, ...args: any[]): R; // RESERVED for LLM auto-instrumentation. Prefer wrapping an instrumented // provider (Anthropic, OpenAI, Bedrock, ...). Use this only for porting // from langfuse-style @observe(as_type="generation"). export function withGeneration(config: DecoratorConfig, fn: (...args: any[]) => R, ...args: any[]): R; // Decorator form — apply to a class method. Requires // "experimentalDecorators": true in tsconfig. export function workflow(config?: Partial | ((thisArg: unknown, ...funcArgs: unknown[]) => Partial)): MethodDecorator; export function task(config?: Partial | ((thisArg: unknown, ...funcArgs: unknown[]) => Partial)): MethodDecorator; export function agent(config?: Partial | ((thisArg: unknown, ...funcArgs: unknown[]) => Partial)): MethodDecorator; export function tool(config?: Partial | ((thisArg: unknown, ...funcArgs: unknown[]) => Partial)): MethodDecorator; // RESERVED — see withGeneration note above. export function generation(config?: Partial | ((thisArg: unknown, ...funcArgs: unknown[]) => Partial)): MethodDecorator; // --------------------------------------------------------------------------- // Compiled tool type // --------------------------------------------------------------------------- export interface CompiledTool { [key: string]: any; } // --------------------------------------------------------------------------- // Tool parameter type // --------------------------------------------------------------------------- export interface ToolParameters { [key: string]: any; } // Content part types export interface TextContentPart { type: 'text'; text: string; } export interface ImageContentPart { type: 'input_image'; image_url: string | { url: string }; } export interface DocumentContentPart { type: 'document'; source: { type: 'url'; url: string }; } export type MessageContentPart = TextContentPart | ImageContentPart | DocumentContentPart; // Message types export interface SimpleChatMessage { role: string; content: string; cache_control?: { type: "ephemeral"; ttl?: "5m" | "1h" }; } export interface UserMessage { type: 'user'; role: 'user'; content: string | MessageContentPart[]; cache_control?: { type: "ephemeral"; ttl?: "5m" | "1h" }; } export interface PlaceholderMessage { type: 'placeholder'; name: string; } export type PromptMessage = SimpleChatMessage | UserMessage | PlaceholderMessage; // Extended prompt body types export interface ExtendedCreateTextPromptBody extends CreateTextPromptBody { modelParams?: ModelParams; toolList?: ToolRef[]; mcpServers?: McpServerAttachmentInput[]; } export interface ExtendedCreateChatPromptBody extends CreateChatPromptBody { modelParams?: ModelParams; toolList?: ToolRef[]; mcpServers?: McpServerAttachmentInput[]; } export interface ExtendedCreateChatPromptBodyWithPlaceholders { type: 'chat'; name: string; prompt: (ChatMessage | PlaceholderMessage)[]; labels?: string[]; config?: Record; modelParams?: ModelParams; toolList?: ToolRef[]; mcpServers?: McpServerAttachmentInput[]; tags?: string[]; isActive?: boolean; commitMessage?: string; structuredOutput?: StructuredOutputSnapshot | null; } export interface ExtendedCreateChatPromptBodyMultimodal { type: 'chat'; name: string; prompt: PromptMessage[]; labels?: string[]; config?: Record; modelParams?: ModelParams; toolList?: ToolRef[]; mcpServers?: McpServerAttachmentInput[]; tags?: string[]; isActive?: boolean; commitMessage?: string; structuredOutput?: StructuredOutputSnapshot | null; } export type ExtendedCreatePromptBody = | ExtendedCreateTextPromptBody | ExtendedCreateChatPromptBodyMultimodal | ExtendedCreateChatPromptBodyWithPlaceholders | ExtendedCreateChatPromptBody; export type CreateChatPromptBodyWithPlaceholders = ExtendedCreateChatPromptBodyWithPlaceholders; // Generation body export interface ExtendedGenerationBody extends GenerationBody { [key: string]: any; } // Agent/config types export interface AISDKAgent { httpAgent?: any; httpsAgent?: any; } export interface AISDKPublicConfig { writeOnlyKey: string; [key: string]: any; } // Prompt client types export type PromptClient = TextPromptClient | ChatPromptClient; export type AISDKParent = any; export type PromptRecord = any; // Usage types export interface ExtendedUsage { input?: number | null; output?: number | null; total?: number | null; unit?: "CHARACTERS" | "TOKENS" | "MILLISECONDS" | "SECONDS" | "IMAGES" | "REQUESTS"; inputCost?: number | null; outputCost?: number | null; totalCost?: number | null; promptTokens?: number | null; completionTokens?: number | null; totalTokens?: number | null; serviceTier?: string; } export interface ExtendedUsageDetails { [key: string]: number; } // Tool ref export interface ToolRef { name: string; version?: number; label?: string; [key: string]: any; } // Evaluator enums/types export type EvaluatorType = "llm_custom" | "code"; export type EvaluatorScope = "GLOBAL" | "PROJECT"; export type CodeLanguage = "javascript" | "python"; export type ThresholdSelection = "greater_than" | "smaller_than"; export interface ScoreRangePrompt { [key: string]: any; } // Response types export interface CreateEvaluatorResponse extends EvaluatorResponseBase {} export interface UpdateEvaluatorResponse extends EvaluatorResponseBase {} // --------------------------------------------------------------------------- // Experiment Orchestrator (local experiment runs) // --------------------------------------------------------------------------- export type EvaluatorInput = EvaluatorFn | { name: string; version?: number }; export interface ExperimentCreateOptions { name: string; dataset: (() => DatasetItem[] | Promise) | { name: string; version?: number } | { file: string }; datasetId?: string; task: (input: unknown) => unknown | Promise; evaluators: EvaluatorInput[]; evaluatorListId?: string; metadata?: Record; } export interface DatasetItem { input: unknown; /** Expected output for this item. Passed to the evaluator as `expectedOutput`. */ expectedOutput?: unknown; [key: string]: unknown; } export type EvaluatorFn = (args: { input: unknown; output: unknown; expectedOutput?: unknown; /** The full dataset item (input, expectedOutput, and any extra columns/metadata). */ context?: Record; /** Injected run params (from AIEVALS_PARAMS_JSON), `{}` when unset. */ params?: Record; }) => EvaluatorResult | Promise; export interface EvaluatorResult { name: string; score: number; /** Optional human-readable explanation (e.g. from an LLM-judge); surfaced on `result.scores` and in the trace. */ reasoning?: string; } export interface ComparisonResult { status: 'ready' | 'no_baseline' | 'pending'; baseline?: { runId: string } | null; baselineMetrics?: unknown | null; evaluators: Array<{ name: string; dataType?: string; currentScore: number | null; baselineScore: number | null; delta: number | null; deltaDirection: string | null; threshold?: number | null; thresholdStatus?: string | null; }>; summary: { verdict: string; failureCount: number; improvedCount: number; regressedCount: number; }; } // Experiment config — Step 1 (local path) export interface LocalExperimentCreateOptions { name: string; dataset?: (() => DatasetItem[] | Promise) | { name: string; version?: number } | { file: string }; datasetId?: string; evaluators?: EvaluatorInput[]; evaluatorListId?: string; description?: string; metadata?: Record; } // Run execution — Step 2 (local path) export interface LocalExperimentRunOptions { task: (input: unknown) => unknown | Promise; maxConcurrency?: number; trialCount?: number; timeout?: number; hooks?: { onTestCase?: (item: DatasetItem, result: ItemResult) => void | Promise; onScore?: (item: DatasetItem, scores: EvaluatorResult[]) => void | Promise; onComplete?: (summary: { successCount: number; errorCount: number }) => void | Promise; }; } // Returned from Experiments.create() for local path — holds evaluators in memory export interface LocalExperimentHandle { id: string; experimentRunId: string; name: string; uiUrl: string; _dataset: (() => DatasetItem[] | Promise) | undefined; _datasetId: string | undefined; _evaluators: EvaluatorInput[]; _evaluatorListId: string | undefined; } export interface ExperimentRunResult { experimentRunId: string; experimentId: string; runName: string; uiUrl: string; results: ItemResult[]; comparison?: ComparisonResult; } export interface LLMMetrics { model?: string; promptTokens?: number; completionTokens?: number; totalTokens?: number; } export interface ItemResult { index: number; input: unknown; output: unknown; expectedOutput?: unknown; scores: EvaluatorResult[]; duration: number; error?: string; llmMetrics?: LLMMetrics; } export declare class ExperimentOrchestrator { constructor(baseUrl: string, ingestionUrl: string, publicKey: string, secretKey: string); /** Two-step Step 1 — define a local experiment (no execution). */ createExperiment(options: LocalExperimentCreateOptions): Promise; /** Two-step Step 2 — execute a defined local experiment with a task. */ executeRun(handle: LocalExperimentHandle, options: LocalExperimentRunOptions): Promise; } // ───────────────────────────────────────────────────────────────────────── // Error hierarchy — branch on `err instanceof TestOpsNotFoundError` etc. // without parsing error.message. All subclasses extend TestOpsApiError, // which exposes statusCode/statusText/response for structured handling. // ───────────────────────────────────────────────────────────────────────── export class TestOpsError extends Error { readonly code: string; readonly context?: Record; } export class TestOpsApiError extends TestOpsError { readonly statusCode: number; readonly statusText: string; readonly response?: any; readonly isClientError: boolean; readonly isServerError: boolean; } export class TestOpsAuthError extends TestOpsApiError {} export class TestOpsNotFoundError extends TestOpsApiError { readonly resourceType?: string; readonly resourceId?: string; } export class TestOpsRateLimitError extends TestOpsApiError { readonly retryAfter?: number; } export class TestOpsValidationError extends TestOpsError {} export class TestOpsConfigError extends TestOpsError {} export class TestOpsNetworkError extends TestOpsError {} export class TestOpsTimeoutError extends TestOpsError {} export class TestOpsExperimentError extends TestOpsError {}