/** * Map a completed production run into a `defineEval` case. * * This module has no database imports — callers load the run, events, spans, * and durable thread input, then persist the returned dataset. Hosted actions * must not write `*.eval.ts`; only the eval CLI `--write` path emits a fixture * file. */ import type { EvalDataset } from "../observability/types.js"; import type { Eval, EvalInput } from "./types.js"; export declare const PROMOTED_EVAL_PRIVACY_VERSION = 6; export declare const PROMOTED_EVAL_REVIEW_LIMITS: { readonly promptLength: 3000; readonly historyTurns: 16; readonly historyTextLength: 1000; readonly expectedTextLength: 500; }; /** Stable, non-reversible reference safe for persisted eval metadata and fixtures. */ export declare function promotedTraceReference(runId: string): string; /** Stable per-owner identity for one promoted run without embedding either identifier. */ export declare function promotedDatasetIdempotencyKey(runId: string, userId?: string | null): string; export declare function promotedDatasetDescription(runId: string): string; export type PromoteTraceError = "not_found" | "run_not_completed" | "no_user_prompt" | "no_signal" | "reviewed_prompt_required" | "reviewed_history_too_long" | "reviewed_text_too_long" | "unsafe_reviewed_text"; export interface PromoteTraceOptions { /** * Caller-reviewed text. Production trace text is never copied automatically. * These fields are still scrubbed and bounded before entering an eval. */ reviewedPrompt?: string; reviewedHistory?: Array<{ role: "user" | "assistant"; text: string; }>; /** Caller-reviewed expected substring; scrubbed and bounded before persistence. */ mustContain?: string; /** Optional generic label, privacy-screened before persistence. */ datasetName?: string; /** Owner of the dataset row. Scoped the same way trace reads filter user_id. */ userId?: string | null; } export interface PromoteTraceSpan { spanType: string; name: string; status: string; } export interface PromoteTraceRun { status: string; } export interface PromoteTraceEvent { seq: number; eventData: string; } export type PromotedEvalScorerSpec = { type: "usesTool"; toolName: string; } | { type: "contains"; needle: string; }; /** JSON-safe `defineEval` payload. Scorer functions cannot survive HTTP. */ export interface PromotedEvalSpec { name: string; input: EvalInput; threshold: number; /** Stable, non-reversible reference to the source production run. */ source: { kind: "trace"; runId: string; }; scorers: PromotedEvalScorerSpec[]; } export interface PromotedEval { eval: Eval; spec: PromotedEvalSpec; dataset: EvalDataset; sourceRunId: string; } export type PromoteTraceResult = { ok: true; value: PromotedEval; } | { ok: false; error: PromoteTraceError; }; export interface PromoteTraceInput { runId: string; run: PromoteTraceRun | null; events: readonly PromoteTraceEvent[]; spans?: readonly PromoteTraceSpan[]; options?: PromoteTraceOptions; /** * Durable chat thread repository (`chat_threads.thread_data`), as an object * or the raw JSON string. User prompts are persisted on the thread, not as * run events. */ threadInput?: unknown; } interface ConversationTurn { role: "user" | "assistant"; text: string; } /** * Rebuild user/assistant turns the same way `buildConversationTranscript` * walks `eventData`, but concatenate consecutive assistant text events so * history is one turn per reply rather than one turn per delta. */ export declare function conversationTurnsFromEvents(events: readonly PromoteTraceEvent[]): ConversationTurn[]; export declare function serializePromotedEval(value: PromotedEval): PromotedEvalSpec; /** * Rebuild the JSON eval spec from a dataset row so a repeat promotion can * return the stored case without replaying the run. */ export declare function promotedEvalSpecFromDataset(dataset: EvalDataset, runId: string): PromotedEvalSpec | null; /** Rebuild a response-safe dataset from a current-version promoted row. */ export declare function sanitizedPromotedDatasetFromDataset(dataset: EvalDataset, runId: string): EvalDataset | null; export declare function generateEvalModuleSource(spec: PromotedEvalSpec): string; /** * Turn a completed production run into a `defineEval` case plus an in-memory * `EvalDataset`. Callers persist the dataset; this function never writes SQL * or files. */ export declare function promoteTraceToEval(input: PromoteTraceInput): PromoteTraceResult; export {};