import type { Sample, ExecutorFn } from '../types/index.js'; import type { ObservationInboxItem } from '../types/observability.js'; interface GenerateSamplesOptions { skillContent: string; count?: number; model: string; executorName: string; /** * 自然语言描述用户希望重点覆盖的场景。会作为额外约束追加到 prompt 末尾, * 优先于"自由发挥"的多样性。空串 / undefined 表示不施加额外约束。 */ focus?: string; /** 不生成 mocks/mocksStrict,eval 时真实执行所有工具调用。 */ noMock?: boolean; } /** * 拼出送给 LLM 的 user prompt。抽出来便于单测验证 focus 是否真的注入了。 * * count 语义: * - number: 强制生成 N 条 * - undefined: 让 LLM 按系统提示里"样本结构决策"的类型对应范围自行判断数量 */ export declare function buildSamplesPrompt({ skillContent, count, focus, noMock }: { skillContent: string; count?: number; focus?: string; noMock?: boolean; }): string; export declare function generateSamples({ skillContent, count, model, executorName, focus, noMock }: GenerateSamplesOptions): Promise<{ samples: Sample[]; costUSD: number; }>; type TraceSignalItem = Pick; export interface StratifiedTraceSignal extends TraceSignalItem { /** Share of total occurrences across all signals (0-1) — drives proportional * sample allocation in the prompt. */ weight: number; } /** * Rank trace signals by frequency and annotate each with its share of the total * occurrences. Lets `omk sample --from-traces` allocate samples *proportional to * how often a failure actually happened* instead of a flat "1-2 per signal" — a * failure seen 100× deserves more regression coverage than one seen twice. * * Deliberately does NOT re-merge signals here. The observation inbox is the only * source of these items and already aggregates `occurrences` by the FULL identity * — `skillName + cwd + sourceKind + signalType + signalSubtype + evidence` * (`inbox.ts` `keyFor`). Re-merging on a narrower key (e.g. type+subtype+evidence) * would fold *different skills / cwd* with the same failure shape into one entry, * mis-attributing the summed occurrences to the first skill and skewing the * regenerated distribution. So we trust the upstream dedup and only sort + weight. * * Does NOT fix the underlying selection bias (traces only capture *failures*), * which is why the `omk sample --from-traces` draft warning still stands — this * only makes the within-failure distribution representative of frequency. */ export declare function stratifyTraceSignals(items: TraceSignalItem[]): StratifiedTraceSignal[]; /** * Build the generation prompt for `omk sample --from-traces`. Renders each * observation-inbox signal (evidence + message window) into a section and asks * the generator to synthesize regression samples that reproduce the observed * production failures. The trace text feeds the *generator* only — never the * judge prompt — so judge-prompt isolation is unaffected. */ export declare function buildSamplesFromTracesPrompt(items: TraceSignalItem[], count?: number): string; export interface GenerateSamplesFromTracesOptions { items: TraceSignalItem[]; count?: number; model: string; executorName?: string; /** Injectable executor (tests). Defaults to createExecutor(executorName). */ executor?: ExecutorFn; } /** * Generate draft eval samples from production-trace observation signals. Mirrors * generateSamples' executor / retry / finalize path but feeds the trace prompt and * stamps `provenance: 'production-trace'`. Output is meant to land in a review draft, * not the live dataset (the CLI enforces that). */ export declare function generateSamplesFromTraces({ items, count, model, executorName, executor: injectedExecutor, }: GenerateSamplesFromTracesOptions): Promise<{ samples: Sample[]; costUSD: number; }>; export declare function sanitizeGeneratedSamples(samples: Sample[], opts?: { skillContent?: string; }): { stripped: string[]; }; export {};