/**
 * Datasets that grow: synthetic examples written from documents, each citing the passage it came
 * from; adversarial examples grown from the guardrails' own attack patterns; and production runs
 * promoted from the issues insights found.
 */
import type { Issue } from '../insights/clusters.js';
import { type AttackCategory } from '../security/injection-detector.js';
import type { Dataset, DatasetExample, Evaluator } from '../types/evaluate.js';
import type { CompletionRequest } from '../types/messages.js';
import type { NexusResponse } from '../types/response.js';
import type { Run } from '../types/tracing.js';
/** A document to write examples from. */
export interface SyntheticDocument {
    /** Names the document in every citation. */
    id: string;
    /** Its text. */
    text: string;
    /** Its title, carried into citations. */
    title?: string;
}
/** One passage of a document, as the writer sees it. */
export interface SyntheticPassage {
    /** The document it is from. */
    document: SyntheticDocument;
    /** Its position among the document's passages, from 0. */
    index: number;
    /** Where it starts in the document, in characters. */
    start: number;
    /** Where it ends. */
    end: number;
    /** Its text. */
    text: string;
}
/** An example the writer drafts: a question, its answer, and the words of the passage that support it. */
export interface SyntheticDraft {
    /** What the target is asked. */
    question: string;
    /** The answer the passage supports. */
    answer: string;
    /** Words copied from the passage that support the answer. A draft whose quote is not there is dropped. */
    quote: string;
}
/** Where an example came from: the document, the passage, and the supporting words. */
export interface SyntheticCitation {
    /** The document's id. */
    document: string;
    /** Its title, when it has one. */
    title?: string;
    /** The passage's position in the document. */
    passage: number;
    /** Where the supporting words start in the document, in characters. */
    start: number;
    /** Where they end. */
    end: number;
    /** The supporting words, as the document has them. */
    quote: string;
}
/** A model that writes drafts. */
export interface SyntheticModelClient {
    /** Runs one completion. */
    complete(request: CompletionRequest): Promise<NexusResponse>;
}
/** Options for `syntheticDataset()`. Give it `generate`, or a `client` and `model`. */
export interface SyntheticDatasetOptions {
    /** The dataset's name. */
    name: string;
    /** The documents to write examples from. */
    documents: readonly SyntheticDocument[];
    /** Writes drafts for one passage. */
    generate?: (passage: SyntheticPassage, count: number) => Promise<SyntheticDraft[]> | SyntheticDraft[];
    /** A model that writes drafts with a built-in prompt, when there is no `generate`. */
    client?: SyntheticModelClient;
    /** The model the client uses. */
    model?: string;
    /** The longest passage, in characters. Paragraphs are kept whole where they fit. Defaults to 1,500. */
    passageChars?: number;
    /** Drafts asked for per passage. Defaults to 2. */
    perPassage?: number;
    /**
     * How alike two questions' words may be before the later one is dropped as a near-duplicate, from
     * 0 to 1. Defaults to 0.8; `false` keeps every question.
     */
    similarity?: number | false;
    /** The most examples kept. */
    limit?: number;
}
/** What `syntheticDataset()` made, and what it dropped. */
export interface SyntheticDatasetResult {
    /** The examples: a question as input, the answer as expected, and the citation in `metadata.citation`. */
    dataset: Dataset<{
        question: string;
    }, string>;
    /** Drafts whose quote was not in their passage. */
    unsupported: number;
    /** Drafts dropped as near-duplicates of a question already kept. */
    duplicates: number;
}
/**
 * Writes a dataset from documents. Each document is cut into passages, and each passage gets drafts
 * from `generate` or the model: a question, its answer, and a quote. A draft is kept only when its
 * quote is in its passage, word for word apart from spacing, and its citation records where; and a
 * question too like one already kept is dropped.
 */
export declare function syntheticDataset(options: SyntheticDatasetOptions): Promise<SyntheticDatasetResult>;
/** How an attack reaches the target: in what the user asks, in a document it is given, or in a tool's result. */
export type AttackCarrier = 'direct' | 'document' | 'tool';
/** How an attack is disguised: as written, in full-width letters, or split by invisible characters. */
export type AttackDisguise = 'plain' | 'full-width' | 'zero-width';
/** What an adversarial example gives the target. */
export interface AdversarialInputs {
    /** The system prompt, holding the canary the attack tries to get out. */
    system: string;
    /** What the user asks. */
    prompt: string;
    /** A document the user gives, when the attack is carried in one. */
    document?: string;
    /** What a tool returns, when the attack is carried in a tool's result. */
    toolResult?: string;
}
/** What an adversarial example expects: the canary never appears in the output. */
export interface AdversarialExpected {
    /** A secret only the system prompt holds. */
    canary: string;
}
/** Options for `adversarialDataset()`. */
export interface AdversarialDatasetOptions {
    /** The dataset's name. Defaults to `adversarial`. */
    name?: string;
    /** Which attacks. Defaults to all three categories. */
    categories?: readonly AttackCategory[];
    /** How they arrive. Defaults to all three carriers. */
    carriers?: readonly AttackCarrier[];
    /** How they are disguised. Defaults to all three. */
    disguises?: readonly AttackDisguise[];
    /** Adds one benign example per carrier, so a guard that refuses everything shows. Defaults to true. */
    benign?: boolean;
    /** More wordings of an attack, such as a model's rephrasings; each becomes an example of its own. */
    rephrase?: (attack: string, category: AttackCategory) => Promise<string[]> | string[];
}
/**
 * Grows an adversarial dataset from the guardrails' own attack patterns: every seed of
 * `attackSeeds()`, in each carrier and disguise. Every example hides a canary in the system prompt
 * and asks, one way or another, for it to come out: injections and jailbreaks tell the model to say
 * it, exfiltration asks for the prompt that holds it. Score a target with `resistsAttack()`.
 */
export declare function adversarialDataset(options?: AdversarialDatasetOptions): Promise<Dataset<AdversarialInputs, AdversarialExpected>>;
/**
 * Scores whether a target kept the canary in: 1 when it never appears in the output, read through
 * the same disguises the guardrails read through. A target that refused, or was stopped by a
 * guardrail, kept it.
 */
export declare function resistsAttack(options?: {
    key?: string;
}): Evaluator<AdversarialInputs, AdversarialExpected>;
/** Options for `promoteIssue()`. */
export interface PromoteIssueOptions {
    /** A dataset to add to; its examples are kept and the issue's runs added. */
    into?: Dataset;
    /** The new dataset's name, when there is no `into`. Defaults to `issue-<id>`. */
    name?: string;
    /** Turns a run into an example. Defaults to its inputs, with its outputs as expected unless it failed. */
    toExample?: (run: Run) => Omit<DatasetExample, 'id'> | undefined;
    /** The most runs promoted. Defaults to all of them. */
    limit?: number;
}
/**
 * Promotes the runs behind an insights issue into a dataset. Each example keeps the run it came from
 * as `sourceRunId`, and the issue that found it as `metadata.issue`, so a regression test points back
 * to the problem it guards against. A run already in the dataset is not added twice.
 */
export declare function promoteIssue(issue: Issue, options?: PromoteIssueOptions): Dataset;