/**
 * LLM grader types — shared interfaces for all LLM-based graders.
 */
import { z } from "zod";
import type { ZodSchema as CopilotZodSchema } from "@github/copilot-sdk";
import type { AggregationStrategy } from "./aggregation.js";
import type { JudgeProviderSpec } from "../../provider/copilot-provider.js";
/** Scoring scale for LLM judges. */
export type ScoringScale = "binary" | "scale_1_5" | "scale_1_10";
/**
 * Reasoning effort level for judge models that support it. Mirrors the Copilot
 * SDK's `ReasoningEffort`. When unset, the judge runs at the model's own
 * default effort — which is neither controlled nor recorded — so callers that
 * care about reproducibility should set it explicitly.
 */
export type ReasoningEffort = "low" | "medium" | "high" | "xhigh";
/** Allowed reasoning-effort values, for validation and error messages. */
export declare const REASONING_EFFORTS: readonly ReasoningEffort[];
/** Type guard for {@link ReasoningEffort}. */
export declare function isValidReasoningEffort(v: unknown): v is ReasoningEffort;
/** Evidence sections that a prompt or panel judge can receive. */
export type LlmGradingEvidence = "trajectory" | "diff" | "golden_patch" | "repo";
/** Evidence values supported by prompt and panel graders. */
export declare const LLM_GRADING_EVIDENCE: readonly ["trajectory", "diff", "golden_patch", "repo"];
/** Configuration for the `prompt` grader in eval.yaml. */
export interface PromptGraderConfig {
    /** Extra instructions for the judge. Additive — it does not replace the
     *  scored criteria, which come from the stimulus rubric. */
    prompt?: string;
    /** Model to use for judging. Overrides eval-level judge_model. */
    model?: string;
    /**
     * Reasoning effort for the judge model. A per-grader override takes
     * precedence over the eval-level `judge_reasoning_effort` default.
     */
    reasoning_effort?: ReasoningEffort;
    /**
     * BYOK provider for the judge session. Injected from the eval-level
     * `defaults.judge_provider`; not typically authored per-grader.
     */
    provider?: JudgeProviderSpec;
    /** Scoring scale. Default: "scale_1_5". */
    scoring?: ScoringScale;
    /** Pass threshold on the normalized [0,1] scale. Default: 0.5 (i.e. 3/5). */
    threshold?: number;
    /** Evidence sections supplied to the judge. Defaults to trajectory-only evidence. */
    evidence?: LlmGradingEvidence[];
    /** How selected evidence is delivered to the judge. Default: "inline". */
    output_delivery?: "inline" | "workspace";
}
/** A single rubric criterion score from the LLM judge. */
export interface RubricScore {
    criterion: string;
    score: number;
    reasoning: string;
}
/** The structured JSON response expected from the LLM judge. */
export interface JudgeResponse {
    rubric_scores: RubricScore[];
    overall_score: number;
    overall_reasoning: string;
}
export interface LlmTokenUsage {
    inputTokens: number;
    outputTokens: number;
    model: string;
}
/**
 * Options for {@link LlmClient.judge}.
 *
 * The model is required to invoke `tool` exactly once with arguments matching
 * `tool.parameters`. The handler-driven path returns those parsed args; if
 * the model fails to call the tool (or calls it with invalid arguments), an
 * in-session reminder loop nudges it before giving up.
 */
/**
 * A schema that satisfies both the Copilot SDK (for tool definition) and
 * our own validation needs (safeParse in the handler).
 */
export interface JudgeToolSchema<T> extends CopilotZodSchema<T> {
    safeParse(value: unknown): {
        success: true;
        data: T;
    } | {
        success: false;
        error: {
            message: string;
        };
    };
}
export interface LlmJudgeOptions<TArgs> {
    model: string;
    /**
     * Reasoning effort for models that support it. When unset, the model runs at
     * its own default effort. Forwarded to the SDK session.
     */
    reasoningEffort?: ReasoningEffort;
    /**
     * Bring-your-own-key (BYOK) provider (from `defaults.judge_provider`). When
     * set, the judge session is pointed at this custom endpoint instead of the
     * default GitHub/Copilot auth chain. The client resolves its `*Env` credential
     * references and redacts them from any surfaced error.
     */
    provider?: JudgeProviderSpec;
    systemMessage: string;
    userMessage: string;
    /** The tool the model MUST invoke exactly once. */
    tool: {
        name: string;
        description: string;
        /** Zod schema describing the tool's parameters. */
        parameters: JudgeToolSchema<TArgs>;
    };
    /** Default 120_000 (2 minutes), passed to each sendAndWait. */
    timeoutMs?: number;
    /**
     * Max number of in-session reminder turns sent after a missed/invalid tool
     * call. Default: 2.
     */
    maxReminders?: number;
    /** Disposable, read-only grading-data workspace for large evidence. */
    workspace?: LlmJudgeWorkspace;
}
export interface LlmJudgeWorkspace {
    /** Session working directory. Contains only Vally-owned grading data. */
    workingDirectory: string;
    /** Canonical paths of the selected evidence files the judge may read. */
    evidenceFiles: readonly string[];
    /** Report non-fatal SDK-session cleanup failures to the owning grader. */
    onCleanupError?: (operation: string, error: unknown) => void;
}
export interface LlmJudgeResponse<TArgs> {
    /** Parsed + Zod-validated tool arguments. */
    args: TArgs;
    /** Cumulative token usage across the initial turn + any reminder turns. */
    tokenUsage?: LlmTokenUsage;
    /** Cumulative wall-clock time for all turns. */
    latencyMs: number;
    /**
     * How many reminder turns were needed (0 if the model called the tool on
     * the first turn).
     */
    remindersUsed: number;
    /**
     * Evidence retrieval totals for workspace delivery, absent for inline.
     * `budgetExhausted` means a read was truncated to nothing because the
     * character budget ran out, so the judge may not have seen everything it
     * asked for.
     */
    evidenceUsage?: {
        toolCalls: number;
        chars: number;
        budgetExhausted: boolean;
    };
}
export interface LlmClient {
    /** Whether this client enforces the workspace-delivery security contract. */
    supportsWorkspaceDelivery?: boolean;
    /**
     * Run a single judge call where the model MUST invoke `options.tool` to
     * deliver its verdict.
     */
    judge<TArgs>(options: LlmJudgeOptions<TArgs>): Promise<LlmJudgeResponse<TArgs>>;
    shutdown(): Promise<void>;
}
export interface RetryOptions {
    /** Maximum number of retries (not counting the initial attempt). Default: 2. */
    maxRetries?: number;
    /** Base delay in ms before first retry. Default: 5000. */
    baseDelayMs?: number;
    /** Maximum delay for a single retry. Default: 60_000. */
    maxDelayMs?: number;
    /** Total budget for all attempts in ms. Default: 600_000 (10 minutes). */
    budgetMs?: number;
}
export interface TrajectoryFormatOptions {
    /** Maximum number of events to include. Default: 100. */
    maxEvents?: number;
    /** Maximum characters per individual event entry. Default: 500. */
    maxCharsPerEvent?: number;
}
export declare const SCALE_RANGES: Record<ScoringScale, {
    min: number;
    max: number;
}>;
export declare const DEFAULT_THRESHOLDS: Record<ScoringScale, number>;
/** Normalize a raw score to [0, 1] given the scoring scale. */
export declare function normalizeScore(raw: number, scale: ScoringScale): number;
/** Default rubric used when none is specified in eval.yaml. */
export declare const DEFAULT_RUBRIC: string[];
/**
 * Provenance of the criteria a judge scored against. `"default"` means
 * {@link DEFAULT_RUBRIC} was substituted because no rubric was defined.
 */
export type RubricSource = "stimulus" | "criteria" | "default";
/** A rubric plus where it came from. */
export interface ResolvedRubric {
    rubric: string[];
    source: RubricSource;
}
/**
 * Magnitude buckets the comparison judge emits, from the *winner's*
 * perspective. The judge sees two responses in a neutral A/B framing and
 * never knows which is the baseline; we map (winner, magnitude) onto the
 * signed treatment-relative {@link ComparisonMagnitude} scale ourselves.
 */
export type JudgeMagnitude = "much-better" | "slightly-better" | "equal";
/** Per-criterion result from the comparison judge (neutral A/B framing). */
export interface ComparisonRubricResult {
    criterion: string;
    winner: "A" | "B" | "tie";
    magnitude: JudgeMagnitude;
    reasoning: string;
}
/** The structured response expected from the comparison judge. */
export interface ComparisonJudgeResponse {
    rubric_results: ComparisonRubricResult[];
    overall_winner: "A" | "B" | "tie";
    overall_magnitude: JudgeMagnitude;
    overall_reasoning: string;
}
/**
 * Zod schema for the `submit_comparison_grade` tool's parameters.
 *
 * The tool parameter is a single flat object (not a union). Tool-calling models
 * reliably emit a valid call only when the top-level parameter schema is one
 * `type: object`; a top-level `anyOf`/union frequently causes the model to fail
 * to produce a tool call at all. The tie⟺equal consistency rule is enforced via
 * `.refine()` — which validation honors (so an inconsistent call is rejected and
 * the judge is reminded with a clear message) but which drops out of the
 * generated JSON Schema, leaving the model a clean flat-object contract.
 */
export declare const ComparisonJudgeResponseSchema: z.ZodType<ComparisonJudgeResponse>;
/**
 * A structured rubric criterion for the panel grader.
 *
 * When a panel config lists `criteria`, the grader scores each criterion
 * across every judge, aggregates per criterion, then rolls the criteria up
 * into a weighted overall score (see {@link PanelGraderConfig.criteria}).
 */
export interface PanelCriterionConfig {
    /** Identifier echoed by judges and shown in per-criterion results. */
    name: string;
    /** Human-readable description shown to judges in the rubric. */
    description?: string;
    /** Relative weight in the overall score; weights are normalized to sum to 1. Default: `1`. */
    weight?: number;
    /** Per-criterion pass threshold on the normalized [0,1] scale. Default: the panel `threshold`. */
    pass_threshold?: number;
    /**
     * Scoring scale the judges use for this criterion. Overrides the panel-level
     * `scoring` for this criterion only, letting a single panel mix scales.
     * Default: the panel `scoring`.
     */
    scoring?: ScoringScale;
    /**
     * Whether this criterion gates the panel verdict. When `true` (default), the
     * criterion must pass for the panel to pass. When `false`, the criterion is
     * advisory: its score still contributes to the weighted overall and is
     * reported, but it never blocks the all-pass gate.
     */
    required?: boolean;
}
/**
 * A single judge in a panel: a model plus its own optional reasoning effort.
 * Panels deliberately mix models, and reasoning-effort support is per-model,
 * so effort is set per judge rather than uniformly across the panel.
 */
export interface PanelJudgeConfig {
    /** Judge model identifier. */
    model: string;
    /** Reasoning effort for this judge's model. Only valid for models that support it. */
    reasoning_effort?: ReasoningEffort;
}
/** Configuration for the `panel` grader in eval.yaml. */
export interface PanelGraderConfig {
    /** Extra instructions for the judges. Additive — it does not replace the
     *  scored criteria, which come from `criteria` or the stimulus rubric. */
    prompt?: string;
    /**
     * Judges to fan out across (normalized). Required (>= 1 entry). In eval.yaml
     * each entry may be a bare model string or a `{ model, reasoning_effort }`
     * object; both normalize to {@link PanelJudgeConfig} here.
     */
    models: PanelJudgeConfig[];
    /** Aggregation strategy across judges. Default: "mean". */
    aggregation?: AggregationStrategy;
    /** Scoring scale used by every judge. Default: "scale_1_5". */
    scoring?: ScoringScale;
    /** Pass threshold on the normalized [0,1] scale. Default: scoring-dependent (`DEFAULT_THRESHOLDS[scoring]`). */
    threshold?: number;
    /** Evidence sections supplied to each judge. Defaults to trajectory-only evidence. */
    evidence?: LlmGradingEvidence[];
    /**
     * Structured rubric criteria. When provided, the grader aggregates each
     * criterion's scores across judges and derives the overall score from the
     * weighted sum of aggregated criterion scores — the panel passes only when
     * every required criterion passes AND the weighted overall clears
     * `overall_threshold`. Criteria marked `required: false` are advisory: scored
     * and weighted into the overall, but excluded from the all-pass gate.
     * When omitted, the panel aggregates each judge's holistic `overall_score`
     * and uses the stimulus rubric as-is.
     */
    criteria?: PanelCriterionConfig[];
    /**
     * Overall weighted-score pass threshold, used only when `criteria` is set.
     * Default: the panel `threshold`.
     */
    overall_threshold?: number;
    /**
     * BYOK provider for the judge sessions (applies to every judge in the panel).
     * Injected from the eval-level `defaults.judge_provider`; not typically
     * authored per-grader.
     */
    provider?: JudgeProviderSpec;
}
//# sourceMappingURL=types.d.ts.map