import type { Node, Edge } from '@xyflow/react';
export type Difficulty = 'Easy' | 'Medium' | 'Hard';
export type DateFormatVariant = 'date' | 'datetime' | 'detailed';
export type JudgeProvider = 'demo' | 'bedrock' | 'openai-compatible' | 'litellm' | 'claude-code' | 'agentic' | 'pi' | 'agent';
export interface AssistantMessage {
    role: 'user' | 'assistant';
    content: string;
    timestamp: string;
}
export interface AssistantContext {
    currentUrl?: string;
    benchmarkId?: string;
    runId?: string;
    traceId?: string;
    testCaseId?: string;
    /**
     * On comparison pages (`/compare/:benchmarkId?runs=a,b,…`), the list of run
     * IDs the user is currently comparing. The assistant pre-loads these into
     * the grounded snapshot so it can answer cross-run questions even before
     * reaching for tools.
     */
    comparisonRunIds?: string[];
}
export type ConnectorProtocol = 'agui-streaming' | 'rest' | 'openai-compatible' | 'subprocess' | 'claude-code' | 'pi' | 'strands' | 'langgraph' | 'mock';
export interface ModelConfig {
    model_id: string;
    display_name: string;
    provider: JudgeProvider;
    context_window: number;
    max_output_tokens: number;
}
export interface BeforeRequestContext {
    endpoint: string;
    payload: any;
    headers: Record<string, string>;
}
export interface AfterResponseContext {
    response: any;
    trajectory: TrajectoryStep[];
    runId?: string;
    /** Full array of raw events from the connector (protocol-specific) */
    rawEvents?: any[];
    /** Connector metadata (e.g., threadId, sessionId, exitCode) */
    metadata?: Record<string, any>;
}
export interface BuildTrajectoryContext {
    spans: Span[];
    runId: string;
}
/**
 * Context passed to a custom judge hook.
 * Contains all data needed for evaluation: trajectory, traces, and expected outcomes.
 * Also provides fetchTraces as an SDK utility for additional trace fetching.
 */
export interface JudgeContext {
    trajectory: TrajectoryStep[];
    traces: Span[];
    expectedOutcomes: string[];
    expectedTrajectory?: string[];
    runId: string;
    /** SDK utility: fetch traces by run IDs from OpenSearch */
    fetchTraces: (runIds: string[]) => Promise<{
        spans: Span[];
    }>;
}
/**
 * Result returned by a custom judge hook.
 */
export interface JudgeResult {
    passFailStatus: 'passed' | 'failed';
    metrics: {
        accuracy: number;
        faithfulness?: number;
        latency_score?: number;
        trajectory_alignment_score?: number;
        [key: string]: number | undefined;
    };
    llmJudgeReasoning: string;
    improvementStrategies?: string[];
}
export interface AgentHooks {
    /**
     * Called before sending request to agent.
     * Use to modify endpoint, payload, or headers.
     */
    beforeRequest?: (context: BeforeRequestContext) => Promise<BeforeRequestContext>;
    /**
     * Called after receiving response from agent.
     * Use to extract runId from custom response formats (e.g., PER memory_id).
     */
    afterResponse?: (context: AfterResponseContext) => Promise<AfterResponseContext>;
    /**
     * Called when building trajectory from OTEL traces.
     * Use to customize trajectory extraction for agents with custom span formats.
     */
    buildTrajectory?: (context: BuildTrajectoryContext) => Promise<TrajectoryStep[]>;
    /**
     * Custom judge hook. When defined, replaces the built-in Bedrock judge.
     * Receives trajectory + traces + expected outcomes, returns pass/fail evaluation.
     *
     * @example
     * ```typescript
     * hooks: {
     *   judge: async ({ trajectory, traces, expectedOutcomes, fetchTraces }) => {
     *     // Custom evaluation logic using traces
     *     const relevantSpans = traces.filter(s => s.attributes?.['gen_ai.system']);
     *     return {
     *       passFailStatus: relevantSpans.length > 0 ? 'passed' : 'failed',
     *       metrics: { accuracy: 85 },
     *       llmJudgeReasoning: 'Custom evaluation based on trace analysis',
     *     };
     *   }
     * }
     * ```
     */
    judge?: (context: JudgeContext) => Promise<JudgeResult>;
}
export interface AgentConfig {
    key: string;
    name: string;
    endpoint: string;
    description?: string;
    enabled?: boolean;
    headers?: Record<string, string>;
    auth?: ConnectorAuthConfig;
    useTraces?: boolean;
    /**
     * Configurable trace polling settings (used when `useTraces: true`).
     *
     * Two distinct polling paths honour these values, with different defaults
     * because they have different ergonomic constraints:
     *
     *   - **Judge poller** (`services/traces/tracePoller.ts`, runs in the
     *     background after the agent finishes, before the LLM judge fires)
     *     defaults to `intervalMs: 10000` and `maxAttempts: 60` — a 10-minute
     *     total budget that's fine because the user already sees a "pending"
     *     badge while it polls.
     *   - **SDK pre-load** (`services/traces/fetchSpansForRun.ts`, runs
     *     synchronously inside a deterministic test body before the body's
     *     first assertion) defaults to `intervalMs: 1000` and `maxAttempts:
     *     10` — a ~10-second total budget so the test isn't blocked.
     *
     * Both paths additionally honour `TRACE_POLL_INTERVAL_MS` and
     * `TRACE_POLL_MAX_ATTEMPTS` env vars (the env vars override the
     * code defaults), and both enforce a hard ceiling of 60 attempts so
     * a misconfigured agent can't lock a test for an unbounded time.
     *
     * Setting either field on this object overrides the path's own default
     * for that specific agent on both paths.
     */
    tracePolling?: {
        intervalMs?: number;
        maxAttempts?: number;
    };
    /**
     * OTel `service.name` resource attribute that this agent reports under.
     * Defaults to {@link AgentConfig.key} when not set, which is correct for
     * agents whose OTel SDK uses the same identifier as the config key (e.g.
     * `claude-code`). Override only when the agent's OTel service name differs
     * from its config key, e.g. `observio` -> `observio-sample-agent`.
     *
     * Used by the Agent Traces page to translate the user's cross-page agent
     * filter (`agent-health:prefs:agentFilter`, which stores agent keys) into
     * the actual `service.name` to filter by in OpenSearch queries.
     */
    traceServiceName?: string;
    connectorType?: ConnectorProtocol;
    connectorConfig?: Record<string, any>;
    hooks?: AgentHooks;
    isCustom?: boolean;
    builtIn?: boolean;
}
/**
 * Authentication config for agents (serializable subset of ConnectorAuth).
 * Used in AgentConfig for config files — avoids importing connector types.
 */
export interface ConnectorAuthConfig {
    type: 'none' | 'basic' | 'bearer' | 'api-key' | 'aws-sigv4';
    username?: string;
    password?: string;
    token?: string;
    awsRegion?: string;
    awsService?: string;
    headers?: Record<string, string>;
}
export interface AppConfig {
    agents: AgentConfig[];
    models: Record<string, ModelConfig>;
    defaults: {
        retry_attempts: number;
        retry_delay_ms: number;
    };
}
export declare enum ToolCallStatus {
    SUCCESS = "SUCCESS",
    FAILURE = "FAILURE"
}
export interface TrajectoryStep {
    id: string;
    timestamp: number;
    type: 'tool_result' | 'assistant' | 'action' | 'response' | 'thinking';
    content: string;
    toolName?: string;
    toolArgs?: Record<string, any>;
    toolOutput?: any;
    status?: ToolCallStatus;
    latencyMs?: number;
}
export interface EvaluationMetrics {
    accuracy?: number;
    faithfulness?: number;
    latency_score?: number;
    trajectory_alignment_score?: number;
    [key: string]: number | undefined;
}
export interface ImprovementStrategy {
    category: string;
    issue: string;
    recommendation: string;
    priority: 'high' | 'medium' | 'low';
}
/**
 * Scoring metric definition for evaluators
 */
export interface ScoringMetric {
    name: string;
    description?: string;
    weight: number;
    scale: number;
}
/**
 * Scoring configuration for an evaluator
 */
export interface ScoringConfig {
    metrics: ScoringMetric[];
    passThreshold: number;
    scale: number;
}
/**
 * Inference configuration for an evaluator
 */
export interface InferenceConfig {
    provider?: JudgeProvider;
    modelId?: string;
    temperature?: number;
    maxTokens?: number;
}
/**
 * Evaluator version - immutable snapshot of evaluator configuration
 */
export interface EvaluatorVersion {
    version: number;
    createdAt: string;
    systemPrompt: string;
    scoringConfig: ScoringConfig;
    inferenceConfig: InferenceConfig;
}
/**
 * Evaluator - pluggable judge configuration
 * Defines how agent performance is evaluated
 */
export interface Evaluator {
    id: string;
    name: string;
    description: string;
    isSystem: boolean;
    tags?: string[];
    currentVersion: number;
    versions: EvaluatorVersion[];
    createdAt: string;
    updatedAt: string;
    author?: string;
    systemPrompt: string;
    scoringConfig: ScoringConfig;
    inferenceConfig: InferenceConfig;
}
export type PassFailStatus = 'passed' | 'failed';
export interface LLMJudgeResponse {
    modelId: string;
    timestamp: string;
    promptTokens: number;
    completionTokens: number;
    latencyMs: number;
    /**
     * Raw judge text exactly as the model returned it (pre-JSON-parse). Set
     * by the routing layer from `JudgeResponse.rawResponse`. Older callers
     * stuffed the parsed `llmJudgeReasoning` into this field as a fallback;
     * post evaluator-prompt-plumbing the field carries the actual unparsed
     * model output for debugging "prompt edited but output didn't change"
     * scenarios.
     */
    rawResponse: string;
    /**
     * Parsed numeric metrics. Open-ended `[key: string]: number` so a saved
     * evaluator can declare arbitrary metric names in its `scoringConfig.metrics`
     * and they flow through here unchanged. Legacy keys (`accuracy`,
     * `faithfulness`, `latency_score`, `trajectory_alignment_score`) remain
     * conventional but are no longer required — evaluators are pluggable.
     */
    parsedMetrics?: {
        [key: string]: number | undefined;
    };
    improvementStrategies?: ImprovementStrategy[];
    error?: string;
    /**
     * Any JSON keys the judge emitted that did NOT map onto a typed wire
     * field or a declared metric. Captured by
     * {@link parseJudgeResponse} (server/services/judgeResponseParser) so the
     * run-detail "Judge debug" surface can show prompt-iteration output (e.g.
     * `improvement_candidates`, `failure_tags`, `confidence`) without a code
     * change. Empty/undefined when the model emitted only typed fields.
     */
    extraFields?: Record<string, unknown>;
    /**
     * Optional debug breadcrumbs persisted when `AH_JUDGE_DEBUG=1` (or in dev
     * mode). Captures exactly what the run-detail UI needs to confirm "the
     * prompt I saved is the prompt that ran" — the system prompt the model
     * received, the user prompt, and which provider executed the call. The
     * raw response itself is on the parent {@link rawResponse}.
     *
     * Disabled by default to keep persisted run docs lean (system prompts
     * can be 10–20 KB).
     */
    judgeDebug?: {
        /** Provider that executed the call: 'bedrock' | 'claude-code' | 'pi' | 'agent' | 'agentic' | 'openai-compatible' | 'litellm'. */
        provider?: string;
        /** Effective model id passed to the provider (post-resolution). */
        modelId?: string;
        /** Evaluator id used (system or user). */
        evaluatorId?: string;
        /** The full system prompt the model received. */
        systemPrompt?: string;
        /** The user-message prompt the model received. */
        userPrompt?: string;
    };
}
export interface RunAnnotation {
    id: string;
    reportId: string;
    text: string;
    timestamp: string;
    tags?: string[];
    author?: string;
}
/**
 * @experimental Generic sidecar metadata for coding agent sessions.
 * One document per session — stores annotations, status, tags, or any
 * user-defined fields. The shape is intentionally open so callers can
 * store whatever debug/analysis data they need.
 */
export interface SessionMetadata {
    agentKind: string;
    sessionId: string;
    /** Open-ended — callers define the schema. */
    [key: string]: unknown;
}
export type MetricsStatus = 'pending' | 'calculating' | 'ready' | 'error';
export interface TestCaseRun {
    id: string;
    timestamp: string;
    /**
     * Human-readable name for this run (e.g. "Baseline", "Claude_02").
     * Set from the user-supplied value in the run config dialog, or auto-generated
     * server-side as `Run <short-id>` if not provided. Optional for backwards
     * compatibility with runs created before the field existed — UI consumers
     * should fall back to a generated label (see `getRunDisplayName`).
     */
    name?: string;
    /** Optional human-readable description of what this run was testing. */
    description?: string;
    testCaseId: string;
    testCaseVersion?: number;
    experimentId?: string;
    experimentRunId?: string;
    agentName: string;
    agentKey?: string;
    modelName: string;
    modelId?: string;
    /**
     * Optional judge model id, separate from {@link modelId} (which is the
     * agent's LLM). Set explicitly via the run config (UI dropdown / CLI
     * `--judge-model` / API `judgeModelId` field) or left unset to fall back
     * to the evaluator's `inferenceConfig.modelId`, then the server-default
     * Bedrock judge model. For agentic providers (`pi`, `agent`, `agentic`,
     * `claude-code`) the value is informational — the provider picks its own
     * model from its credentialed registry. Stored on the run document so the
     * "Judge debug" surface and audit trail show which judge model was used.
     */
    judgeModelId?: string;
    agentEndpoint?: string;
    evaluatorId?: string;
    status: 'running' | 'completed' | 'failed';
    passFailStatus?: PassFailStatus;
    trajectory: TrajectoryStep[];
    metrics: EvaluationMetrics;
    /**
     * @deprecated Use `getJudgeReasoningText(report)` /
     * `getJudgeMatcherResults(report)` from `lib/matchers/judgeAccessor`.
     * The canonical judge surface is now `matcherResults[]` with
     * `method: 'llm-judge'`. This flat-string field is kept as an
     * Option-B backward-compat shim — it carries the most recent judge
     * reasoning so old direct readers keep working, but new code MUST
     * use the accessor.
     */
    llmJudgeReasoning: string;
    improvementStrategies?: ImprovementStrategy[];
    llmJudgeResponse?: LLMJudgeResponse;
    /**
     * W3C OTel trace id (32 hex). Stamped onto the run document at save time
     * when polled spans expose one, used as the strongest correlation key for
     * the run-detail Traces tab and the agent (trace) judge's `query_spans`
     * tool. See #190 (this field as a top-level shortcut over re-extracting
     * from `spans[0]`) and #264 (unified trace correlation strategies).
     *
     * Distinct from {@link runId} (the connector's run id, e.g.
     * `subprocess-<timestamp>`); pre-fix the runner mis-stamped runId here
     * which broke `traceId`-based queries.
     */
    traceId?: string;
    openSearchLogs?: OpenSearchLog[];
    annotations?: RunAnnotation[];
    runId?: string;
    /**
     * Agent-emitted session id (e.g. Claude Code stamps `session.id` on every
     * span of a run). Captured from the connector result and used as a precise
     * per-run trace correlator (Strategy D) for agents that emit it but don't
     * propagate W3C context or tag our `agent_health.run.id`.
     */
    sessionId?: string;
    logs?: OpenSearchLog[];
    rawEvents?: any[];
    connectorProtocol?: ConnectorProtocol;
    matcherResults?: import('../lib/matchers/types.js').MatcherResult[];
    performanceMetrics?: TestCasePerformanceMetrics;
    metricsStatus?: MetricsStatus;
    traceFetchAttempts?: number;
    lastTraceFetchAt?: string;
    traceError?: string;
    spans?: Span[];
}
export type EvaluationReport = TestCaseRun;
export interface AgentContextItem {
    description: string;
    value: string;
}
export interface AgentToolDefinition {
    name: string;
    description: string;
    parameters: {
        type: 'object';
        properties: Record<string, {
            type: string;
            description: string;
            enum?: string[];
        }>;
        required?: string[];
    };
}
export type Category = 'Baseline' | 'Smart Contextual Menu' | 'RCA' | 'Conversational Queries' | 'Top 10 Browsed Products' | 'Errors by Service' | 'Group by Error Type' | string;
export interface TestCaseVersion {
    version: number;
    createdAt: string;
    initialPrompt?: string;
    context: AgentContextItem[];
    tools?: AgentToolDefinition[];
    expectedPPL?: string;
    expectedOutcomes?: string[];
    expectedTrajectory?: {
        step: number;
        description: string;
        requiredTools: string[];
    }[];
    followUpQuestions?: {
        trigger: 'results_available' | 'error' | 'always';
        question: string;
        businessValue: string;
    }[];
}
export interface TestCase {
    id: string;
    name: string;
    description: string;
    labels: string[];
    /** @deprecated Use labels with 'category:' prefix instead */
    category: Category;
    /** @deprecated Use labels with 'subcategory:' prefix instead */
    subcategory?: string;
    /** @deprecated Use labels with 'difficulty:' prefix instead */
    difficulty: Difficulty;
    currentVersion: number;
    versions: TestCaseVersion[];
    sourceFile?: string;
    sourceHash?: string;
    isPromoted: boolean;
    createdAt: string;
    updatedAt: string;
    lastRunAt?: string;
    initialPrompt?: string;
    context: AgentContextItem[];
    tools?: AgentToolDefinition[];
    expectedPPL?: string;
    expectedOutcomes?: string[];
    expectedTrajectory?: {
        step: number;
        description: string;
        requiredTools: string[];
    }[];
    followUpQuestions?: {
        trigger: 'results_available' | 'error' | 'always';
        question: string;
        businessValue: string;
    }[];
}
export interface OpenSearchLog {
    timestamp: string;
    index: string;
    message: string;
    level?: string;
    source?: string;
    [key: string]: any;
}
export interface LogQueryParams {
    startTime: Date;
    endTime: Date;
    size?: number;
    query?: string;
}
export interface TraceMetrics {
    runId: string;
    traceId?: string;
    inputTokens: number;
    outputTokens: number;
    totalTokens: number;
    costUsd: number;
    durationMs: number;
    llmCalls: number;
    toolCalls: number;
    toolsUsed: string[];
    status: 'success' | 'error' | 'pending';
}
export interface SpanEvent {
    name: string;
    time: string;
    attributes?: Record<string, any>;
}
export interface Span {
    traceId: string;
    spanId: string;
    parentSpanId?: string;
    name: string;
    startTime: string;
    endTime: string;
    duration?: number;
    status: 'OK' | 'ERROR' | 'UNSET';
    attributes?: Record<string, any>;
    events?: SpanEvent[];
    children?: Span[];
    depth?: number;
    hasChildren?: boolean;
}
export interface TimeRange {
    startTime: number;
    endTime: number;
    duration: number;
}
export interface TraceQueryParams {
    traceId?: string;
    runIds?: string[];
    sessionId?: string;
    startTime?: number;
    endTime?: number;
    size?: number;
    serviceName?: string;
    textSearch?: string;
    cursor?: string;
    /**
     * Strategy C (opt-in): include any spans where `serviceName` matches AND
     * `startTime` falls within `[startedAt, endedAt]`. Used by the run-report
     * Traces tab as a fallback for agents that don't propagate W3C trace context
     * (TRACEPARENT) and don't tag spans with `gen_ai.request.id` matching our
     * runId. May surface unrelated spans (concurrent runs, cross-team noise).
     * See AGENTS.md → Trace correlation conventions.
     *
     * Strategy D: when `sessionId` is set on an entry, correlate precisely on
     * `attributes.session.id` (unioned with the service.name + window fallback).
     */
    agents?: Array<{
        serviceName: string;
        startedAt: number;
        endedAt: number;
        sessionId?: string;
    }>;
}
export interface ConversationMessage {
    id: string;
    timestamp: string;
    role: 'user' | 'assistant' | 'tool_call' | 'tool_result' | 'system';
    content: string;
    metadata?: {
        spanId?: string;
        spanName?: string;
        toolName?: string;
        model?: string;
        inputTokens?: number;
        outputTokens?: number;
        durationMs?: number;
    };
}
export interface TraceSearchResult {
    spans: Span[];
    total: number;
    warning?: string;
    warningCategory?: 'auth' | 'connection' | 'index_not_found' | 'not_configured' | 'unknown';
    suggestion?: string;
    nextCursor?: string | null;
    hasMore?: boolean;
}
/**
 * Summary of a single trace (grouped spans)
 * Used for trace list display before selecting one for detailed view
 */
export interface TraceSummary {
    traceId: string;
    serviceName: string;
    spanCount: number;
    rootSpanName: string;
    startTime: string;
    duration: number;
    hasErrors: boolean;
    hasEvalSpans?: boolean;
    spans: Span[];
}
/**
 * Span category based on OTel GenAI semantic conventions
 * @see https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-agent-spans/
 */
export type SpanCategory = 'AGENT' | 'LLM' | 'TOOL' | 'EVAL' | 'ERROR' | 'OTHER';
/**
 * Extended span with category metadata for tree visualization
 */
export interface CategorizedSpan extends Span {
    category: SpanCategory;
    categoryLabel: string;
    categoryColor: string;
    categoryIcon: string;
    displayName: string;
}
/**
 * Configuration for tool similarity grouping
 */
export interface ToolSimilarityConfig {
    /** Which tool arguments to use for determining "sameness" */
    keyArguments: string[];
    /** Whether grouping is enabled */
    enabled: boolean;
}
/**
 * Grouped tool spans for similarity view
 */
export interface ToolGroup {
    toolName: string;
    keyArgsValues: Record<string, any>;
    spans: CategorizedSpan[];
    count: number;
    totalDuration: number;
    avgDuration: number;
}
/**
 * Aligned span pair for tree comparison
 */
export interface AlignedSpanPair {
    type: 'matched' | 'added' | 'removed' | 'modified';
    leftSpan?: CategorizedSpan;
    rightSpan?: CategorizedSpan;
    similarity?: number;
    children?: AlignedSpanPair[];
}
/**
 * Result of comparing two trace trees
 */
export interface TraceComparisonResult {
    alignedTree: AlignedSpanPair[];
    stats: {
        totalLeft: number;
        totalRight: number;
        matched: number;
        added: number;
        removed: number;
        modified: number;
    };
}
/**
 * Data payload for span nodes in React Flow
 * Index signature required for React Flow compatibility
 */
export interface SpanNodeData extends Record<string, unknown> {
    span: CategorizedSpan;
    totalDuration: number;
}
/**
 * Result of transforming spans to React Flow format
 */
export interface FlowTransformResult {
    nodes: Node<SpanNodeData>[];
    edges: Edge[];
}
/**
 * Options for flow transformation
 */
export interface FlowTransformOptions {
    direction?: 'TB' | 'LR';
    mode?: 'hierarchy' | 'execution-order';
    nodeWidth?: number;
    nodeHeight?: number;
    nodeSpacingX?: number;
    nodeSpacingY?: number;
}
/**
 * Group of spans detected as parallel execution
 */
export interface ParallelGroup {
    spans: CategorizedSpan[];
    startTime: number;
    endTime: number;
}
/**
 * Result of checking OTEL GenAI semantic convention compliance
 */
export interface OTelComplianceResult {
    isCompliant: boolean;
    missingAttributes: string[];
}
/**
 * Compressed node for Intent view - represents one or more consecutive same-category spans
 */
export interface IntentNode {
    id: string;
    category: SpanCategory;
    spans: CategorizedSpan[];
    count: number;
    displayName: string;
    subtitle: string;
    hasWarnings: boolean;
    executionOrder: number;
    startIndex: number;
    totalDuration: number;
}
/**
 * Metadata about storage availability and data source
 * Included in list responses to inform clients about data provenance
 */
export interface StorageMetadata {
    /** Whether storage backend is configured (env vars set) */
    storageConfigured: boolean;
    /** Whether storage backend was reachable on this request */
    storageReachable: boolean;
    /** Count of items from persistent storage */
    realDataCount: number;
    /** Count of items from built-in sample data */
    sampleDataCount: number;
    /** Whether sample/demo data was included in this response */
    sampleDataIncluded?: boolean;
    /** Optional warning messages (e.g., connection errors) */
    warnings?: string[];
}
/**
 * Generic list response wrapper with metadata
 */
export interface ListResponse<T> {
    data: T[];
    total: number;
    meta: StorageMetadata;
}
/** Server-side performance metrics for a single test case evaluation */
export interface TestCasePerformanceMetrics {
    durationMs: number;
    agentDurationMs: number;
    judgeDurationMs?: number;
    judgeAttempts?: number;
}
/** Server-side performance metrics for an entire benchmark run */
export interface RunPerformanceMetrics {
    durationMs: number;
    concurrency: number;
    avgTestCaseDurationMs: number;
    maxTestCaseDurationMs: number;
    minTestCaseDurationMs: number;
}
export interface RunStats {
    /** Number of test cases that passed (passFailStatus === 'passed') */
    passed: number;
    /** Number of test cases that failed (passFailStatus === 'failed' or execution failed) */
    failed: number;
    /** Number of test cases still pending (running, or report not yet available) */
    pending: number;
    /**
     * Number of test cases where the *evaluator* could not produce a verdict
     * (e.g. judge validation error, trace polling timeout, post-trace callback
     * failed). Excluded from `passed` and `failed` so a misconfigured evaluator
     * doesn't silently poison aggregate pass rates.
     *
     * Optional for backward-compat: older stored runs predate this field and
     * read as 0.
     */
    errored?: number;
    /** Total number of test cases in the run */
    total: number;
}
export type RunResultStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
export type BenchmarkRunStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
export interface BenchmarkVersion {
    version: number;
    createdAt: string;
    testCaseIds: string[];
}
export interface TestCaseSnapshot {
    id: string;
    version: number;
    name: string;
}
export interface BenchmarkRun {
    id: string;
    name: string;
    description?: string;
    createdAt: string;
    status?: BenchmarkRunStatus;
    error?: string;
    agentKey: string;
    agentEndpoint?: string;
    modelId: string;
    /**
     * Optional judge model id, distinct from {@link modelId} (the agent's
     * LLM). Customer input via the run config dialog / CLI `--judge-model` /
     * API. Falls back to `evaluator.inferenceConfig.modelId`, then the
     * server-default Bedrock judge model. Ignored by agentic providers
     * (`pi`, `agent`, `agentic`, `claude-code`) which pick their own model.
     */
    judgeModelId?: string;
    evaluatorId?: string;
    headers?: Record<string, string>;
    concurrency?: number;
    benchmarkVersion?: number;
    testCaseSnapshots?: TestCaseSnapshot[];
    results: Record<string, {
        reportId: string;
        status: RunResultStatus;
        error?: string;
        performanceMetrics?: TestCasePerformanceMetrics;
    }>;
    stats?: RunStats;
    performanceMetrics?: RunPerformanceMetrics;
}
export interface Benchmark {
    id: string;
    name: string;
    description?: string;
    createdAt: string;
    updatedAt: string;
    currentVersion: number;
    versions: BenchmarkVersion[];
    testCaseIds: string[];
    runs: BenchmarkRun[];
}
export interface BenchmarkProgress {
    currentTestCaseIndex: number;
    startedCount?: number;
    completedCount?: number;
    totalTestCases: number;
    currentRunId: string;
    currentTestCaseId: string;
    status: 'running' | 'completed' | 'failed' | 'cancelled';
}
export interface BenchmarkStartedEvent {
    runId: string;
    testCases: Array<{
        id: string;
        name: string;
        status: 'pending';
    }>;
}
/** @deprecated Use BenchmarkRunStatus instead */
export type ExperimentRunStatus = BenchmarkRunStatus;
/** @deprecated Use BenchmarkRun instead */
export type ExperimentRun = BenchmarkRun;
/** @deprecated Use Benchmark instead */
export type Experiment = Benchmark;
/** @deprecated Use BenchmarkProgress instead */
export type ExperimentProgress = BenchmarkProgress;
/** @deprecated Use BenchmarkStartedEvent instead */
export type ExperimentStartedEvent = BenchmarkStartedEvent;
/**
 * Discriminator for documents in evals_benchmarks index.
 * Legacy docs without this field default to 'benchmark' via normalization.
 */
export type EvalDocType = 'benchmark' | 'evaluation-run';
/**
 * Describes where test cases came from for an evaluation run.
 * Multiple sources can be combined (union, deduplicated by test case ID).
 */
export type TestCaseSource = {
    type: 'benchmark';
    benchmarkId: string;
    benchmarkVersion?: number;
} | {
    type: 'test-case-ids';
    ids: string[];
} | {
    type: 'file-import';
    filenames: string[];
    testCaseIds: string[];
} | {
    type: 'code-import';
    filenames: string[];
    testCaseIds: string[];
} | {
    type: 'directory-import';
    dirPaths: string[];
    testCaseIds: string[];
} | {
    type: 'label-filter';
    labels: string[];
};
/**
 * EvaluationRun — first-class execution record.
 * Stored as top-level doc in evals_benchmarks index with docType: 'evaluation-run'.
 * Replaces embedded BenchmarkRun as primary execution entity.
 */
export interface EvaluationRun {
    id: string;
    docType: 'evaluation-run';
    name: string;
    description?: string;
    createdAt: string;
    completedAt?: string;
    status: BenchmarkRunStatus;
    error?: string;
    agentKey: string;
    agentEndpoint?: string;
    modelId: string;
    /**
     * Optional judge model id, distinct from {@link modelId} (the agent's
     * LLM). Same precedence rules as on {@link BenchmarkRun.judgeModelId}.
     */
    judgeModelId?: string;
    evaluatorId?: string;
    headers?: Record<string, string>;
    concurrency?: number;
    sources: TestCaseSource[];
    trigger: 'ui' | 'cli' | 'api' | 'schedule';
    testCaseSnapshots: TestCaseSnapshot[];
    results: Record<string, {
        reportId: string;
        status: RunResultStatus;
        error?: string;
        performanceMetrics?: TestCasePerformanceMetrics;
    }>;
    stats?: RunStats;
    performanceMetrics?: RunPerformanceMetrics;
    benchmarkId?: string;
    benchmarkVersion?: number;
}
export interface TestCaseVersionRef {
    id: string;
    version: string;
    hash: string;
}
export interface RunAggregateMetrics {
    runId: string;
    runName: string;
    createdAt: string;
    modelId: string;
    agentKey: string;
    totalTestCases: number;
    passedCount: number;
    failedCount: number;
    /** Test cases the evaluator couldn't verdict (#242); excluded from pass rate. */
    erroredCount?: number;
    avgAccuracy: number;
    passRatePercent: number;
    totalTokens?: number;
    totalInputTokens?: number;
    totalOutputTokens?: number;
    totalCostUsd?: number;
    avgDurationMs?: number;
    totalLlmCalls?: number;
    totalToolCalls?: number;
}
export interface TestCaseRunResult {
    reportId?: string;
    status: 'completed' | 'failed' | 'missing';
    passFailStatus?: PassFailStatus;
    /**
     * Issue #242: when the evaluator could not produce a verdict
     * (`metricsStatus: 'error'` on the report), the comparison row carries
     * this flag so MetricCell can render an amber `Errored` chip distinct
     * from `Failed`. The legacy `passFailStatus` field on these reports is
     * cleared (`null`), so without this flag the cell would silently fall
     * through to `Failed` styling.
     */
    errored?: boolean;
    accuracy?: number;
    faithfulness?: number;
    trajectoryAlignment?: number;
    latencyScore?: number;
    testCaseVersion?: string;
    /** Error message if status is 'failed' */
    error?: string;
}
export interface TestCaseComparisonRow {
    testCaseId: string;
    testCaseName: string;
    labels: string[];
    /** @deprecated Use labels instead */
    category: Category;
    /** @deprecated Use labels instead */
    difficulty: Difficulty;
    results: Record<string, TestCaseRunResult>;
    hasVersionDifference: boolean;
    versions: string[];
}
export type RunConfigInput = Pick<BenchmarkRun, 'name' | 'description' | 'agentKey' | 'modelId' | 'judgeModelId' | 'agentEndpoint' | 'headers' | 'concurrency' | 'evaluatorId'>;
import type { Request, Response } from 'express';
export interface TypedRequest<T = any> extends Request {
    body: T;
}
export interface TypedResponse<T = any> extends Response {
    json: (body: T) => this;
}
export interface ExpectedStep {
    description: string;
    requiredTools?: string[];
}
export interface JudgeRequest {
    trajectory: TrajectoryStep[];
    expectedTrajectory: ExpectedStep[];
    expectedOutcomes?: string[];
    logs?: OpenSearchLog[];
    modelId?: string;
    evaluatorId?: string;
}
export interface JudgeResponse {
    passFailStatus: PassFailStatus;
    metrics: EvaluationMetrics;
    llmJudgeReasoning: string;
    improvementStrategies: ImprovementStrategy[];
    duration: number;
}
export interface AgentProxyRequest {
    endpoint: string;
    payload: any;
    headers?: Record<string, string>;
}
export interface StorageConfig {
    endpoint?: string;
    username?: string;
    password?: string;
    indexes: {
        testCases: string;
        benchmarks: string;
        runs: string;
        analytics: string;
        evaluators: string;
    };
}
export interface OpenSearchConfig {
    endpoint: string;
    username: string;
    password: string;
    indexPattern: string;
}
export interface LogsQuery {
    runId?: string;
    query?: string;
    startTime?: number;
    endTime?: number;
    size?: number;
}
export interface LogsResponse {
    hits: {
        hits: any[];
        total: any;
    };
    logs: OpenSearchLog[];
    total: number;
}
export interface HealthStatus {
    status: 'ok' | 'error' | 'not_configured';
    error?: string;
    errorCategory?: 'auth' | 'connection' | 'index_not_found' | 'unknown';
    suggestion?: string;
    index?: string;
    cluster?: any;
}
export interface AggregateMetrics {
    totalRuns: number;
    successRate: number;
    totalCostUsd: number;
    avgCostUsd: number;
    avgDurationMs: number;
    p50DurationMs: number;
    p95DurationMs: number;
    avgTokens: number;
    totalInputTokens: number;
    totalOutputTokens: number;
    avgLlmCalls: number;
    avgToolCalls: number;
}
export interface MetricsResult {
    runId: string;
    traceId: string | null;
    inputTokens: number;
    outputTokens: number;
    totalTokens: number;
    costUsd: number;
    durationMs: number;
    llmCalls: number;
    toolCalls: number;
    toolsUsed: string[];
    status: 'pending' | 'success' | 'error';
}
/**
 * Authentication type for OpenSearch clusters
 * - 'none': No authentication (e.g. local development clusters)
 * - 'basic': Username/password authentication (default, backwards compatible)
 * - 'sigv4': AWS SigV4 request signing for managed OpenSearch / Serverless
 */
export type ClusterAuthType = 'none' | 'basic' | 'sigv4';
/**
 * Base cluster configuration (endpoint + credentials)
 * Used for connecting to OpenSearch or other data sources
 */
export interface ClusterConfig {
    endpoint: string;
    authType?: ClusterAuthType;
    username?: string;
    password?: string;
    awsProfile?: string;
    awsRegion?: string;
    awsService?: 'es' | 'aoss';
    tlsSkipVerify?: boolean;
}
/**
 * Storage cluster configuration - endpoint + credentials only
 * Index names (evals_test_cases, evals_experiments, evals_runs, evals_analytics)
 * are hardcoded in the adapter and not user-configurable.
 */
export type StorageClusterConfig = ClusterConfig;
/**
 * Observability cluster configuration - endpoint + credentials + OTEL index patterns
 * Used for traces, logs, and metrics from OpenTelemetry instrumentation
 */
export interface ObservabilityClusterConfig extends ClusterConfig {
    indexes?: {
        traces?: string;
        logs?: string;
        metrics?: string;
    };
}
/**
 * Full data source configuration stored in localStorage
 * Both storage and observability can point to the same or different clusters
 */
export interface DataSourceConfig {
    storage?: StorageClusterConfig;
    observability?: ObservabilityClusterConfig;
}
/**
 * Adapter type for data sources
 * 'file' is the default (JSON files in .agent-health/data/)
 * 'opensearch' when storage cluster is configured
 * 'memory' is for testing/demo
 */
export type DataSourceAdapterType = 'file' | 'opensearch' | 'memory';
export * from './skills.js';
//# sourceMappingURL=index.d.ts.map