import { Compile } from "typebox/compile";
export { resolveWorkflowSkillPath } from "./eval-capture-extension.js";
import { type AgentIdentity, type JsonSchema, type JsonValue, type StaticWorkflowCall, type StaticWorkflowExecution, type WorkflowErrorCode } from "./index.js";
export type SignificantAction = {
    kind: "tool";
    name: string;
} | {
    kind: "text";
} | {
    kind: "thinking";
};
export type SequenceExpectation = readonly string[] | {
    equals?: readonly string[];
    startsWith?: readonly string[];
};
export type JsonResultType = "null" | "boolean" | "number" | "integer" | "string" | "array" | "object";
export interface JsonResultShape {
    type?: JsonResultType;
    equals?: JsonValue;
    nonEmpty?: boolean;
    requiredKeys?: readonly string[];
    propertyTypes?: Readonly<Record<string, JsonResultType>>;
    forbiddenProperties?: readonly string[];
    count?: number;
    minCount?: number;
    properties?: Readonly<Record<string, JsonResultShape>>;
}
export interface ExpectedReplayResult {
    workflowIndex?: number;
    equals?: JsonValue;
    match?: JsonResultShape;
}
export interface OutputSchemaShape {
    type?: string;
    requiredKeys?: readonly string[];
    propertyTypes?: Readonly<Record<string, string>>;
    forbiddenProperties?: readonly string[];
    count?: number;
    minCount?: number;
}
export interface AgentPolicyExpectation {
    callIndex: number;
    role?: string;
    model?: string;
    forbidOptions?: readonly ("role" | "model" | "thinking" | "tools" | "retries")[];
    tools?: {
        mode: "omitted" | "empty" | "exact";
        values?: readonly string[];
    };
}
export interface AgentStructureExpectation {
    execution: StaticWorkflowExecution;
    operation?: "parallel" | "pipeline";
    agents: readonly AgentOrderExpectation[];
}
export interface AgentOrderExpectation {
    role?: string;
    model?: string;
    promptIncludes?: string;
    execution?: StaticWorkflowExecution;
}
export interface DataFlowExpectation {
    binding: string;
    toAgentIndex: number;
}
export interface ParentAssistantBatch {
    index: number;
    parts: readonly JsonValue[];
    tools: readonly string[];
    usage?: ParentUsage;
}
export interface ParentToolResult {
    toolCallId?: string;
    details?: JsonValue;
    isError?: boolean;
    text?: string;
}
export interface ParentToolCall {
    name: string;
    arguments: JsonValue;
}
export interface ParentUsage {
    input: number;
    output: number;
    cacheRead: number;
    cacheWrite: number;
    totalTokens: number;
    cost: number;
    models: readonly {
        model: string;
        cost: number;
    }[];
}
export interface ParentOracle {
    assistantBatches: readonly ParentAssistantBatch[];
    workflowToolResults: readonly ParentToolResult[];
    skillReads: readonly string[];
    firstSignificantAction?: SignificantAction;
    firstTool?: string;
    firstBatchToolSequence: readonly string[];
    toolsBeforeFirstWorkflow: readonly string[];
    firstWorkflowBatchToolSequence: readonly string[];
    parentToolSequence: readonly string[];
    workflowCallCount: number;
    usage: ParentUsage;
}
export interface CapturedWorkflowCall {
    batch: number;
    toolCallId?: string;
    arguments: JsonValue;
    script?: string;
}
export interface EvalExpectations {
    firstSignificantAction?: SignificantAction;
    firstTool?: string;
    firstBatchToolSequence?: SequenceExpectation;
    parentToolSequence?: SequenceExpectation;
    workflowCallCount?: number | {
        min?: number;
        max?: number;
    };
    requiredOperations?: readonly StaticWorkflowCall["kind"][];
    forbiddenOperations?: readonly StaticWorkflowCall["kind"][];
    requiredRoles?: readonly string[];
    minimumAgentCalls?: number;
    requireOutputSchema?: OutputSchemaShape | boolean;
    expectedResults?: readonly ExpectedReplayResult[];
    agentPolicies?: readonly AgentPolicyExpectation[];
    requiredAgentOrder?: readonly AgentOrderExpectation[];
    requiredAgentStructures?: readonly AgentStructureExpectation[];
    requiredDataFlow?: readonly DataFlowExpectation[];
}
export interface SemanticCriterion {
    id: string;
    description: string;
}
export interface WorkflowEvalCase {
    id: string;
    prompt: string;
    timeoutMs?: number;
    maxCost: number;
    expectations: EvalExpectations;
    expectedWorkflowCalls?: number;
    semanticCriteria?: readonly SemanticCriterion[];
}
export interface ReplayAgentCall {
    prompt: string;
    options: Readonly<Record<string, JsonValue>>;
    identity: AgentIdentity;
}
export interface ReplayTrace {
    agentCalls: readonly ReplayAgentCall[];
    phases: readonly string[];
    logs: readonly string[];
    maxConcurrentAgents: number;
}
export interface ReplayReport {
    script: string;
    result?: JsonValue;
    trace?: ReplayTrace;
    error?: string;
}
export interface ProductionValidationReport {
    callIndex: number;
    valid: boolean;
    errorCode?: WorkflowErrorCode;
    message?: string;
}
export interface CriterionResult {
    id: string;
    pass: boolean;
    evidence: string;
}
export interface StaticCandidateReport {
    callIndices: readonly number[];
    criteria: readonly CriterionResult[];
    passed: boolean;
}
export interface SemanticJudgeReport {
    criteria: readonly CriterionResult[];
    usage: ParentUsage;
    raw: string;
}
export type EvalAccounting = ParentUsage;
export interface EvalMetrics {
    parentUsageThroughCandidate: ParentUsage | null;
    parentOutputTokensThroughCandidate: number | null;
    nonWorkflowToolSequenceBeforeCandidate: readonly string[];
    nonWorkflowToolCallCountBeforeCandidate: number;
    workflowCallCountBeforeCandidate: number;
    invalidWorkflowCallCount: number;
    productionValidationErrorCodes: readonly string[];
    candidateCallIndices: readonly number[];
    staticCandidates: readonly StaticCandidateReport[];
    semanticCriteria: readonly CriterionResult[];
    anyValidCandidate: boolean;
    requiredWorkflowCallCount: number;
    surplusWorkflowCallCount: number;
}
export interface EvalCaseResult {
    id: string;
    status: "passed" | "failed" | "timed_out" | "budget_exceeded" | "skipped";
    limits: {
        timeoutMs?: number;
        maxCost: number;
    };
    oracle?: ParentOracle;
    workflows: readonly CapturedWorkflowCall[];
    productionValidation: readonly ProductionValidationReport[];
    semanticJudge?: SemanticJudgeReport;
    metrics: EvalMetrics;
    accounting: EvalAccounting;
    accountingTrustworthy: boolean;
    diagnostics: readonly string[];
    errors: readonly string[];
    cleanup: {
        processExited: boolean;
        processGroupTerminated: boolean;
        tempRootRemoved: boolean;
        captureIdentityVerified: boolean;
        realWorkflowAgentsLaunched: number | null;
    };
}
export declare const SAFE_PARENT_EVAL_TOOLS: readonly ["read", "grep", "find", "bash", "workflow", "workflow_retry"];
export declare function validateWorkflowEvalCases(values: readonly unknown[], source?: string): readonly WorkflowEvalCase[];
export declare function loadWorkflowEvalCases(directory?: string): readonly WorkflowEvalCase[];
export declare const INITIAL_WORKFLOW_EVAL_CASES: readonly WorkflowEvalCase[];
export declare function matchesJsonResult(shape: JsonResultShape, value: JsonValue): boolean;
export declare function matchesOutputSchema(shape: OutputSchemaShape, schema: JsonSchema): boolean;
export declare function extractParentOracle(entries: readonly unknown[]): ParentOracle;
export declare function extractParentToolCalls(oracle: ParentOracle): ParentToolCall[];
export declare function extractParentOracleFile(path: string): ParentOracle;
export declare function extractCapturedWorkflows(oracle: ParentOracle): CapturedWorkflowCall[];
export declare function captureValidationReports(oracle: ParentOracle, calls: readonly CapturedWorkflowCall[]): {
    reports: ProductionValidationReport[];
    errors: string[];
    verified: boolean;
};
export declare function evalExpectationErrors(oracle: ParentOracle, expectations: EvalExpectations): string[];
export declare function recoverySelectionErrors(evalCase: Pick<WorkflowEvalCase, "id">, oracle: ParentOracle): string[];
export declare function replayExpectationErrors(calls: readonly CapturedWorkflowCall[], reports: readonly ReplayReport[], expectations: EvalExpectations): string[];
export declare function staticExpectationResults(calls: readonly CapturedWorkflowCall[], expectations: EvalExpectations): CriterionResult[];
export declare function selectStaticCandidate(calls: readonly CapturedWorkflowCall[], validations: readonly ProductionValidationReport[], expectations: EvalExpectations, requiredCount?: number): {
    callIndices: readonly number[];
    reports: readonly StaticCandidateReport[];
};
export declare function assertEvalScriptSafe(script: string): void;
type JsonSchemaValidator = ReturnType<typeof Compile>;
export declare function matchesJsonSchema(schema: JsonSchemaValidator, value: JsonValue): boolean;
export declare function replayWorkflowScript(script: string, args?: JsonValue, signal?: AbortSignal): Promise<{
    result: JsonValue;
    trace: ReplayTrace;
}>;
export interface CaptureCaseInput {
    case: WorkflowEvalCase;
    model: string;
    provider?: string;
    thinking?: string;
    piCommand?: string;
    maxCost: number;
}
export declare function parseSemanticJudge(raw: string, criteria: readonly SemanticCriterion[]): CriterionResult[];
export declare function findSessionFile(directory: string, sessionId: string): string | undefined;
export declare function captureEvalCase(input: CaptureCaseInput): Promise<EvalCaseResult>;
export interface IsolatedProcessOptions {
    childPath: string;
    timeoutMs?: number;
    env?: NodeJS.ProcessEnv;
    onStderr?: (chunk: string) => void;
}
export interface IsolatedProcessResult {
    value?: unknown;
    timedOut: boolean;
    exitCode: number | null;
    processGroupTerminated: boolean;
    stderr: string;
    error?: string;
}
export declare function runIsolatedProcess(payload: unknown, options: IsolatedProcessOptions): Promise<IsolatedProcessResult>;
export interface WorkflowEvalRunOptions {
    cases?: readonly WorkflowEvalCase[];
    caseIds?: readonly string[];
    model?: string;
    provider?: string;
    thinking?: string;
    piCommand?: string;
    timeoutMs?: number;
    spendCeiling?: number;
    artifactsDir?: string;
    onProgress?: (message: string) => void;
}
export interface WorkflowEvalRunResult {
    artifactDir: string;
    cases: readonly EvalCaseResult[];
    spent: number;
    skipped: readonly string[];
}
export declare function runWorkflowEvals(options?: WorkflowEvalRunOptions): Promise<WorkflowEvalRunResult>;
export declare function formatEvalSummary(result: WorkflowEvalRunResult): string;