/**
 * Evaluation System Types
 *
 * Types for the calibration and evaluation harness that measures
 * scanner precision, recall, and stability.
 *
 * @module eval/types
 */
import type { Severity } from "../certification/types.js";
/**
 * A labeled test case with known vulnerabilities
 */
export interface TestFixture {
    /** Unique identifier for this fixture */
    id: string;
    /** Human-readable name */
    name: string;
    /** Description of what this fixture tests */
    description: string;
    /** Category of vulnerability */
    category: VulnerabilityCategory;
    /** Source of the fixture (e.g., "OWASP WebGoat", "custom") */
    source: string;
    /** Files in this fixture */
    files: FixtureFile[];
    /** Expected findings (ground truth) */
    expectedFindings: ExpectedFinding[];
    /** Tags for filtering */
    tags?: string[];
}
/**
 * A file within a test fixture
 */
export interface FixtureFile {
    /** Relative path within fixture */
    path: string;
    /** File content */
    content: string;
    /** Language for syntax highlighting */
    language: string;
}
/**
 * An expected finding (ground truth label)
 */
export interface ExpectedFinding {
    /** File where the issue exists */
    file: string;
    /** Line number (1-indexed) */
    line: number;
    /** Expected severity */
    severity: Severity;
    /** Category of vulnerability */
    category: VulnerabilityCategory;
    /** CWE ID if applicable */
    cweId?: string;
    /** Brief description of the issue */
    description: string;
    /** Whether this is a true positive or false positive test */
    isVulnerable: boolean;
}
/**
 * Vulnerability categories for classification
 */
export type VulnerabilityCategory = "sql-injection" | "xss" | "secrets" | "rls-bypass" | "type-safety" | "dependency-vuln" | "auth-bypass" | "path-traversal" | "command-injection" | "ssrf" | "xxe" | "insecure-deserialization" | "broken-access-control" | "security-misconfiguration" | "other";
/**
 * Result of running evaluation on a single fixture
 */
export interface FixtureResult {
    /** Fixture that was evaluated */
    fixtureId: string;
    /** Whether the scan completed successfully */
    success: boolean;
    /** Error message if failed */
    error?: string;
    /** Findings discovered by scanners */
    actualFindings: ActualFinding[];
    /** Expected findings from ground truth */
    expectedFindings: ExpectedFinding[];
    /** True positives (correctly identified) */
    truePositives: MatchedFinding[];
    /** False positives (incorrectly flagged) */
    falsePositives: ActualFinding[];
    /** False negatives (missed) */
    falseNegatives: ExpectedFinding[];
    /** Duration in milliseconds */
    duration: number;
}
/**
 * A finding discovered by the scanner
 */
export interface ActualFinding {
    /** Scanner that found this */
    scanner: string;
    /** Rule ID */
    ruleId: string;
    /** File path */
    file: string;
    /** Line number */
    line: number;
    /** Severity */
    severity: Severity;
    /** Message */
    message: string;
    /** Confidence (always 100 for deterministic scanners) */
    confidence: number;
}
/**
 * A matched finding (true positive)
 */
export interface MatchedFinding {
    expected: ExpectedFinding;
    actual: ActualFinding;
}
/**
 * Aggregate metrics across all fixtures
 */
export interface EvaluationMetrics {
    /** Total fixtures evaluated */
    totalFixtures: number;
    /** Fixtures that completed successfully */
    successfulFixtures: number;
    /** Fixtures that failed */
    failedFixtures: number;
    /** Total expected findings */
    totalExpected: number;
    /** Total actual findings */
    totalActual: number;
    /** Total true positives */
    totalTruePositives: number;
    /** Total false positives */
    totalFalsePositives: number;
    /** Total false negatives */
    totalFalseNegatives: number;
    /** Precision: TP / (TP + FP) */
    precision: number;
    /** Recall: TP / (TP + FN) */
    recall: number;
    /** F1 Score: 2 * (precision * recall) / (precision + recall) */
    f1Score: number;
    /** Metrics by category */
    byCategory: Record<VulnerabilityCategory, CategoryMetrics>;
    /** Metrics by scanner */
    byScanner: Record<string, ScannerMetrics>;
    /** Total evaluation duration */
    totalDuration: number;
    /** Timestamp */
    timestamp: string;
}
/**
 * Metrics for a specific category
 */
export interface CategoryMetrics {
    expected: number;
    truePositives: number;
    falsePositives: number;
    falseNegatives: number;
    precision: number;
    recall: number;
    f1Score: number;
}
/**
 * Metrics for a specific scanner
 */
export interface ScannerMetrics {
    findings: number;
    truePositives: number;
    falsePositives: number;
    precision: number;
}
/**
 * Stability metrics from multiple runs
 */
export interface StabilityMetrics {
    /** Number of runs */
    runs: number;
    /** Findings that appeared in all runs */
    consistentFindings: number;
    /** Findings that appeared in some runs */
    inconsistentFindings: number;
    /** Stability percentage (consistent / total) */
    stabilityPercent: number;
    /** Per-fixture stability */
    byFixture: Record<string, number>;
}
/**
 * Complete evaluation report
 */
export interface EvaluationReport {
    /** Version of the evaluation harness */
    version: string;
    /** When the evaluation was run */
    timestamp: string;
    /** Git commit SHA if available */
    commitSha?: string;
    /** Overall metrics */
    metrics: EvaluationMetrics;
    /** Stability metrics if multiple runs */
    stability?: StabilityMetrics;
    /** Per-fixture results */
    fixtureResults: FixtureResult[];
    /** Configuration used */
    config: EvaluationConfig;
}
/**
 * Evaluation configuration
 */
export interface EvaluationConfig {
    /** Which scanners to run */
    scanners: string[];
    /** Fixtures to include (glob patterns) */
    includeFixtures?: string[];
    /** Fixtures to exclude (glob patterns) */
    excludeFixtures?: string[];
    /** Number of runs for stability testing */
    stabilityRuns?: number;
    /** Timeout per fixture in milliseconds */
    timeout?: number;
    /** Whether to run in parallel */
    parallel?: boolean;
    /**
     * Directory of versioned custom regression fixtures (minted from logged
     * misses). Merged with the built-in fixtures when set. See
     * {@link ../eval/custom-fixtures}.
     */
    customFixturesDir?: string;
}
/**
 * Default evaluation configuration
 */
export declare const DEFAULT_EVAL_CONFIG: EvaluationConfig;
/**
 * Target metrics for publication
 */
export declare const TARGET_METRICS: {
    precision: number;
    recall: number;
    stability: number;
    agreement: number;
};
//# sourceMappingURL=types.d.ts.map