import { dirname } from 'node:path'; import type { AgentMessage, AgentProvider } from '@duckcodeailabs/dql-agent'; export type CassetteMode = 'record' | 'replay' | 'live'; /** One recorded tool call, so replay can re-run it against the live tools. */ export interface CassetteToolCall { name: string; args: unknown; } /** * How this response entered the cassette store. `providerName` is part of the * dispatch fingerprint; it is not itself a claim that the response was * recorded from that provider. */ export type CassetteProvenance = { kind: 'recorded_provider'; replayClassification: 'recorded_provider'; providerQuality: 'eligible'; } | { kind: 'migrated_legacy_deterministic_fixture'; sourceLegacyKey: string; replayClassification: 'orchestration_replay_only'; providerQuality: 'excluded'; } | { /** * A deliberately authored response over the checked-in sanitized * fixture. It exercises replay/orchestration only; it is never presented * as a live-provider response or provider-quality evidence. */ kind: 'synthetic_deterministic_orchestration_fixture'; replayClassification: 'orchestration_replay_only'; providerQuality: 'excluded'; createdAt: string; creationMethod: 'sanitized_fixture_sql'; source: 'current_scoped_runtime_dispatch'; }; export interface CassetteEntry { key: string; operation: 'generate' | 'generate_with_tools'; /** * Legacy timestamp for a response recorded from a provider or imported from * an older cassette. Synthetic fixtures use `createdAt` instead so they do * not imply a live provider recording. */ recordedAt?: string; /** Creation timestamp for an explicitly synthetic deterministic fixture. */ createdAt?: string; providerName: string; /** The model's final text. */ text: string; /** Tool calls the model made, in order (generate_with_tools only). */ toolCalls?: CassetteToolCall[]; /** * Provenance is additive so historical entries remain readable. Entries * without it are deliberately excluded from real-provider quality claims. */ provenance?: CassetteProvenance; /** * V2 stores only hashes and structural dispatch facts. It never stores a * prompt, credentials, or retrieved business context beyond the model text * that a cassette has always intentionally contained. */ fingerprintDiagnostics?: CassetteFingerprintDiagnosticsV2; } /** Eval-only canonicalization. Production dispatches never opt into this. */ export interface EvalCassetteCanonicalizationV2 { version: 2; /** Exact absolute project-root prefix supplied by the eval host. */ projectRoot: string; } export interface CassetteFingerprintDiagnosticsV2 { version: 2; messageCount: number; messageRoles: AgentMessage['role'][]; preCanonicalHash: string; postCanonicalHash: string; appliedRuleClasses: string[]; } export interface CassetteFingerprint { key: string; /** Exact legacy full-prompt hash, retained only for safe v1 replay migration. */ legacyKey?: string; diagnostics?: CassetteFingerprintDiagnosticsV2; } /** * What a cassette directory can support in an eval report. A migrated or * historical-unknown entry still supports deterministic orchestration replay; * neither is allowed to stand in for a current provider-quality measurement. */ export interface CassetteEvidenceSummary { totalEntries: number; recordedProviderEntries: number; migratedLegacyDeterministicFixtureEntries: number; syntheticDeterministicOrchestrationFixtureEntries: number; unknownProvenanceEntries: number; orchestrationReplayEligible: boolean; realProviderQualityEligible: boolean; realProviderQualityExclusionReasons: string[]; } export declare function evalCassetteCanonicalizationV2(projectRoot: string): EvalCassetteCanonicalizationV2; /** * Stable key for one provider dispatch. * * Deliberately includes the full message list: the prompt IS the input, and two * turns that differ only in retrieved context are genuinely different calls. * Tool NAMES are included but not their schemas — a description edit should not * invalidate every cassette in the suite. */ export declare function cassetteFingerprint(input: { providerName: string; operation: CassetteEntry['operation']; messages: AgentMessage[]; toolNames?: string[]; options?: { reasoningEffort?: unknown; maxTokens?: unknown; temperature?: unknown; } | undefined; canonicalization?: EvalCassetteCanonicalizationV2; }): CassetteFingerprint; export declare function cassetteKey(input: Parameters[0]): string; export declare class CassetteStore { private readonly dir; private readonly loaded; constructor(dir: string); get(key: string): CassetteEntry | undefined; put(entry: CassetteEntry): void; size(): number; entries(): CassetteEntry[]; /** * A replay-only runtime with no configured provider still needs the identity * that was used to derive cassette keys. Keep that identity in the cassette * store rather than guessing a provider from user settings. */ providerNames(): string[]; } export declare function cassetteEvidenceSummary(store: CassetteStore): CassetteEvidenceSummary; export declare class CassetteMissError extends Error { readonly key: string; readonly fingerprintDiagnostics?: CassetteFingerprintDiagnosticsV2; constructor(key: string, operation: string, fingerprintDiagnostics?: CassetteFingerprintDiagnosticsV2); } /** * Wrap a provider so its dispatches are recorded or replayed. * * On replay of a tool loop the recorded tool calls are RE-INVOKED against the * live tools rather than stubbed. The model's choices stay frozen (that is the * point), but compilation, validation, and execution still really happen — so * the suite keeps catching a broken compiler or a wrong number, which a fully * stubbed replay would sail straight past. */ export declare function withCassette(provider: AgentProvider, store: CassetteStore, mode: CassetteMode, canonicalization?: EvalCassetteCanonicalizationV2): AgentProvider; /** * Pick the cassette mode from the environment. * * Defaults to `replay` for ANY value that is not explicitly `record` or `live`, * including a missing or misspelled one. If someone sets the cassette directory * in CI and forgets the mode, the safe outcome is "never call a live model", * not a surprise bill and a non-deterministic suite. */ export declare function resolveCassetteModeFromEnv(env: Record): CassetteMode; /** Where a project's eval cassettes live. */ export declare function cassetteDirFor(projectRoot: string, suiteName: string): string; export { dirname as _dirname }; //# sourceMappingURL=agent-eval-cassette.d.ts.map