import type { TestContext } from "node:test"; import { type RepositoryBenchmarkCaseV1, type RepositoryFoundryGenerationPlanV1, type RepositoryHiddenFixtureSuiteV1, type RepositorySeedCrossBatchQualificationV1 } from "@velum-labs/routekit-eval-setup"; /** * Synthetic TEST-ONLY host metadata over a real two-commit Git repository. * Nothing here represents a provider receipt, independent benchmark admission, * a production qualification, or a live/hidden foundry experiment. */ export type EvalTrialSyntheticHostArtifact = { readonly version: 1; readonly plan: RepositoryFoundryGenerationPlanV1; readonly qualification: RepositorySeedCrossBatchQualificationV1; readonly generation: { readonly version: 1; readonly benchmarkCase: RepositoryBenchmarkCaseV1; readonly sourceContext: { readonly initialCommit: string; readonly referenceCommit: string; readonly initialSupportingPaths: readonly string[]; readonly referenceSupportingPaths: readonly string[]; }; readonly fixtureProposal: { readonly fixtures: RepositoryHiddenFixtureSuiteV1["fixtures"]; readonly overlays: RepositoryHiddenFixtureSuiteV1["overlays"]; readonly knownLimitations: readonly string[]; }; }; /** Proves that full private artifact bytes, not just visible JSON, are bound. */ readonly hostOnlyAnnotation: string; }; export declare const evalTrialSourceSentinels: { readonly preparation: "SYNTHETIC_HOST_PREPARATION_COMMAND"; readonly publicTest: "SYNTHETIC_PUBLIC_CHECKOUT_TEST"; readonly referenceTest: "SYNTHETIC_REFERENCE_ONLY_REGRESSION"; readonly hiddenFixture: "SYNTHETIC_INDEPENDENT_HIDDEN_FIXTURE"; readonly referenceSolutionId: "synthetic-host-reference-solution"; readonly hostAnnotation: "Synthetic host-only bytes: café 🌍; not candidate-visible."; }; export declare const evalTrialSourcePaths: { readonly source: "src/add.js"; readonly publicTest: "src/add.test.js"; readonly protected: "package.json"; }; export declare const evalTrialInitialSource = "export const add = (left, right) => left - right;\n"; export declare const evalTrialCorrectSource = "export const add = (left, right) => left + right;\n"; export declare const evalTrialSourceRecipeIds: { readonly preparation: "synthetic-trusted-preparation"; readonly validation: "synthetic-public-build"; readonly baseline: "synthetic-baseline-control"; readonly grade: "synthetic-public-grade"; }; export declare const evalTrialSourceRawDigest: (value: string | Uint8Array) => string; /** * Both lanes deliberately use real immutable Git metadata. Request-lane tests * do not execute any repository harness, model, gateway, or external service. */ export declare function makeEvalTrialSourceFixture(t: TestContext, options?: { readonly lane?: "repository-agent" | "request"; }): Promise<{ repositoryRoot: string; benchmarkArtifactPath: string; benchmarkArtifactDigest: string; artifact: EvalTrialSyntheticHostArtifact; artifactText: string; visibleTask: { readonly lane: "request"; readonly id: string; readonly request: { readonly endpoint: "responses" | "chat" | "anthropic"; readonly method: "POST"; readonly path: string; readonly headers: readonly { readonly name: string; readonly value: string; }[]; readonly bodyJson: string; }; } | { readonly lane: "repository-agent"; readonly id: string; readonly requestText: string; readonly checkoutCommit: string; readonly allowedTools: readonly ("search" | "shell" | "edit" | "read" | "test")[]; readonly constraints: readonly string[]; }; seed: { readonly status: "rejected" | "candidate" | "qualified"; readonly id: string; readonly version: 1; readonly source: { readonly kind: "historical-fix"; readonly changeEpisodeId: string; } | { readonly kind: "historical-feature"; readonly changeEpisodeId: string; } | { readonly kind: "historical-refactor"; readonly changeEpisodeId: string; } | { readonly kind: "existing-test"; readonly testIds: readonly string[]; } | { readonly kind: "public-api"; readonly surfaceIds: readonly string[]; } | { readonly kind: "schema-contract"; readonly protocolIds: readonly string[]; } | { readonly kind: "production-request"; readonly requestId: string; } | { readonly kind: "reviewed-synthetic-gap"; readonly reviewId: string; }; readonly rejectionReasons: readonly string[]; readonly summary: string; readonly capabilityEvidence: readonly { readonly id: string; readonly kind: "symbol" | "command" | "protocol" | "test" | "schema" | "package" | "public-surface" | "history-episode" | "workload-signal" | "documentation"; readonly path?: string | undefined; }[]; readonly initialState: { readonly commit: string; readonly parentCommit?: string | undefined; }; readonly targetBehavior: readonly { readonly id: string; readonly description: string; readonly critical: boolean; readonly evidence: readonly { readonly id: string; readonly kind: "symbol" | "command" | "protocol" | "test" | "schema" | "package" | "public-surface" | "history-episode" | "workload-signal" | "documentation"; readonly path?: string | undefined; }[]; }[]; readonly baselineObservations: readonly { readonly id: string; readonly kind: "schema" | "test-command" | "property" | "event-trace" | "structural"; readonly subjectId: string; readonly outcome: "unknown" | "fail" | "pass"; readonly detail: string; }[]; readonly preChangeObservations: readonly { readonly id: string; readonly kind: "schema" | "test-command" | "property" | "event-trace" | "structural"; readonly subjectId: string; readonly outcome: "unknown" | "fail" | "pass"; readonly detail: string; }[]; readonly postChangeObservations: readonly { readonly id: string; readonly kind: "schema" | "test-command" | "property" | "event-trace" | "structural"; readonly subjectId: string; readonly outcome: "unknown" | "fail" | "pass"; readonly detail: string; }[]; readonly candidateTestIds: readonly string[]; readonly environment: { readonly protectedControlPaths: readonly string[]; readonly trustedPreparationRecipes: readonly { readonly id: string; readonly kind: "custom" | "check" | "test" | "build" | "schema"; readonly cwd: string; readonly executable: string; readonly args: readonly string[]; readonly timeoutMs: number; }[]; readonly candidateValidationRecipes: readonly { readonly id: string; readonly kind: "custom" | "check" | "test" | "build" | "schema"; readonly cwd: string; readonly executable: string; readonly args: readonly string[]; readonly timeoutMs: number; }[]; readonly baselineGradeRecipes: readonly { readonly id: string; readonly kind: "custom" | "check" | "test" | "build" | "schema"; readonly cwd: string; readonly executable: string; readonly args: readonly string[]; readonly timeoutMs: number; }[]; readonly gradeRecipes: readonly { readonly id: string; readonly kind: "custom" | "check" | "test" | "build" | "schema"; readonly cwd: string; readonly executable: string; readonly args: readonly string[]; readonly timeoutMs: number; }[]; readonly packageManager?: { readonly name: "pnpm"; readonly version: string; } | undefined; readonly lockfilePath?: string | undefined; }; readonly confidence: "medium" | "high" | "review-required"; readonly risks: readonly ("review-required" | "ambiguous-intent" | "bundled-change" | "external-service" | "flaky-baseline" | "reference-only-oracle" | "weak-test-delta" | "unrealistic-mutation" | "private-context" | "cross-batch-instability")[]; readonly referenceCommit?: string | undefined; }; suite: { readonly version: 1; readonly caseId: string; readonly fixtures: readonly ({ readonly kind: "historical-regression"; readonly expectationMode: "changes"; readonly id: string; readonly description: string; readonly expectedBehavior: string; readonly source: "historical" | "generated" | "reviewed"; readonly testPath: string; } | { readonly kind: "boundary"; readonly expectationMode: "changes" | "preserved"; readonly id: string; readonly description: string; readonly expectedBehavior: string; readonly source: "historical" | "generated" | "reviewed"; readonly testPath: string; } | { readonly kind: "counterfactual"; readonly expectationMode: "changes"; readonly id: string; readonly description: string; readonly expectedBehavior: string; readonly source: "historical" | "generated" | "reviewed"; readonly testPath: string; } | { readonly kind: "metamorphic"; readonly expectationMode: "preserved"; readonly id: string; readonly description: string; readonly expectedBehavior: string; readonly source: "historical" | "generated" | "reviewed"; readonly testPath: string; })[]; readonly overlays: readonly { readonly path: string; readonly content: string; readonly fixtureIds: readonly string[]; }[]; }; initialCommit: string; initialTree: string; referenceCommit: string; referenceTree: string; writeArtifact: (value?: unknown) => Promise; }>;