import { describe, it, expect } from "vitest"; import { ObserveInputSchema, CompressOutputSchema, SummaryOutputSchema, SearchInputSchema, ContextInputSchema, RememberInputSchema, } from "../src/eval/schemas.js"; import { validateInput, validateOutput } from "../src/eval/validator.js"; import { scoreCompression, scoreSummary, scoreContextRelevance, } from "../src/eval/quality.js"; describe("Zod Schemas", () => { describe("ObserveInputSchema", () => { it("accepts valid input", () => { const result = ObserveInputSchema.safeParse({ hookType: "post_tool_use", sessionId: "ses_abc", project: "my-project", cwd: "/home/user", timestamp: "2026-01-01T00:00:00Z", data: { tool_name: "Read" }, }); expect(result.success).toBe(true); }); it("rejects missing sessionId", () => { const result = ObserveInputSchema.safeParse({ hookType: "post_tool_use", project: "my-project", cwd: "/home/user", timestamp: "2026-01-01T00:00:00Z", data: {}, }); expect(result.success).toBe(false); }); it("rejects invalid hookType", () => { const result = ObserveInputSchema.safeParse({ hookType: "invalid_hook", sessionId: "ses_abc", project: "my-project", cwd: "/home/user", timestamp: "2026-01-01T00:00:00Z", data: {}, }); expect(result.success).toBe(false); }); }); describe("CompressOutputSchema", () => { it("accepts valid output", () => { const result = CompressOutputSchema.safeParse({ type: "file_edit", title: "Edit auth module", facts: ["Added JWT validation"], narrative: "Modified the auth middleware to validate tokens", concepts: ["auth"], files: ["src/auth.ts"], importance: 7, }); expect(result.success).toBe(true); }); it("rejects empty facts array", () => { const result = CompressOutputSchema.safeParse({ type: "file_edit", title: "Edit auth module", facts: [], narrative: "Modified the auth middleware to validate tokens", concepts: [], files: [], importance: 5, }); expect(result.success).toBe(false); }); it("rejects title over 120 chars", () => { const result = CompressOutputSchema.safeParse({ type: "file_edit", title: "x".repeat(121), facts: ["fact"], narrative: "A narrative that is long enough", concepts: [], files: [], importance: 5, }); expect(result.success).toBe(false); }); it("rejects importance outside 1-10", () => { const result = CompressOutputSchema.safeParse({ type: "file_edit", title: "Test", facts: ["fact"], narrative: "A valid narrative here", concepts: [], files: [], importance: 11, }); expect(result.success).toBe(false); }); it("rejects narrative under 10 chars", () => { const result = CompressOutputSchema.safeParse({ type: "file_edit", title: "Test", facts: ["fact"], narrative: "short", concepts: [], files: [], importance: 5, }); expect(result.success).toBe(false); }); }); describe("SummaryOutputSchema", () => { it("accepts valid summary", () => { const result = SummaryOutputSchema.safeParse({ title: "Session Summary", narrative: "This session focused on implementing authentication features and fixing bugs", keyDecisions: ["Use JWT"], filesModified: ["auth.ts"], concepts: ["auth"], }); expect(result.success).toBe(true); }); it("rejects short narrative", () => { const result = SummaryOutputSchema.safeParse({ title: "Summary", narrative: "Too short", keyDecisions: [], filesModified: [], concepts: [], }); expect(result.success).toBe(false); }); }); describe("SearchInputSchema", () => { it("accepts valid search", () => { expect(SearchInputSchema.safeParse({ query: "auth" }).success).toBe(true); }); it("accepts search with limit", () => { expect( SearchInputSchema.safeParse({ query: "auth", limit: 10 }).success, ).toBe(true); }); it("rejects empty query", () => { expect(SearchInputSchema.safeParse({ query: "" }).success).toBe(false); }); }); describe("ContextInputSchema", () => { it("accepts valid input", () => { expect( ContextInputSchema.safeParse({ sessionId: "ses_1", project: "proj", }).success, ).toBe(true); }); }); describe("RememberInputSchema", () => { it("accepts valid input", () => { expect( RememberInputSchema.safeParse({ content: "Always use TypeScript", type: "preference", }).success, ).toBe(true); }); it("rejects empty content", () => { expect( RememberInputSchema.safeParse({ content: "" }).success, ).toBe(false); }); }); }); describe("Validator", () => { it("returns valid with correct data", () => { const result = validateInput(SearchInputSchema, { query: "test" }, "search"); expect(result.valid).toBe(true); if (result.valid) { expect(result.data.query).toBe("test"); } }); it("returns invalid with error details", () => { const result = validateInput(SearchInputSchema, { query: "" }, "search"); expect(result.valid).toBe(false); if (!result.valid) { expect(result.result.functionId).toBe("search"); expect(result.result.errors.length).toBeGreaterThan(0); } }); it("validateOutput works same as validateInput", () => { const result = validateOutput( CompressOutputSchema, { type: "file_edit", title: "Test", facts: ["a"], narrative: "A long enough narrative", concepts: [], files: [], importance: 5, }, "compress", ); expect(result.valid).toBe(true); }); }); describe("Quality Scoring", () => { describe("scoreCompression", () => { it("returns 0 for empty object", () => { expect(scoreCompression({})).toBe(0); }); it("returns 100 for perfect observation", () => { const score = scoreCompression({ type: "file_edit", title: "A good title", facts: ["fact 1", "fact 2", "fact 3"], narrative: "A narrative that is definitely more than fifty characters long and provides good context", concepts: ["auth", "jwt"], importance: 7, }); expect(score).toBe(100); }); it("scores partial observations between 0 and 100", () => { const score = scoreCompression({ title: "Test", facts: ["one"], narrative: "Short but valid narrative", }); expect(score).toBeGreaterThan(0); expect(score).toBeLessThan(100); }); }); describe("scoreSummary", () => { it("returns 0 for empty object", () => { expect(scoreSummary({})).toBe(0); }); it("returns high score for complete summary", () => { const score = scoreSummary({ title: "Session Summary Title", narrative: "This is a detailed narrative about what happened during the session with enough content to be meaningful and complete for review purposes", keyDecisions: ["Used JWT for auth", "Chose PostgreSQL"], filesModified: ["src/auth.ts", "src/db.ts"], concepts: ["authentication", "database"], }); expect(score).toBeGreaterThanOrEqual(90); }); }); describe("scoreContextRelevance", () => { it("returns 0 for empty context", () => { expect(scoreContextRelevance("", "proj")).toBe(0); }); it("scores higher when project is mentioned", () => { const withProject = scoreContextRelevance( "This is for my-project with details", "my-project", ); const without = scoreContextRelevance( "Some generic context details", "my-project", ); expect(withProject).toBeGreaterThan(without); }); it("scores higher with more XML sections", () => { const multi = scoreContextRelevance( "ABCD", "test", ); const single = scoreContextRelevance("A", "test"); expect(multi).toBeGreaterThan(single); }); }); });