import { beforeEach, describe, expect, mock, test } from 'bun:test' const apiRequest = mock(async (_endpoint: string, _options?: RequestInit): Promise => ({ data: {} })) class ApiRequestError extends Error { constructor(public readonly status: number, message: string) { super(message) this.name = 'ApiRequestError' } } void mock.module('./common.ts', () => ({ apiRequest, ApiRequestError, getAuthenticatedWorkspaceId: () => 'workspace-id', })) describe('agent-test-cases', () => { beforeEach(() => { apiRequest.mockClear() apiRequest.mockImplementation(async () => ({ data: {} })) }) test('lists test cases with repeated array query params', async () => { const { listAgentTestCases } = await import('./agent-test-cases.ts') apiRequest.mockImplementation(async () => ({ data: [] })) await listAgentTestCases('agent-1', { commitHash: 'abc123', title: 'search', status: ['failed', 'error'], tagIds: ['t1', 't2'], createdBy: ['u1'], createdAtSince: '2026-01-01T00:00:00Z', }) const [endpoint] = apiRequest.mock.calls[0]! const query = new URLSearchParams(String(endpoint).split('?')[1]) expect(String(endpoint).startsWith('/agent/agent-1/tests?')).toBe(true) expect(query.get('commitHash')).toBe('abc123') expect(query.get('title')).toBe('search') expect(query.getAll('status')).toEqual(['failed', 'error']) expect(query.getAll('tagIds')).toEqual(['t1', 't2']) expect(query.getAll('createdBy')).toEqual(['u1']) expect(query.get('createdAtSince')).toBe('2026-01-01T00:00:00Z') }) test('omits the query string when no options are given', async () => { const { listAgentTestCases } = await import('./agent-test-cases.ts') apiRequest.mockImplementation(async () => ({ data: [] })) await listAgentTestCases('agent-1') expect(apiRequest.mock.calls[0]![0]).toBe('/agent/agent-1/tests') }) test('pre-serializes bodies so vault refs are not rewritten by parsePayload', async () => { const { createAgentTestCase } = await import('./agent-test-cases.ts') await createAgentTestCase('agent-1', { inputs: [{ type: 'file', name: 'doc', vaultRef: 'vault://abc', filename: 'doc.pdf' }], }) const [endpoint, options] = apiRequest.mock.calls[0]! expect(endpoint).toBe('/agent/agent-1/tests') expect(options?.method).toBe('POST') expect(typeof options?.body).toBe('string') const body = JSON.parse(String(options?.body)) expect(body.inputs[0].vaultRef).toBe('vault://abc') }) test('unwraps the { data } envelope', async () => { const { createAgentTestCase } = await import('./agent-test-cases.ts') apiRequest.mockImplementation(async () => ({ data: { id: 'tc-1', title: 'Test' } })) const testCase = await createAgentTestCase('agent-1', { title: 'Test' }) expect(testCase).toEqual({ id: 'tc-1', title: 'Test' } as any) }) test('runs a test case against a commit', async () => { const { runAgentTestCase } = await import('./agent-test-cases.ts') apiRequest.mockImplementation(async () => ({ data: { executionId: 'exec-1' } })) const result = await runAgentTestCase('agent-1', 'tc-1', 'abc123') const [endpoint, options] = apiRequest.mock.calls[0]! expect(endpoint).toBe('/agent/agent-1/tests/tc-1/run') expect(JSON.parse(String(options?.body))).toEqual({ commitHash: 'abc123' }) expect(result.executionId).toBe('exec-1') }) test('continues a session with a message', async () => { const { continueAgentTestCase } = await import('./agent-test-cases.ts') apiRequest.mockImplementation(async () => ({ data: { executionId: 'exec-2' } })) await continueAgentTestCase('agent-1', 'tc-1', 'abc123', 'next turn') const [endpoint, options] = apiRequest.mock.calls[0]! expect(endpoint).toBe('/agent/agent-1/tests/tc-1/continue') expect(JSON.parse(String(options?.body))).toEqual({ commitHash: 'abc123', message: 'next turn' }) }) test('sends attribute feedback wrapped in attributes', async () => { const { updateAgentTestCaseAttributeFeedback } = await import('./agent-test-cases.ts') await updateAgentTestCaseAttributeFeedback('agent-1', 'tc-1', 'abc123', [ { path: 'summary', feedback: 1 }, { path: 'total', feedback: 0, message: 'wrong' }, ]) const [endpoint, options] = apiRequest.mock.calls[0]! expect(endpoint).toBe('/agent/agent-1/tests/tc-1/runs/abc123/attribute-feedback') expect(options?.method).toBe('PATCH') const body = JSON.parse(String(options?.body)) expect(body.attributes).toHaveLength(2) expect(body.attributes[1]).toEqual({ path: 'total', feedback: 0, message: 'wrong' }) }) test('requires commitHash on the stats endpoint query', async () => { const { getAgentTestCaseStats } = await import('./agent-test-cases.ts') await getAgentTestCaseStats('agent-1', 'abc123', { tagIds: ['t1'] }) const [endpoint] = apiRequest.mock.calls[0]! const query = new URLSearchParams(String(endpoint).split('?')[1]) expect(String(endpoint).startsWith('/agent/agent-1/tests/stats?')).toBe(true) expect(query.get('commitHash')).toBe('abc123') expect(query.getAll('tagIds')).toEqual(['t1']) }) test('run history keeps pagination fields from the response envelope', async () => { const { listAgentTestCaseRuns } = await import('./agent-test-cases.ts') // This endpoint's envelope is { data: entries, nextCursor, totalRuns } — // pagination fields live beside data, not inside it apiRequest.mockImplementation(async () => ({ data: [{ commitHash: 'abc123', score: 1 }], nextCursor: 'cursor-1', totalRuns: 5, })) const result = await listAgentTestCaseRuns('agent-1', 'tc-1', { limit: 1 }) expect(apiRequest.mock.calls[0]![0]).toBe('/agent/agent-1/tests/tc-1/runs?limit=1') expect(result.data).toHaveLength(1) expect(result.nextCursor).toBe('cursor-1') expect(result.totalRuns).toBe(5) }) describe('buildAgentTestInputs', () => { test('splits text and vault file values', async () => { const { buildAgentTestInputs } = await import('./agent-test-cases.ts') const inputs = buildAgentTestInputs({ context: 'Some text', document: 'vault://abc123', }, { fileNames: { document: 'report.pdf' } }) expect(inputs).toEqual([ { type: 'text', name: 'context', content: 'Some text' }, { type: 'file', name: 'document', vaultRef: 'vault://abc123', filename: 'report.pdf' }, ]) }) test('defaults the filename to the input name', async () => { const { buildAgentTestInputs } = await import('./agent-test-cases.ts') const inputs = buildAgentTestInputs({ document: 'vault://abc123' }) expect(inputs[0]).toEqual({ type: 'file', name: 'document', vaultRef: 'vault://abc123', filename: 'document' }) }) }) describe('createAgentTestCasePayload', () => { test('builds payload with expectations and derived title', async () => { const { createAgentTestCasePayload } = await import('./agent-test-cases.ts') const payload = createAgentTestCasePayload({ input: 'Hello world, this is a long input value' }, { evaluationInstructions: 'Must greet back', }) expect(payload.title).toBe(`Test: ${'Hello world, this is a long input value'.slice(0, 30)}...`) expect(payload.evaluationInstructions).toBe('Must greet back') expect(payload.inputs).toHaveLength(1) }) test('falls back to the first filename for file-only cases', async () => { const { createAgentTestCasePayload } = await import('./agent-test-cases.ts') const payload = createAgentTestCasePayload({ doc: 'vault://abc' }, { fileNames: { doc: 'contract.pdf' } }) expect(payload.title).toBe('Test: contract.pdf') }) }) describe('getAgentTestCaseRunStatus', () => { const baseRun = { executionSessionId: null as string | null, executionStatus: null as any, evaluationSessionId: null as string | null, evaluationStatus: null as any, feedback: null as 0 | 1 | null, } test('derives every status branch', async () => { const { getAgentTestCaseRunStatus } = await import('./agent-test-cases.ts') expect(getAgentTestCaseRunStatus(null)).toBe('new') expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'error' })).toBe('error') expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed', evaluationStatus: 'error' })).toBe('error') expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'running' })).toBe('running') expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed', evaluationSessionId: 's', evaluationStatus: 'running' })).toBe('running') expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed' })).toBe('executed') expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed', evaluationSessionId: 's', evaluationStatus: 'completed', feedback: 1 })).toBe('success') expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed', evaluationSessionId: 's', evaluationStatus: 'completed', feedback: 0 })).toBe('failed') expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed', evaluationSessionId: 's', evaluationStatus: 'failed' })).toBe('failed') expect(getAgentTestCaseRunStatus({ ...baseRun, executionSessionId: 's' })).toBe('executed') }) }) describe('waitForAgentTestCaseRun', () => { test('retries 404s until the run appears, then resolves on terminal status', async () => { const { waitForAgentTestCaseRun } = await import('./agent-test-cases.ts') let calls = 0 apiRequest.mockImplementation(async () => { calls++ if (calls === 1) throw new ApiRequestError(404, 'API request failed: 404 Not Found - {}') if (calls === 2) { return { data: { executionStatus: 'running', executionSessionId: 's', evaluationSessionId: null, evaluationStatus: null, feedback: null } } } return { data: { executionStatus: 'completed', executionSessionId: 's', evaluationSessionId: 'e', evaluationStatus: 'completed', feedback: 1 } } }) const { status } = await waitForAgentTestCaseRun('agent-1', 'tc-1', 'abc123', { intervalMs: 1 }) expect(status).toBe('success') expect(calls).toBe(3) }) test('stops at executed when until is executed', async () => { const { waitForAgentTestCaseRun } = await import('./agent-test-cases.ts') apiRequest.mockImplementation(async () => ({ data: { executionStatus: 'completed', executionSessionId: 's', evaluationSessionId: null, evaluationStatus: null, feedback: null }, })) const { status } = await waitForAgentTestCaseRun('agent-1', 'tc-1', 'abc123', { intervalMs: 1, until: 'executed' }) expect(status).toBe('executed') }) test('throws after maxAttempts without a terminal status', async () => { const { waitForAgentTestCaseRun } = await import('./agent-test-cases.ts') apiRequest.mockImplementation(async () => ({ data: { executionStatus: 'running', executionSessionId: 's', evaluationSessionId: null, evaluationStatus: null, feedback: null }, })) expect(waitForAgentTestCaseRun('agent-1', 'tc-1', 'abc123', { intervalMs: 1, maxAttempts: 2 })) .rejects .toThrow('did not complete within timeout') }) test('rethrows non-404 API errors', async () => { const { waitForAgentTestCaseRun } = await import('./agent-test-cases.ts') apiRequest.mockImplementation(async () => { throw new ApiRequestError(500, 'API request failed: 500 Internal Server Error - {}') }) expect(waitForAgentTestCaseRun('agent-1', 'tc-1', 'abc123', { intervalMs: 1 })) .rejects .toThrow('500') }) test('rethrows untyped errors even when the message mentions 404', async () => { const { waitForAgentTestCaseRun } = await import('./agent-test-cases.ts') apiRequest.mockImplementation(async () => { throw new Error('fetch failed with 404 somewhere') }) expect(waitForAgentTestCaseRun('agent-1', 'tc-1', 'abc123', { intervalMs: 1 })) .rejects .toThrow('fetch failed') }) }) describe('getLatestAgentCommit', () => { test('returns the newest commit hash', async () => { const { getLatestAgentCommit } = await import('./agent-test-cases.ts') apiRequest.mockImplementation(async () => ({ data: { commits: [{ commitHash: 'abc123' }], branch: 'main', pagination: { page: 1, limit: 1, totalCount: 5, hasMore: true } }, })) const hash = await getLatestAgentCommit('agent-1') expect(apiRequest.mock.calls[0]![0]).toBe('/agent/agent-1/history?limit=1') expect(hash).toBe('abc123') }) test('throws when the agent has no commits', async () => { const { getLatestAgentCommit } = await import('./agent-test-cases.ts') apiRequest.mockImplementation(async () => ({ data: { commits: [], branch: 'main', pagination: { page: 1, limit: 1, totalCount: 0, hasMore: false } }, })) expect(getLatestAgentCommit('agent-1')).rejects.toThrow('has no commits') }) }) })