import { apiRequest } from './common.ts' import { markdownToHtml } from './canvas.ts' // ============================================================================ // Types // ============================================================================ export interface TestCaseFile { index: number name: string mimeType: string fileUrl: string variableName: string url: string } export interface TestCaseMetadata { source?: 'workflow' | 'workstation' | 'craft' | 'test-case' taskId?: string } export interface TestCase { id: string title: string messages: Array<{ role: string, content: string }> variables: Record variablesRichContent?: Record files: TestCaseFile[] promptId: string expectedOutput?: string answers?: Record promptApplicationId?: string metadata: TestCaseMetadata workspaceId: string createdBy: string createdAt: string updatedAt: string deletedAt?: string } export interface Generation { id: string content: string type: 'text' | 'image' | 'audio' | 'json' | 'function' testCaseId?: string promptVersionId: string promptId?: string status: TestCaseStatus inputMessages: Array<{ role: string, content: string }> environment: 'test-case' | 'craft-preview' resolvedInput: string feedback?: 0 | 1 | null attributeFeedback?: Record validations?: Record variables: Record createdAt: string updatedAt: string startedAt?: string endedAt?: string } export type TestCaseStatus = | 'initializing' | 'ready' | 'starting' | 'running' | 'finalizing' | 'completed' | 'aborting' | 'aborted' | 'timeout' | 'failed' | 'pending_review' | 'pending_run' export interface TestCaseWithGeneration extends TestCase { generations?: Generation[] } // ============================================================================ // Create Test Case Payload // ============================================================================ export interface CreateTestCasePayload { title?: string promptId: string messages: Array<{ role: string, content: string }> variables?: Record | null variablesRichContent: Record | null files?: TestCaseFile[] | null expectedOutput?: string answers?: Record metadata?: TestCaseMetadata } // ============================================================================ // List Test Cases Options // ============================================================================ export interface ListTestCasesOptions { /** Filter by prompt version IDs */ promptVersionIds?: string[] /** Maximum number of records to return (default: 20) */ limit?: number /** Number of records to skip */ offset?: number /** Include related object associations */ objectLinks?: boolean /** Filter by records created after this date (ISO 8601) */ createdAtSince?: string /** Filter by records created before this date (ISO 8601) */ createdAtUntil?: string /** Filter by records updated after this date (ISO 8601) */ updatedAtSince?: string /** Filter by records updated before this date (ISO 8601) */ updatedAtUntil?: string /** Filter by creator user IDs */ createdBy?: string[] /** Search by test case title */ testCaseTitle?: string /** Filter by execution status */ status?: TestCaseStatus[] /** Filter by tag IDs */ tagIds?: string[] } export interface ListTestCasesResult { data: TestCaseWithGeneration[] meta: { totalCount: number limit: number offset: number } } // ============================================================================ // API Functions // ============================================================================ /** * List test cases for a prompt/canvas with pagination and filtering * * @param promptId - The canvas/prompt ID (required) * @param options - Pagination and filter options * @returns Paginated result with test cases and metadata * * @example * // Basic usage * const result = await listTestCases('PROMPT_ID') * * @example * // With pagination * const result = await listTestCases('PROMPT_ID', { limit: 10, offset: 20 }) * * @example * // With filters * const result = await listTestCases('PROMPT_ID', { * status: ['completed', 'failed'], * createdAtSince: '2024-01-01T00:00:00Z', * limit: 50 * }) */ export async function listTestCases( promptId: string, options?: ListTestCasesOptions, ): Promise { const params = new URLSearchParams({ promptId }) if (options?.promptVersionIds?.length) { params.set('promptVersionIds', options.promptVersionIds.join(',')) } if (options?.limit !== undefined) { params.set('limit', String(options.limit)) } if (options?.offset !== undefined) { params.set('offset', String(options.offset)) } if (options?.objectLinks !== undefined) { params.set('objectLinks', String(options.objectLinks)) } if (options?.createdAtSince) { params.set('createdAtSince', options.createdAtSince) } if (options?.createdAtUntil) { params.set('createdAtUntil', options.createdAtUntil) } if (options?.updatedAtSince) { params.set('updatedAtSince', options.updatedAtSince) } if (options?.updatedAtUntil) { params.set('updatedAtUntil', options.updatedAtUntil) } if (options?.createdBy?.length) { params.set('createdBy', options.createdBy.join(',')) } if (options?.testCaseTitle) { params.set('testCaseTitle', options.testCaseTitle) } if (options?.status?.length) { params.set('status', options.status.join(',')) } if (options?.tagIds?.length) { params.set('tagIds', options.tagIds.join(',')) } return apiRequest(`/test-case?${params.toString()}`) } /** * Get a single test case by ID */ export async function getTestCase(testCaseId: string): Promise { return apiRequest(`/test-case/${testCaseId}`) } /** * Create a new test case for a canvas */ export async function createTestCase(payload: CreateTestCasePayload): Promise { const resolved = await resolveTestCaseFileNames(payload) return apiRequest('/test-case', { method: 'POST', body: JSON.stringify({ ...resolved, title: resolved.title ?? 'Untitled Test Case', messages: resolved.messages ?? [], }), }) } /** * Create multiple test cases at once */ export async function createTestCases(payloads: CreateTestCasePayload[]): Promise { const resolved = await Promise.all(payloads.map(payload => resolveTestCaseFileNames(payload))) return apiRequest('/test-case', { method: 'POST', body: JSON.stringify(resolved.map(payload => ({ ...payload, title: payload.title ?? 'Untitled Test Case', messages: payload.messages ?? [], }))), }) } /** * Update an existing test case */ export async function updateTestCase( testCaseId: string, payload: Partial, ): Promise { return apiRequest(`/test-case/${testCaseId}`, { method: 'PATCH', body: JSON.stringify(payload), }) } /** * Delete a test case */ export async function deleteTestCase(testCaseId: string): Promise { await apiRequest(`/test-case/${testCaseId}`, { method: 'DELETE', }) } /** * Run a single test case against a prompt version * Returns execution info - poll getTestCase() to check status */ export async function runTestCase( testCaseId: string, promptVersionId: string, ): Promise<{ executionId: string, testCaseIds: string[], generationId: string }> { return apiRequest(`/test-case/${testCaseId}/run`, { method: 'POST', body: JSON.stringify({ promptVersionId }), }) } export interface RunTestCasesFilters { createdAtSince?: string createdAtUntil?: string updatedAtSince?: string updatedAtUntil?: string createdBy?: string[] testCaseTitle?: string status?: TestCaseStatus[] tagIds?: string[] } /** * Run test cases against a prompt version. * Returns execution info - poll listTestCases() to check status. * * Pass explicit ids, or omit them and use `filters` to run a slice of the suite. */ export async function runTestCases( testCaseIds: string[], promptVersionId: string, ): Promise<{ executionId: string, executionIds: string[], testCaseIds: string[] }> { return apiRequest('/test-case/run-all', { method: 'POST', body: JSON.stringify({ promptVersionId, testCaseIds }), }) } /** * Run an entire suite against a version, without enumerating ids first. * * Omitting both `testCaseIds` and `filters` runs **every** test case on the prompt — which is what * you want before comparing two versions, since a partial re-run makes the comparison meaningless. * * `filters` is unreliable: filtering by `tagIds` has been observed returning zero test case ids for * tags that demonstrably match. Check the returned `testCaseIds` before assuming anything ran, and * fall back to resolving the ids yourself with `listTestCases` and passing them explicitly. */ export async function runAllTestCases( promptVersionId: string, options?: { testCaseIds?: string[], filters?: RunTestCasesFilters }, ): Promise<{ executionId: string, executionIds: string[], testCaseIds: string[] }> { return apiRequest('/test-case/run-all', { method: 'POST', body: JSON.stringify({ promptVersionId, ...(options?.testCaseIds && { testCaseIds: options.testCaseIds }), ...(options?.filters && { filters: options.filters }), }), }) } /** * Run every test case on a version and wait for all of them to settle. * * Returns the test cases with their generations, ready to feed to `getTestCaseReview` or to * `getVersionStats` for the scoreboard. */ export async function runSuiteAndWait( promptId: string, promptVersionId: string, options?: { filters?: RunTestCasesFilters, intervalMs?: number, maxAttempts?: number }, ): Promise { const { testCaseIds } = await runAllTestCases(promptVersionId, { filters: options?.filters }) await Promise.all(testCaseIds.map(id => waitForTestCase(id, { intervalMs: options?.intervalMs, maxAttempts: options?.maxAttempts, }).catch(() => undefined))) const { data } = await listTestCases(promptId, { promptVersionIds: [promptVersionId] }) return data.filter(testCase => testCaseIds.includes(testCase.id)) } /** * Abort a running test case execution */ export async function abortTestCase( testCaseId: string, ): Promise<{ success: boolean, message: string }> { return apiRequest(`/test-case/${testCaseId}/abort`, { method: 'POST', }) } // ============================================================================ // Helper Functions // ============================================================================ /** * Create a test case payload with variable values * Automatically generates both markdown (variables) and HTML (variablesRichContent) */ const MIME_TYPES: Record = { '.pdf': 'application/pdf', '.doc': 'application/msword', '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', '.xls': 'application/vnd.ms-excel', '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', '.ppt': 'application/vnd.ms-powerpoint', '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.gif': 'image/gif', '.svg': 'image/svg+xml', '.webp': 'image/webp', '.csv': 'text/csv', '.txt': 'text/plain', '.json': 'application/json', '.xml': 'application/xml', '.html': 'text/html', '.mp3': 'audio/mpeg', '.mp4': 'video/mp4', '.wav': 'audio/wav', '.zip': 'application/zip', } function guessMimeType(filename: string): string { const ext = filename.slice(filename.lastIndexOf('.')).toLowerCase() return MIME_TYPES[ext] ?? 'application/octet-stream' } function generateFileNodeId(): string { const chars = '0123456789ABCDEFGHJKMNPQRSTVWXYZ' let id = '' for (let i = 0; i < 26; i++) id += chars[Math.floor(Math.random() * chars.length)] return id } function buildFileNodeHtml(vaultRef: string, fileName: string, mimeType: string, multimodal = true): string { const id = generateFileNodeId() return `

` } export interface VaultFileNameLookup { (vaultRef: string): Promise<{ originalFileName?: string | null mimeType?: string | null } | null | undefined> } async function defaultFileNameLookup(vaultRef: string) { // Imported lazily so callers that never touch files do not pull in the Vault SDK. const { getFileMetadata } = await import('./vault.ts') return getFileMetadata(vaultRef) } /** * Backfill file names and mime types from Vault metadata. * * A vault reference is content-addressed and carries no name, so a payload built without explicit * `fileNames` falls back to the variable name — extensionless, guessed as `application/octet-stream`, * and rejected by the API as "Unsupported file type". The upload already recorded the real * `originalFileName` and `mimeType`, so look them up instead of failing. */ export async function resolveTestCaseFileNames( payload: CreateTestCasePayload, lookup: VaultFileNameLookup = defaultFileNameLookup, ): Promise { const needsResolution = payload.files?.filter(file => !file.name.includes('.')) ?? [] if (!needsResolution.length) return payload const resolved = new Map() await Promise.all(needsResolution.map(async (file) => { try { const metadata = await lookup(file.fileUrl) const originalFileName = metadata?.originalFileName if (originalFileName) { resolved.set(file.fileUrl, { name: originalFileName, mimeType: metadata?.mimeType ?? guessMimeType(originalFileName), }) } } catch { // Leave the caller-provided name in place; the API error is more useful than ours. } })) if (!resolved.size) return payload let richContent = payload.variablesRichContent const files = payload.files!.map((file) => { const match = resolved.get(file.fileUrl) if (!match || file.name.includes('.')) return file const previous = richContent?.[file.variableName] if (previous) { richContent = { ...richContent, [file.variableName]: previous .replace(`filename="${file.name}"`, `filename="${match.name}"`) .replace(`filetype="${file.mimeType}"`, `filetype="${match.mimeType}"`), } } return { ...file, name: match.name, mimeType: match.mimeType } }) return { ...payload, files, variablesRichContent: richContent } } export function createTestCasePayload( promptId: string, variables: Record, options?: { title?: string, expectedOutput?: string, fileNames?: Record }, ): CreateTestCasePayload { const allVariables: Record = {} const variablesRichContent: Record = {} const files: TestCaseFile[] = [] let fileIndex = 0 for (const [key, value] of Object.entries(variables)) { if (typeof value === 'string' && value.startsWith('vault://')) { const fileName = options?.fileNames?.[key] ?? key const mimeType = guessMimeType(fileName) files.push({ index: fileIndex++, name: fileName, mimeType, fileUrl: value, variableName: key, url: value, }) allVariables[key] = '' variablesRichContent[key] = buildFileNodeHtml(value, fileName, mimeType) } else { allVariables[key] = value variablesRichContent[key] = markdownToHtml(typeof value === 'string' ? value : '') } } const firstTextValue = Object.entries(allVariables).find(([, v]) => v !== '')?.[1] const firstFileName = files[0]?.name return { promptId, title: options?.title ?? `Test: ${firstTextValue?.slice(0, 30) || firstFileName || 'Untitled'}...`, variables: allVariables, variablesRichContent, files: files.length ? files : undefined, messages: [], expectedOutput: options?.expectedOutput, metadata: { source: 'test-case' }, } } /** * Wait for a test case to complete (polls until done or timeout) */ /** * Extract files from a test case's rich content variables. * Parses `` tags with filename/fileurl attributes from variablesRichContent. * * @returns Array of { name, vaultUrl, variableName } */ export function extractTestCaseFiles( testCase: TestCase | TestCaseWithGeneration, ): Array<{ name: string; vaultUrl: string; variableName: string }> { const files: Array<{ name: string; vaultUrl: string; variableName: string }> = [] if (testCase.files?.length) { for (const f of testCase.files) { files.push({ name: f.name, vaultUrl: f.fileUrl, variableName: f.variableName }) } return files } // Fallback: parse from rich content variables const richContent = testCase.variablesRichContent ?? {} const fileRegex = /filename="([^"]+)"[^>]*fileurl="(vault:\/\/[^"]+)"/g for (const [varName, html] of Object.entries(richContent)) { let m: RegExpExecArray | null while ((m = fileRegex.exec(html)) !== null) { files.push({ name: m[1] ?? '', vaultUrl: m[2] ?? '', variableName: varName }) } } return files } /** * Download all files from a test case to a local directory. * Uses tela.downloadFile for each vault reference. * * @param testCase - The test case to download files from * @param outDir - Directory to save files to * @returns Array of { name, path, size } for each downloaded file */ /** * Reduce a stored file name to something safe to write inside a directory. * * The name travels from whoever uploaded the file, through Vault metadata and the test case's rich * content, so it is not ours to trust: a name carrying path separators would otherwise land outside * `outDir`. */ export function toSafeFileName(name: string, fallback = 'file'): string { const base = name.split(/[/\\]/).pop() ?? '' const cleaned = base.replace(/^\.+/, '').trim() return cleaned || fallback } /** * Make `name` unique against names already used, preserving the extension. * * Two files on one test case can legitimately share a name — different variables often carry * `invoice.pdf` — and writing both to the same path silently loses one. * * Comparison is **case-insensitive**: macOS and Windows treat `Invoice.PDF` and `invoice.pdf` as the * same file, so a case-sensitive check would let the second overwrite the first on exactly the * platforms this runs on. */ export function toUniqueFileName(name: string, taken: Set): string { const takenLower = new Set([...taken].map(entry => entry.toLowerCase())) if (!takenLower.has(name.toLowerCase())) return name const dot = name.lastIndexOf('.') const stem = dot > 0 ? name.slice(0, dot) : name const extension = dot > 0 ? name.slice(dot) : '' for (let suffix = 1; ; suffix++) { const candidate = `${stem}-${suffix}${extension}` if (!takenLower.has(candidate.toLowerCase())) return candidate } } export async function downloadTestCaseFiles( testCase: TestCase | TestCaseWithGeneration, outDir: string, downloadFn: (vaultUrl: string) => Promise, ): Promise> { const { mkdirSync, writeFileSync } = await import('node:fs') const { isAbsolute, join, relative, resolve } = await import('node:path') const files = extractTestCaseFiles(testCase) if (!files.length) return [] mkdirSync(outDir, { recursive: true }) const root = resolve(outDir) const taken = new Set() const results: Array<{ name: string, path: string, size: number }> = [] for (const [index, file] of files.entries()) { const name = toUniqueFileName(toSafeFileName(file.name, `file-${index}`), taken) const filePath = resolve(join(root, name)) // Compare via `relative` rather than a string prefix: on Windows `resolve` returns // backslashes, and a `${root}/` prefix check would reject every legitimate path. const inside = relative(root, filePath) if (!inside || inside.startsWith('..') || isAbsolute(inside)) { throw new Error( `Refusing to write "${file.name}" outside ${outDir}. Rename the file on the test case first.`, ) } taken.add(name) const blob = await downloadFn(file.vaultUrl) const buffer = Buffer.from(await blob.arrayBuffer()) writeFileSync(filePath, buffer) results.push({ name, path: filePath, size: buffer.length }) } return results } export async function waitForTestCase( testCaseId: string, options?: { intervalMs?: number, maxAttempts?: number }, ): Promise { const interval = options?.intervalMs ?? 2000 const maxAttempts = options?.maxAttempts ?? 150 // 5 minutes default for (let i = 0; i < maxAttempts; i++) { const testCase = await getTestCase(testCaseId) const latestGen = testCase.generations?.[0] if (latestGen?.status && ['completed', 'failed', 'aborted', 'timeout'].includes(latestGen.status)) { return testCase } await new Promise(resolve => setTimeout(resolve, interval)) } throw new Error(`Test case ${testCaseId} did not complete within timeout`) }