import { Command } from 'commander'; import * as fs from 'fs'; import ora from 'ora'; import chalk from 'chalk'; import { IndemnClient } from '../sdk/client.js'; import { EvalSDK } from '../sdk/eval.js'; import { RubricsSDK } from '../sdk/rubrics.js'; import { TestSetsSDK } from '../sdk/test-sets.js'; import { exportEvalMarkdown, exportEvalReport } from '../report/export.js'; import { formatTable, printJson, printError, printSuccess, confirm } from './utils/output.js'; export function registerEvalCommands(program: Command): void { const eval_ = program.command('eval').description('Run and manage evaluations'); eval_ .command('run') .argument('', 'Agent ID') .description('Run an evaluation') .requiredOption('--test-set ', 'Test set ID') .option('--rubric ', 'Rubric ID') .option('--concurrency ', 'Concurrency level') .option('--wait', 'Wait for evaluation to complete') .option('--eval-model ', 'Model for evaluation judging') .option('--keep', 'Keep evaluation data after completion') .option('--limit ', 'Limit number of items to evaluate') .option('--component-scope ', 'Component scope filter') .option('--component-ids ', 'Comma-separated component IDs') .option('--conversation-ids ', 'Comma-separated conversation IDs') .option('--mode ', 'Evaluation mode (e.g. "transcript")') .action(async (agentId: string, options) => { try { const client = new IndemnClient(); const sdk = new EvalSDK(client); const spinner = ora('Triggering evaluation...').start(); const input: Record = { bot_id: agentId, test_set_id: options.testSet, }; if (options.rubric) input.rubric_id = options.rubric; if (options.concurrency) input.concurrency = parseInt(options.concurrency, 10); if (options.evalModel) input.eval_model = options.evalModel; if (options.keep !== undefined) input.keep = options.keep; if (options.limit) input.limit = parseInt(options.limit, 10); if (options.componentScope) input.component_scope = options.componentScope; if (options.componentIds) input.component_ids = options.componentIds.split(','); if (options.conversationIds) input.conversation_ids = options.conversationIds.split(','); if (options.mode) input.mode = options.mode; const result = await sdk.triggerEvaluation(input as any); spinner.stop(); printSuccess(`Evaluation started: ${result.run_id}`); console.log(` Status: ${result.status}`); if (result.total) console.log(` Total items: ${result.total}`); if (options.wait) { const pollSpinner = ora('Waiting for evaluation to complete...').start(); try { const run = await sdk.pollRunUntilComplete(result.run_id); pollSpinner.stop(); printSuccess(`Evaluation completed: ${run.status}`); if (run.passed !== undefined) console.log(` Passed: ${run.passed}`); if (run.failed !== undefined) console.log(` Failed: ${run.failed}`); if (run.criteria_passed !== undefined && run.criteria_total !== undefined) { console.log(` Criteria: ${run.criteria_passed}/${run.criteria_total}`); } } catch (err) { pollSpinner.fail('Evaluation did not complete in time'); throw err; } } } catch (err) { printError(err); process.exit(1); } }); eval_ .command('list') .description('List evaluation runs') .option('--agent-id ', 'Filter by agent ID') .option('--json', 'Output as JSON') .option('--limit ', 'Limit number of results') .option('--page ', 'Page number (use with --limit)') .action(async (options) => { try { const client = new IndemnClient(); const sdk = new EvalSDK(client); const spinner = ora('Fetching evaluation runs...').start(); let runs = await sdk.listRuns(options.agentId); spinner.stop(); if (options.page && options.limit) { const page = parseInt(options.page, 10); const limit = parseInt(options.limit, 10); runs = runs.slice((page - 1) * limit, page * limit); } else if (options.limit) { runs = runs.slice(0, parseInt(options.limit, 10)); } if (options.json) { printJson(runs); return; } if (runs.length === 0) { console.log('No evaluation runs found.'); return; } const rows = runs.map((r) => [ r.run_id || '', r.status || '', r.agent_id || '', r.passed !== undefined ? String(r.passed) : '', r.failed !== undefined ? String(r.failed) : '', r.started_at ? new Date(r.started_at).toLocaleDateString() : '', ]); console.log(formatTable(['Run ID', 'Status', 'Agent', 'Passed', 'Failed', 'Started'], rows)); } catch (err) { printError(err); process.exit(1); } }); eval_ .command('status') .argument('', 'Evaluation run ID') .description('Get evaluation run status') .option('--json', 'Output as JSON') .action(async (runId: string, options) => { try { const client = new IndemnClient(); const sdk = new EvalSDK(client); const spinner = ora('Fetching run status...').start(); const run = await sdk.getRunStatus(runId); spinner.stop(); if (options.json) { printJson(run); return; } const statusColor = run.status === 'completed' ? chalk.green : run.status === 'failed' ? chalk.red : chalk.yellow; console.log(`Run ID: ${run.run_id}`); console.log(`Status: ${statusColor(run.status)}`); if (run.agent_id) console.log(`Agent: ${run.agent_id}`); if (run.total !== undefined) console.log(`Total: ${run.total}`); if (run.completed !== undefined) console.log(`Done: ${run.completed}`); if (run.passed !== undefined) console.log(`Passed: ${run.passed}`); if (run.failed !== undefined) console.log(`Failed: ${run.failed}`); if (run.criteria_passed !== undefined && run.criteria_total !== undefined) { console.log(`Criteria: ${run.criteria_passed}/${run.criteria_total}`); } if (run.started_at) console.log(`Started: ${run.started_at}`); if (run.completed_at) console.log(`Finished: ${run.completed_at}`); if (run.error) console.log(`Error: ${chalk.red(run.error)}`); } catch (err) { printError(err); process.exit(1); } }); eval_ .command('results') .argument('', 'Evaluation run ID') .description('Get evaluation results') .option('--json', 'Output as JSON') .action(async (runId: string, options) => { try { const client = new IndemnClient(); const sdk = new EvalSDK(client); const spinner = ora('Fetching results...').start(); const results = await sdk.getRunResults(runId); spinner.stop(); if (options.json) { printJson(results); return; } if (results.length === 0) { console.log('No results available.'); return; } // Print results as formatted JSON since structure varies printJson(results); } catch (err) { printError(err); process.exit(1); } }); eval_ .command('bot-context') .argument('', 'Agent ID') .description('Get bot context used for evaluations') .option('--json', 'Output as JSON') .action(async (agentId: string, options) => { try { const client = new IndemnClient(); const sdk = new EvalSDK(client); const spinner = ora('Fetching bot context...').start(); const context = await sdk.getBotContext(agentId); spinner.stop(); if (options.json) { printJson(context); return; } if (context.system_prompt) { console.log(chalk.bold('System Prompt:')); console.log(context.system_prompt); console.log(); } if (context.tools && Array.isArray(context.tools)) { console.log(chalk.bold(`Tools (${context.tools.length}):`)); for (const tool of context.tools) { console.log(` - ${tool.name || tool._id}: ${tool.description || ''}`); } console.log(); } if (context.knowledge_bases && Array.isArray(context.knowledge_bases)) { console.log(chalk.bold(`Knowledge Bases (${context.knowledge_bases.length}):`)); for (const kb of context.knowledge_bases) { console.log(` - ${kb.name || kb._id}: ${kb.type || ''}`); } } } catch (err) { printError(err); process.exit(1); } }); eval_ .command('export') .argument('', 'Evaluation run ID') .description('Export evaluation results as markdown') .option('--output ', 'Output file path') .action(async (runId: string, options) => { try { const client = new IndemnClient(); const spinner = ora('Fetching evaluation data...').start(); const result = await exportEvalMarkdown(runId, options.output, client); spinner.stop(); printSuccess(`Exported to ${result.file_path}`); } catch (err) { printError(err); process.exit(1); } }); eval_ .command('report') .argument('', 'Evaluation run ID') .description('Generate branded evaluation report PDF') .option('--output ', 'Output file path') .action(async (runId: string, options) => { try { const client = new IndemnClient(); const spinner = ora('Fetching evaluation data...').start(); spinner.text = 'Generating PDF...'; const result = await exportEvalReport(runId, options.output, client); spinner.stop(); printSuccess(`Report saved to ${result.file_path}`); } catch (err) { printError(err); process.exit(1); } }); // Rubric commands — registered on program, not nested under eval const rubric = program.command('rubric').description('Manage evaluation rubrics'); rubric .command('list') .description('List rubrics') .option('--agent-id ', 'Filter by agent ID') .option('--json', 'Output as JSON') .option('--limit ', 'Limit number of results') .option('--page ', 'Page number (use with --limit)') .action(async (options) => { try { const client = new IndemnClient(); const sdk = new RubricsSDK(client); const spinner = ora('Fetching rubrics...').start(); let rubrics = await sdk.listRubrics(options.agentId); spinner.stop(); if (options.page && options.limit) { const page = parseInt(options.page, 10); const limit = parseInt(options.limit, 10); rubrics = rubrics.slice((page - 1) * limit, page * limit); } else if (options.limit) { rubrics = rubrics.slice(0, parseInt(options.limit, 10)); } if (options.json) { printJson(rubrics); return; } if (rubrics.length === 0) { console.log('No rubrics found.'); return; } const rows = rubrics.map((r) => [ r._id || r.rubric_id || '', r.name, String(r.version || 1), String(r.rules?.length || 0), r.updatedAt ? new Date(r.updatedAt).toLocaleDateString() : '', ]); console.log(formatTable(['ID', 'Name', 'Version', 'Rules', 'Updated'], rows)); } catch (err) { printError(err); process.exit(1); } }); rubric .command('get') .argument('', 'Rubric ID') .description('Get rubric details') .option('--json', 'Output as JSON') .option('--revision ', 'Get a specific version/revision') .action(async (id: string, options) => { try { const client = new IndemnClient(); const sdk = new RubricsSDK(client); const spinner = ora('Fetching rubric...').start(); const r = options.revision ? await sdk.getVersion(id, parseInt(options.revision, 10)) : await sdk.getRubric(id); spinner.stop(); if (options.json) { printJson(r); return; } console.log(`ID: ${r._id || r.rubric_id}`); console.log(`Name: ${r.name}`); console.log(`Description: ${r.description || '(none)'}`); console.log(`Version: ${r.version || 1}`); console.log(`Rules: ${r.rules?.length || 0}`); if (r.rules?.length) { console.log(); for (const rule of r.rules) { console.log(` [${rule.severity}] ${rule.name}: ${rule.description}`); } } } catch (err) { printError(err); process.exit(1); } }); rubric .command('create') .description('Create a rubric') .option('--file ', 'Path to JSON file with rubric definition') .option('--name ', 'Rubric name') .option('--agent-id ', 'Agent ID') .action(async (options) => { try { let input: Record; if (options.file) { const filePath = options.file as string; if (!fs.existsSync(filePath)) { printError(`File not found: ${filePath}`); process.exit(1); } const content = fs.readFileSync(filePath, 'utf-8'); try { input = JSON.parse(content); } catch { printError('File must contain valid JSON.'); process.exit(1); } // Merge CLI options into file content (CLI flags override file values) if (options.name) input.name = options.name; if (options.agentId) input.agent_id = options.agentId; } else if (options.name) { input = { name: options.name, rules: [] }; if (options.agentId) input.agent_id = options.agentId; } else { printError('Provide --file with rubric JSON or --name for an empty rubric.'); process.exit(1); } const client = new IndemnClient(); const sdk = new RubricsSDK(client); const spinner = ora('Creating rubric...').start(); const rubricResult = await sdk.createRubric(input as any); spinner.stop(); printSuccess(`Rubric created: ${rubricResult._id || rubricResult.rubric_id}`); console.log(` Name: ${rubricResult.name}`); console.log(` Rules: ${rubricResult.rules?.length || 0}`); } catch (err) { printError(err); process.exit(1); } }); rubric .command('update') .argument('', 'Rubric ID') .description('Update a rubric') .option('--file ', 'Path to JSON file with rubric updates') .option('--name ', 'New rubric name') .action(async (id: string, options) => { try { let input: Record; if (options.file) { const filePath = options.file as string; if (!fs.existsSync(filePath)) { printError(`File not found: ${filePath}`); process.exit(1); } const content = fs.readFileSync(filePath, 'utf-8'); try { input = JSON.parse(content); } catch { printError('File must contain valid JSON.'); process.exit(1); } } else if (options.name) { input = { name: options.name }; } else { printError('Provide --file with rubric JSON or --name.'); process.exit(1); } const client = new IndemnClient(); const sdk = new RubricsSDK(client); const spinner = ora('Updating rubric...').start(); await sdk.updateRubric(id, input as any); spinner.stop(); printSuccess(`Rubric ${id} updated.`); } catch (err) { printError(err); process.exit(1); } }); rubric .command('delete') .argument('', 'Rubric ID') .description('Delete a rubric') .action(async (id: string) => { try { const confirmed = await confirm(`Delete rubric ${id}? This cannot be undone.`); if (!confirmed) { console.log('Cancelled.'); return; } const client = new IndemnClient(); const sdk = new RubricsSDK(client); const spinner = ora('Deleting rubric...').start(); await sdk.deleteRubric(id); spinner.stop(); printSuccess(`Rubric ${id} deleted.`); } catch (err) { printError(err); process.exit(1); } }); rubric .command('versions') .argument('', 'Rubric ID') .description('List rubric versions') .option('--json', 'Output as JSON') .action(async (id: string, options) => { try { const client = new IndemnClient(); const sdk = new RubricsSDK(client); const spinner = ora('Fetching versions...').start(); const versions = await sdk.listVersions(id); spinner.stop(); if (options.json) { printJson(versions); return; } if (versions.length === 0) { console.log('No versions found.'); return; } const rows = versions.map((v) => [ String(v.version || ''), String(v.rules?.length || 0), v.updatedAt ? new Date(v.updatedAt).toLocaleDateString() : '', ]); console.log(formatTable(['Version', 'Rules', 'Updated'], rows)); } catch (err) { printError(err); process.exit(1); } }); // Test set commands — registered on program, not nested under eval const testset = program.command('testset').description('Manage evaluation test sets'); testset .command('list') .description('List test sets') .option('--agent-id ', 'Filter by agent ID') .option('--json', 'Output as JSON') .option('--limit ', 'Limit number of results') .option('--page ', 'Page number (use with --limit)') .action(async (options) => { try { const client = new IndemnClient(); const sdk = new TestSetsSDK(client); const spinner = ora('Fetching test sets...').start(); let sets: any[]; try { sets = await sdk.listTestSets(options.agentId); } catch (err: any) { spinner.stop(); if (!options.agentId && err?.message?.includes('500')) { printError('The server returned an error listing all test sets. Try filtering by agent: indemn testset list --agent-id '); process.exit(1); } throw err; } spinner.stop(); if (options.page && options.limit) { const page = parseInt(options.page, 10); const limit = parseInt(options.limit, 10); sets = sets.slice((page - 1) * limit, page * limit); } else if (options.limit) { sets = sets.slice(0, parseInt(options.limit, 10)); } if (options.json) { printJson(sets); return; } if (sets.length === 0) { console.log('No test sets found.'); return; } const rows = sets.map((s) => [ s._id || s.test_set_id || '', s.name, String(s.version || 1), String(s.items?.length || 0), s.updatedAt ? new Date(s.updatedAt).toLocaleDateString() : '', ]); console.log(formatTable(['ID', 'Name', 'Version', 'Items', 'Updated'], rows)); } catch (err) { printError(err); process.exit(1); } }); testset .command('get') .argument('', 'Test set ID') .description('Get test set details') .option('--json', 'Output as JSON') .option('--revision ', 'Get a specific version/revision') .action(async (id: string, options) => { try { const client = new IndemnClient(); const sdk = new TestSetsSDK(client); const spinner = ora('Fetching test set...').start(); const ts = options.revision ? await sdk.getVersion(id, parseInt(options.revision, 10)) : await sdk.getTestSet(id); spinner.stop(); if (options.json) { printJson(ts); return; } console.log(`ID: ${ts._id || ts.test_set_id}`); console.log(`Name: ${ts.name}`); console.log(`Description: ${ts.description || '(none)'}`); console.log(`Version: ${ts.version || 1}`); console.log(`Items: ${ts.items?.length || 0}`); if (ts.items?.length) { console.log(); for (const item of ts.items) { console.log(` [${item.type}] ${item.name}: ${item.inputs.message || item.inputs.initial_message || ''}`); } } } catch (err) { printError(err); process.exit(1); } }); testset .command('create') .description('Create a test set') .option('--file ', 'Path to JSON file with test set definition') .option('--name ', 'Test set name') .option('--agent-id ', 'Agent ID') .action(async (options) => { try { let input: Record; if (options.file) { const filePath = options.file as string; if (!fs.existsSync(filePath)) { printError(`File not found: ${filePath}`); process.exit(1); } const content = fs.readFileSync(filePath, 'utf-8'); try { input = JSON.parse(content); } catch { printError('File must contain valid JSON.'); process.exit(1); } // Merge CLI options into file content (CLI flags override file values) if (options.name) input.name = options.name; if (options.agentId) input.agent_id = options.agentId; } else if (options.name) { input = { name: options.name, items: [] }; if (options.agentId) input.agent_id = options.agentId; } else { printError('Provide --file with test set JSON or --name for an empty test set.'); process.exit(1); } const client = new IndemnClient(); const sdk = new TestSetsSDK(client); const spinner = ora('Creating test set...').start(); const testSetResult = await sdk.createTestSet(input as any); spinner.stop(); printSuccess(`Test set created: ${testSetResult._id || testSetResult.test_set_id}`); console.log(` Name: ${testSetResult.name}`); console.log(` Items: ${testSetResult.items?.length || 0}`); } catch (err) { printError(err); process.exit(1); } }); testset .command('update') .argument('', 'Test set ID') .description('Update a test set') .option('--file ', 'Path to JSON file with test set updates') .option('--name ', 'New test set name') .action(async (id: string, options) => { try { let input: Record; if (options.file) { const filePath = options.file as string; if (!fs.existsSync(filePath)) { printError(`File not found: ${filePath}`); process.exit(1); } const content = fs.readFileSync(filePath, 'utf-8'); try { input = JSON.parse(content); } catch { printError('File must contain valid JSON.'); process.exit(1); } } else if (options.name) { input = { name: options.name }; } else { printError('Provide --file with test set JSON or --name.'); process.exit(1); } const client = new IndemnClient(); const sdk = new TestSetsSDK(client); const spinner = ora('Updating test set...').start(); await sdk.updateTestSet(id, input as any); spinner.stop(); printSuccess(`Test set ${id} updated.`); } catch (err) { printError(err); process.exit(1); } }); testset .command('delete') .argument('', 'Test set ID') .description('Delete a test set') .action(async (id: string) => { try { const confirmed = await confirm(`Delete test set ${id}? This cannot be undone.`); if (!confirmed) { console.log('Cancelled.'); return; } const client = new IndemnClient(); const sdk = new TestSetsSDK(client); const spinner = ora('Deleting test set...').start(); await sdk.deleteTestSet(id); spinner.stop(); printSuccess(`Test set ${id} deleted.`); } catch (err) { printError(err); process.exit(1); } }); testset .command('versions') .argument('', 'Test set ID') .description('List test set versions') .option('--json', 'Output as JSON') .action(async (id: string, options) => { try { const client = new IndemnClient(); const sdk = new TestSetsSDK(client); const spinner = ora('Fetching versions...').start(); const versions = await sdk.listVersions(id); spinner.stop(); if (options.json) { printJson(versions); return; } if (versions.length === 0) { console.log('No versions found.'); return; } const rows = versions.map((v) => [ String(v.version || ''), String(v.items?.length || 0), v.updatedAt ? new Date(v.updatedAt).toLocaleDateString() : '', ]); console.log(formatTable(['Version', 'Items', 'Updated'], rows)); } catch (err) { printError(err); process.exit(1); } }); }