import { Command } from "commander"; import chalk from "chalk"; import ora from "ora"; import { apiRequest } from "../lib/api-client.js"; import type { TestCaseData, SandboxRunSummary } from "@skills-hub-ai/shared"; interface TestResult { label: string; input: string; expected: string | null; actual: string | null; passed: boolean; durationMs: number | null; error: string | null; } async function pollSandboxRun( slug: string, runId: string, timeoutMs = 60_000, ): Promise { const start = Date.now(); let delay = 1000; while (Date.now() - start < timeoutMs) { const run = await apiRequest( `/api/v1/skills/${encodeURIComponent(slug)}/sandbox/${encodeURIComponent(runId)}`, ); if ( run.status === "COMPLETED" || run.status === "FAILED" || run.status === "TIMEOUT" ) { return run; } await new Promise((resolve) => setTimeout(resolve, delay)); delay = Math.min(delay * 1.5, 8000); } throw new Error( `Sandbox run timed out after ${Math.round(timeoutMs / 1000)}s`, ); } export const testCommand = new Command("test") .description("Run test cases for a skill against the sandbox") .argument("", "Skill slug to test") .option("--timeout ", "Timeout per test case in ms", "60000") .action(async (slug: string, options) => { const spinner = ora(`Fetching test cases for ${slug}...`).start(); const timeout = Number(options.timeout) || 60_000; try { const testCases = await apiRequest( `/api/v1/skills/${encodeURIComponent(slug)}/test-cases`, ); if (!testCases || testCases.length === 0) { spinner.info(`No test cases found for ${chalk.bold(slug)}`); console.log( ` Create test cases at ${chalk.cyan(`https://skills-hub.ai/skills/${slug}`)}`, ); return; } spinner.succeed(`Found ${chalk.bold(testCases.length)} test case(s)`); console.log(); const results: TestResult[] = []; for (let i = 0; i < testCases.length; i++) { const tc = testCases[i]; const testSpinner = ora( ` [${i + 1}/${testCases.length}] ${tc.label}...`, ).start(); try { // Run in sandbox const run = await apiRequest<{ id: string }>( `/api/v1/skills/${encodeURIComponent(slug)}/sandbox`, { method: "POST", body: JSON.stringify({ input: tc.input, testCaseId: tc.id }), }, ); // Poll for completion const result = await pollSandboxRun(slug, run.id, timeout); const passed = result.status === "COMPLETED" && (!tc.expectedOutput || (result.output?.trim() ?? "").includes(tc.expectedOutput.trim())); results.push({ label: tc.label, input: tc.input, expected: tc.expectedOutput, actual: result.output ?? null, passed, durationMs: result.durationMs ?? null, error: result.errorMessage ?? null, }); if (passed) { testSpinner.succeed( ` ${chalk.green("PASS")} ${tc.label}${result.durationMs ? chalk.dim(` (${result.durationMs}ms)`) : ""}`, ); } else { testSpinner.fail( ` ${chalk.red("FAIL")} ${tc.label}${result.errorMessage ? `: ${result.errorMessage}` : ""}`, ); } } catch (err) { results.push({ label: tc.label, input: tc.input, expected: tc.expectedOutput, actual: null, passed: false, durationMs: null, error: err instanceof Error ? err.message : "Unknown error", }); testSpinner.fail( ` ${chalk.red("ERROR")} ${tc.label}: ${err instanceof Error ? err.message : "Unknown"}`, ); } } // Summary console.log(); const passed = results.filter((r) => r.passed).length; const failed = results.length - passed; if (failed === 0) { console.log(chalk.green(` All ${passed} test(s) passed`)); } else { console.log( chalk.red(` ${failed} of ${results.length} test(s) failed`), ); process.exitCode = 1; } } catch (err) { spinner.fail( chalk.red(err instanceof Error ? err.message : "Test run failed"), ); process.exit(1); } });