{"version":3,"sources":["/Users/Raffael.Prem/WebstormProjects/agentest/dist/index.cjs","../src/runner/reporters/console.ts","../src/runner/reporters/json.ts","../src/runner/reporters/githubActions.ts"],"names":[],"mappings":"AAAA;AACE;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACF,wDAA6B;AAC7B;AACE;AACF,wDAA6B;AAC7B;AACA;AC9BA,IAAM,KAAA,EAAO,uBAAA;AACb,IAAM,KAAA,EAAO,uBAAA;AACb,IAAM,KAAA,EAAO,SAAA;AACb,IAAM,IAAA,EAAM,SAAA;AACZ,IAAM,MAAA,EAAQ,SAAA;AACd,IAAM,IAAA,EAAM,UAAA;AACZ,IAAM,MAAA,EAAQ,UAAA;AACd,IAAM,OAAA,EAAS,UAAA;AACf,IAAM,KAAA,EAAO,UAAA;AAEb,IAAM,eAAA,EAAiB,CAAC,QAAA,EAAK,QAAA,EAAK,QAAA,EAAK,QAAA,EAAK,QAAA,EAAK,QAAA,EAAK,QAAA,EAAK,QAAA,EAAK,QAAA,EAAK,QAAG,CAAA;AAEjE,IAAM,gBAAA,YAAN,MAA0C;AAAA,iBACvC,gBAAA,EAAyD,KAAA;AAAA,kBACzD,aAAA,EAAe,EAAA;AAAA,kBACf,cAAA,EAAgB,GAAA;AAAA,kBAChB,MAAA,mBAAQ,OAAA,CAAQ,MAAA,CAAO,KAAA,UAAS,QAAA;AAAA,EAChC;AAAA,EACA;AAAA,EAER,WAAA,CAAY,UAAA,EAAqC,OAAA,EAAmB;AAClE,IAAA,IAAA,CAAK,WAAA,mBAAa,UAAA,UAAc,CAAC,GAAA;AACjC,IAAA,IAAA,CAAK,QAAA,mBAAU,OAAA,UAAW,OAAA;AAAA,EAC5B;AAAA,EAEA,UAAA,CAAW,SAAA,EAAuB,MAAA,EAAyB;AACzD,IAAA,OAAA,CAAQ,GAAA,CAAI,CAAA;AACZ,IAAA,GAAA,CAAI,OAAA,GAAU,MAAA,CAAO,OAAA,EAAS,CAAA,EAAG;AAC/B,MAAA,OAAA,CAAQ,GAAA;AAAA,QACN,CAAA,EAAA;AACF,MAAA;AACA,MAAA;AACF,IAAA;AACE,MAAA;AACF,IAAA;AACA,IAAA;AACF,EAAA;AAEA,EAAA;AACE,IAAA;AACE,MAAA;AACA,MAAA;AACA,MAAA;AACA,MAAA;AACF,IAAA;AAEA,IAAA;AACA,IAAA;AACK,IAAA;AAED,IAAA;AACF,MAAA;AACF,IAAA;AAEE,MAAA;AACF,IAAA;AACF,EAAA;AAEA,EAAA;AACO,IAAA;AAEL,IAAA;AACA,IAAA;AAEA,IAAA;AACK,IAAA;AACL,IAAA;AACA,IAAA;AACF,EAAA;AAEA,EAAA;AACO,IAAA;AACL,IAAA;AACA,IAAA;AACK,MAAA;AACL,IAAA;AAGA,IAAA;AACI,IAAA;AACF,MAAA;AACA,MAAA;AACE,QAAA;AACF,MAAA;AACF,IAAA;AACA,IAAA;AACF,EAAA;AAEA,EAAA;AACO,IAAA;AAEL,IAAA;AAEA,IAAA;AACE,MAAA;AACA,MAAA;AACA,MAAA;AACF,IAAA;AAGK,IAAA;AACL,IAAA;AACA,IAAA;AACF,EAAA;AAEA,EAAA;AACO,IAAA;AAEL,IAAA;AACA,IAAA;AAEA,IAAA;AACE,MAAA;AACA,MAAA;AACA,MAAA;AACE,QAAA;AACF,MAAA;AACF,IAAA;AAEA,IAAA;AACA,IAAA;AACA,IAAA;AACA,IAAA;AACF,EAAA;AAEQ,EAAA;AAEN,IAAA;AACE,MAAA;AACA,MAAA;AAEA,MAAA;AACE,QAAA;AACA,QAAA;AACF,MAAA;AAEA,MAAA;AACA,MAAA;AAEA,MAAA;AAGA,MAAA;AACE,QAAA;AAAM,UAAA;AAIN,QAAA;AACF,MAAA;AAGA,MAAA;AACE,QAAA;AACE,UAAA;AACF,QAAA;AACE,UAAA;AACA,UAAA;AACE,YAAA;AAAiE,UAAA;AAEnE,UAAA;AACE,YAAA;AAA6D,UAAA;AAE/D,UAAA;AACF,QAAA;AACF,MAAA;AAGA,MAAA;AACA,MAAA;AACE,QAAA;AACA,QAAA;AACE,UAAA;AACF,QAAA;AACE,UAAA;AACE,YAAA;AACA,YAAA;AACE,cAAA;AAA2D,YAAA;AAE7D,YAAA;AACE,cAAA;AAAuD,YAAA;AAEzD,YAAA;AACE,cAAA;AAA+D,YAAA;AAEjE,YAAA;AAAiD,UAAA;AAErD,QAAA;AACF,MAAA;AAGA,MAAA;AACE,QAAA;AACA,QAAA;AACE,UAAA;AACA,UAAA;AAEA,UAAA;AACF,QAAA;AACF,MAAA;AAEA,MAAA;AAEA,MAAA;AACE,QAAA;AACF,MAAA;AACF,IAAA;AAGI,IAAA;AACF,MAAA;AACA,MAAA;AACE,QAAA;AAQA,QAAA;AAAQ,UAAA;AAER,QAAA;AACA,QAAA;AACF,MAAA;AACF,IAAA;AAGK,IAAA;AACH,MAAA;AACA,MAAA;AACE,QAAA;AACA,QAAA;AACE,UAAA;AAAQ,YAAA;AACgG,UAAA;AAE1G,QAAA;AACF,MAAA;AACF,IAAA;AACF,EAAA;AAEQ,EAAA;AAEN,IAAA;AAEA,IAAA;AACE,MAAA;AACF,IAAA;AAGA,IAAA;AACA,IAAA;AACE,MAAA;AACE,QAAA;AACF,MAAA;AACF,IAAA;AAEI,IAAA;AAEJ,IAAA;AACA,IAAA;AAEA,IAAA;AACE,MAAA;AACA,MAAA;AAEA,MAAA;AACE,QAAA;AACA,QAAA;AACA,QAAA;AACA,QAAA;AACD,MAAA;AAED,MAAA;AACF,IAAA;AACA,IAAA;AACF,EAAA;AAEQ,EAAA;AACF,IAAA;AACC,IAAA;AACA,IAAA;AACH,MAAA;AACA,MAAA;AACG,IAAA;AACP,EAAA;AAEQ,EAAA;AACN,IAAA;AACA,IAAA;AACF,EAAA;AAEQ,EAAA;AACF,IAAA;AACF,MAAA;AACA,MAAA;AACA,MAAA;AACF,IAAA;AACF,EAAA;AAEQ,EAAA;AACN,IAAA;AACA,IAAA;AACE,MAAA;AAEA,MAAA;AACE,QAAA;AACA,QAAA;AACF,MAAA;AAEA,MAAA;AACA,MAAA;AACF,IAAA;AACA,IAAA;AACF,EAAA;AACF;ADpCS;AACA;AEvSA;AACF;AAcM;AACH,EAAA;AACA,EAAA;AAER,EAAA;AACO,IAAA;AACA,IAAA;AACP,EAAA;AAEM,EAAA;AACJ,IAAA;AACA,IAAA;AACF,EAAA;AAEM,EAAA;AACJ,IAAA;AACA,IAAA;AACF,EAAA;AAEc,EAAA;AACZ,IAAA;AACA,IAAA;AACI,IAAA;AACF,MAAA;AACE,QAAA;AACF,MAAA;AACF,IAAA;AACA,IAAA;AACA,IAAA;AACK,IAAA;AACH,MAAA;AACE,QAAA;AACF,MAAA;AACF,IAAA;AACA,IAAA;AACA,IAAA;AACF,EAAA;AAEQ,EAAA;AACN,IAAA;AACE,MAAA;AACE,QAAA;AACA,QAAA;AACA,QAAA;AACA,QAAA;AACF,MAAA;AACA,MAAA;AACF,IAAA;AACF,EAAA;AAEQ,EAAA;AACN,IAAA;AACA,IAAA;AACE,MAAA;AACE,QAAA;AACA,QAAA;AACA,QAAA;AACF,MAAA;AACF,IAAA;AAEA,IAAA;AACE,MAAA;AACA,MAAA;AACA,MAAA;AACE,QAAA;AACA,QAAA;AACF,MAAA;AACA,MAAA;AACE,QAAA;AACA,QAAA;AAAqC,UAAA;AACzB,UAAA;AAEZ,QAAA;AACA,MAAA;AACJ,IAAA;AACF,EAAA;AAEQ,EAAA;AACN,IAAA;AACE,MAAA;AACA,MAAA;AACA,MAAA;AACE,QAAA;AACA,QAAA;AAEA,QAAA;AAAO,UAAA;AACgB,UAAA;AACA,UAAA;AACS,YAAA;AACf,YAAA;AACE,YAAA;AACC,YAAA;AACH,UAAA;AACb,UAAA;AAEE,YAAA;AAC2D,cAAA;AACzC,cAAA;AACF,YAAA;AACZ,YAAA;AACyB,UAAA;AAE7B,UAAA;AACsB,UAAA;AAE0C,YAAA;AACnD,YAAA;AACR,UAAA;AAEX,QAAA;AACD,MAAA;AACD,MAAA;AACF,IAAA;AACF,EAAA;AACF;AFgRS;AACA;AGlZA;AAYA;AACA,EAAA;AACT;AAEM;AACJ,EAAA;AACM,EAAA;AACN,EAAA;AACK,EAAA;AACP;AAEa;AACH,EAAA;AACA,EAAA;AAER,EAAA;AACO,IAAA;AACA,IAAA;AACP,EAAA;AAEA,EAAA;AACO,IAAA;AACP,EAAA;AAEA,EAAA;AACO,IAAA;AACA,IAAA;AACP,EAAA;AAEA,EAAA;AACE,IAAA;AACE,MAAA;AACF,IAAA;AACF,EAAA;AAEA,EAAA;AACE,IAAA;AACE,MAAA;AACE,QAAA;AAAK,UAAA;AACH,UAAA;AACmB,UAAA;AAErB,QAAA;AACF,MAAA;AACF,IAAA;AACK,IAAA;AACP,EAAA;AAAA;AAIQ,EAAA;AACN,IAAA;AAEK,IAAA;AACH,MAAA;AACE,QAAA;AACA,QAAA;AACA,QAAA;AACF,MAAA;AACF,IAAA;AAEA,IAAA;AACE,MAAA;AACA,MAAA;AACE,QAAA;AACA,QAAA;AACA,QAAA;AACF,MAAA;AACF,IAAA;AACF,EAAA;AAEQ,EAAA;AACN,IAAA;AACE,MAAA;AACA,MAAA;AACE,QAAA;AAAK,UAAA;AACH,UAAA;AACgC,UAAA;AAElC,QAAA;AACF,MAAA;AACF,IAAA;AACF,EAAA;AAEQ,EAAA;AACN,IAAA;AACA,IAAA;AACA,IAAA;AACF,EAAA;AAEQ,EAAA;AACF,IAAA;AACA,IAAA;AACJ,IAAA;AACF,EAAA;AAAA;AAIQ,EAAA;AACD,IAAA;AACD,IAAA;AACF,MAAA;AACF,IAAA;AACE,MAAA;AACF,IAAA;AACF,EAAA;AAEQ,EAAA;AACN,IAAA;AACA,IAAA;AACE,MAAA;AACA,MAAA;AACA,MAAA;AACA,MAAA;AACF,IAAA;AAGA,IAAA;AACA,IAAA;AACA,IAAA;AACA,IAAA;AAEA,IAAA;AACE,MAAA;AACA,MAAA;AACA,MAAA;AACA,MAAA;AAGA,MAAA;AACA,MAAA;AACE,QAAA;AACA,QAAA;AACD,MAAA;AAED,MAAA;AACE,QAAA;AACF,MAAA;AACF,IAAA;AAGA,IAAA;AACI,IAAA;AACF,MAAA;AACA,MAAA;AACA,MAAA;AACA,MAAA;AACE,QAAA;AACA,QAAA;AAAM,UAAA;AAEN,QAAA;AACF,MAAA;AACF,IAAA;AAGA,IAAA;AAAsD,MAAA;AAEtD,IAAA;AACI,IAAA;AACF,MAAA;AACA,MAAA;AACA,MAAA;AACA,MAAA;AACE,QAAA;AAAM,UAAA;AAEN,QAAA;AACF,MAAA;AACF,IAAA;AAEA,IAAA;AACA,IAAA;AACF,EAAA;AAEQ,EAAA;AACN,IAAA;AACA,IAAA;AAGA,IAAA;AACA,IAAA;AACE,MAAA;AACE,QAAA;AACE,UAAA;AACF,QAAA;AACF,MAAA;AACF,IAAA;AACA,IAAA;AAGA,IAAA;AACA,IAAA;AACA,IAAA;AAEA,IAAA;AACE,MAAA;AACA,MAAA;AAGA,MAAA;AACA,MAAA;AACA,MAAA;AACE,QAAA;AACA,QAAA;AACE,UAAA;AACA,UAAA;AACF,QAAA;AACF,MAAA;AAEA,MAAA;AACE,QAAA;AACA,QAAA;AACD,MAAA;AAED,MAAA;AACE,QAAA;AACF,MAAA;AACF,IAAA;AAGA,IAAA;AACA,IAAA;AACE,MAAA;AACA,MAAA;AACA,MAAA;AACA,MAAA;AAEA,MAAA;AACE,QAAA;AACA,QAAA;AACA,QAAA;AACA,QAAA;AACF,MAAA;AACA,MAAA;AACF,IAAA;AAEA,IAAA;AACF,EAAA;AAAA;AAIQ,EAAA;AACN,IAAA;AACA,IAAA;AACE,MAAA;AACE,QAAA;AACF,MAAA;AACF,IAAA;AACA,IAAA;AACF,EAAA;AAEQ,EAAA;AACN,IAAA;AACF,EAAA;AAEQ,EAAA;AACN,IAAA;AAEA,IAAA;AACI,IAAA;AACF,MAAA;AACF,IAAA;AAEA,IAAA;AACI,IAAA;AACF,MAAA;AACF,IAAA;AAEA,IAAA;AAA6E,MAAA;AAE7E,IAAA;AACI,IAAA;AACF,MAAA;AACF,IAAA;AAEI,IAAA;AACF,MAAA;AACG,QAAA;AACH,MAAA;AACA,MAAA;AACE,QAAA;AACF,MAAA;AACF,IAAA;AAEA,IAAA;AACI,IAAA;AACF,MAAA;AACF,IAAA;AAEA,IAAA;AACF,EAAA;AACF;AHqVS;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA","file":"/Users/Raffael.Prem/WebstormProjects/agentest/dist/index.cjs","sourcesContent":[null,"import type { Scenario } from '../../scenario/types.js'\nimport type {\n  ScenarioResult,\n  RunResult,\n  ComparisonScenarioResult,\n  ComparisonRunResult,\n} from '../runner.js'\nimport type { Reporter, ProgressEvent } from './types.js'\nimport type { TurnRecord } from '../../simulator/simulator.js'\nimport type { ConversationEvaluation } from '../../evaluator/evaluator.js'\nimport {\n  computeTurnAverages,\n  computeMetricAverages,\n  computeThresholdViolations,\n} from '../../evaluator/scoring.js'\n\nconst PASS = '\\x1b[32m\\u2713\\x1b[0m'\nconst FAIL = '\\x1b[31m\\u2717\\x1b[0m'\nconst BOLD = '\\x1b[1m'\nconst DIM = '\\x1b[2m'\nconst RESET = '\\x1b[0m'\nconst RED = '\\x1b[31m'\nconst GREEN = '\\x1b[32m'\nconst YELLOW = '\\x1b[33m'\nconst CYAN = '\\x1b[36m'\n\nconst SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']\n\nexport class ConsoleReporter implements Reporter {\n  private spinnerInterval: ReturnType<typeof setInterval> | null = null\n  private spinnerFrame = 0\n  private currentStatus = ''\n  private isTTY = process.stderr.isTTY ?? false\n  private thresholds: Record<string, number>\n  private verbose: boolean\n\n  constructor(thresholds?: Record<string, number>, verbose?: boolean) {\n    this.thresholds = thresholds ?? {}\n    this.verbose = verbose ?? false\n  }\n\n  onRunStart(scenarios: Scenario[], agents?: string[]): void {\n    console.log()\n    if (agents && agents.length > 1) {\n      console.log(\n        `${BOLD}Agentest${RESET} ${DIM}comparing ${agents.length} agents across ${scenarios.length} scenario(s)${RESET}`,\n      )\n      console.log(`${DIM}Agents: ${agents.join(', ')}${RESET}`)\n    } else {\n      console.log(`${BOLD}Agentest${RESET} ${DIM}running ${scenarios.length} scenario(s)${RESET}`)\n    }\n    console.log()\n  }\n\n  onProgress(event: ProgressEvent): void {\n    const phaseLabels: Record<ProgressEvent['phase'], string> = {\n      simulating: 'simulating',\n      evaluating: 'evaluating',\n      trajectories: 'checking trajectories',\n      deduplicating: 'deduplicating errors',\n    }\n\n    const label = phaseLabels[event.phase]\n    const detail = event.detail ? ` ${DIM}${event.detail}${RESET}` : ''\n    this.currentStatus = `${CYAN}${event.scenario}${RESET} → ${label}${detail}`\n\n    if (this.isTTY) {\n      this.startSpinner()\n    } else {\n      // Non-TTY: print one line per phase change\n      console.log(`  ${DIM}▸${RESET} ${this.currentStatus}`)\n    }\n  }\n\n  onScenarioEnd(result: ScenarioResult): void {\n    this.stopSpinner()\n\n    const icon = result.passed ? PASS : FAIL\n    const name = result.scenario.name\n\n    console.log(`${icon} ${name}`)\n    this.printScenarioDetails(result)\n    console.log(`${DIM}${'─'.repeat(40)}${RESET}`)\n    console.log()\n  }\n\n  onRunEnd(result: RunResult): void {\n    this.stopSpinner()\n    const color = result.passed ? GREEN : RED\n    console.log(\n      `${BOLD}${color}${result.passedScenarios}/${result.totalScenarios} scenarios passed${RESET}`,\n    )\n    // Flaky scenarios that only passed on retry stay visible — a rising count\n    // here means the suite is degrading even while it stays green.\n    const flaky = result.scenarioResults.filter((r) => r.retried)\n    if (flaky.length > 0) {\n      console.log(`${YELLOW}${flaky.length} passed only on retry:${RESET}`)\n      for (const r of flaky) {\n        console.log(`${YELLOW}  ↻ ${r.scenario.name} (retry ${r.retried})${RESET}`)\n      }\n    }\n    console.log()\n  }\n\n  onComparisonScenarioEnd(comparison: ComparisonScenarioResult): void {\n    this.stopSpinner()\n\n    console.log(`${BOLD}${comparison.scenario.name}${RESET}`)\n\n    for (const { agentName, scenarioResult } of comparison.agentResults) {\n      const icon = scenarioResult.passed ? PASS : FAIL\n      console.log(`  ${icon} ${CYAN}${agentName}${RESET}`)\n      this.printScenarioDetails(scenarioResult, '    ')\n    }\n\n    // Print metric comparison table\n    this.printMetricComparison(comparison)\n    console.log(`${DIM}${'─'.repeat(40)}${RESET}`)\n    console.log()\n  }\n\n  onComparisonRunEnd(result: ComparisonRunResult): void {\n    this.stopSpinner()\n\n    console.log(`${BOLD}Comparison Summary${RESET}`)\n    console.log()\n\n    for (const agentName of result.agents) {\n      const agentResult = result.perAgent.get(agentName)!\n      const color = agentResult.passed ? GREEN : RED\n      console.log(\n        `  ${color}${agentName}${RESET}: ${agentResult.passedScenarios}/${agentResult.totalScenarios} scenarios passed`,\n      )\n    }\n\n    console.log()\n    const overallColor = result.passed ? GREEN : RED\n    console.log(`${BOLD}${overallColor}Overall: ${result.passed ? 'PASS' : 'FAIL'}${RESET}`)\n    console.log()\n  }\n\n  private printScenarioDetails(result: ScenarioResult, indent = '  '): void {\n    // Conversation summaries\n    for (const conv of result.simulation.conversations) {\n      const turnCount = conv.turns.length\n      const prefix = conv.error ? `${indent}${FAIL}` : `${indent}${PASS}`\n\n      if (conv.error) {\n        console.log(`${prefix} ${conv.conversationId} ${RED}(error: ${conv.error})${RESET}`)\n        continue\n      }\n\n      const evaluation = result.evaluations.get(conv.conversationId)\n      const trajectory = result.trajectoryResults.get(conv.conversationId)\n\n      const parts: string[] = [`${turnCount} turns`]\n\n      // Goal completion\n      if (evaluation?.goalCompletion) {\n        parts.push(\n          evaluation.goalCompletion.value === 1\n            ? `${GREEN}goal met${RESET}`\n            : `${RED}goal not met${RESET}`,\n        )\n      }\n\n      // Trajectory\n      if (trajectory) {\n        if (trajectory.matched) {\n          parts.push(`${GREEN}trajectory matched${RESET}`)\n        } else {\n          const details: string[] = ['trajectory failed']\n          if (trajectory.forbiddenCalls.length > 0) {\n            details.push(`forbidden: ${trajectory.forbiddenCalls.join(', ')}`)\n          }\n          if (trajectory.missingCalls.length > 0) {\n            details.push(`missing: ${trajectory.missingCalls.join(', ')}`)\n          }\n          parts.push(`${RED}${details.join(' — ')}${RESET}`)\n        }\n      }\n\n      // Per-turn trajectory\n      const perTurnResults = result.perTurnTrajectoryResults.get(conv.conversationId)\n      if (perTurnResults) {\n        const failed = perTurnResults.filter((r) => !r.result.matched)\n        if (failed.length === 0) {\n          parts.push(`${GREEN}per-turn trajectories matched${RESET}`)\n        } else {\n          for (const f of failed) {\n            const details: string[] = [`turn ${f.turnIndex}`]\n            if (f.result.missingCalls.length > 0) {\n              details.push(`missing: ${f.result.missingCalls.join(', ')}`)\n            }\n            if (f.result.extraCalls.length > 0) {\n              details.push(`extra: ${f.result.extraCalls.join(', ')}`)\n            }\n            if (f.result.forbiddenCalls.length > 0) {\n              details.push(`forbidden: ${f.result.forbiddenCalls.join(', ')}`)\n            }\n            parts.push(`${RED}${details.join(' — ')}${RESET}`)\n          }\n        }\n      }\n\n      // Average scores\n      if (evaluation) {\n        const avgScores = computeTurnAverages(evaluation.turnEvaluations)\n        for (const [metric, avg] of Object.entries(avgScores)) {\n          const threshold = this.thresholds[metric]\n          const color =\n            threshold != null && avg < threshold ? RED : avg >= 4 ? GREEN : avg >= 3 ? YELLOW : RED\n          parts.push(`${metric}: ${color}${avg.toFixed(1)}${RESET}`)\n        }\n      }\n\n      console.log(`${prefix} ${conv.conversationId} ${DIM}(${parts.join(', ')})${RESET}`)\n\n      if (this.verbose) {\n        this.printTranscript(conv.turns)\n      }\n    }\n\n    // Unique errors\n    if (result.errors.length > 0) {\n      console.log(`${indent}${DIM}Unique errors:${RESET}`)\n      for (const err of result.errors) {\n        const sevColor =\n          err.severity === 'critical'\n            ? RED\n            : err.severity === 'high'\n              ? RED\n              : err.severity === 'medium'\n                ? YELLOW\n                : DIM\n        console.log(\n          `${indent}  ${sevColor}[${err.severity}]${RESET} ${err.label} ${DIM}(${err.occurrences.length} occurrence(s))${RESET}`,\n        )\n        console.log(`${indent}    ${DIM}${err.explanation}${RESET}`)\n      }\n    }\n\n    // Threshold failures\n    if (!result.passed && Object.keys(this.thresholds).length > 0) {\n      const failures = computeThresholdViolations(result.evaluations.values(), this.thresholds)\n      if (failures.length > 0) {\n        console.log(`${indent}${RED}Threshold failures:${RESET}`)\n        for (const f of failures) {\n          console.log(\n            `${indent}  ${RED}${f.metric}${RESET}: avg ${RED}${f.avg.toFixed(1)}${RESET} < threshold ${f.threshold}`,\n          )\n        }\n      }\n    }\n  }\n\n  private printMetricComparison(comparison: ComparisonScenarioResult): void {\n    // Collect average scores per agent\n    const agentScores = new Map<string, Record<string, number>>()\n\n    for (const { agentName, scenarioResult } of comparison.agentResults) {\n      agentScores.set(agentName, computeMetricAverages(scenarioResult.evaluations.values()))\n    }\n\n    // Get all metric names\n    const metricNames = new Set<string>()\n    for (const scores of agentScores.values()) {\n      for (const name of Object.keys(scores)) {\n        metricNames.add(name)\n      }\n    }\n\n    if (metricNames.size === 0) return\n\n    console.log(`  ${DIM}── comparison ──${RESET}`)\n    const agentNames = comparison.agentResults.map((r) => r.agentName)\n\n    for (const metric of metricNames) {\n      const scores = agentNames.map((name) => agentScores.get(name)?.[metric])\n      const best = Math.max(...scores.filter((s): s is number => s != null))\n\n      const formatted = agentNames.map((name, i) => {\n        const score = scores[i]\n        if (score == null) return `${DIM}—${RESET}`\n        const color = score === best ? GREEN : score >= best - 0.5 ? YELLOW : RED\n        return `${name}: ${color}${score.toFixed(1)}${RESET}`\n      })\n\n      console.log(`  ${DIM}${metric}:${RESET} ${formatted.join(`${DIM} | ${RESET}`)}`)\n    }\n    console.log(`  ${DIM}── end comparison ──${RESET}`)\n  }\n\n  private startSpinner(): void {\n    if (this.spinnerInterval) return\n    this.renderSpinner()\n    this.spinnerInterval = setInterval(() => {\n      this.spinnerFrame = (this.spinnerFrame + 1) % SPINNER_FRAMES.length\n      this.renderSpinner()\n    }, 80)\n  }\n\n  private renderSpinner(): void {\n    const frame = `${CYAN}${SPINNER_FRAMES[this.spinnerFrame]}${RESET}`\n    process.stderr.write(`\\r\\x1b[K  ${frame} ${this.currentStatus}`)\n  }\n\n  private stopSpinner(): void {\n    if (this.spinnerInterval) {\n      clearInterval(this.spinnerInterval)\n      this.spinnerInterval = null\n      process.stderr.write('\\r\\x1b[K')\n    }\n  }\n\n  private printTranscript(turns: TurnRecord[]): void {\n    console.log(`    ${DIM}── transcript ──${RESET}`)\n    for (const turn of turns) {\n      console.log(`    ${BOLD}[User]${RESET}  ${turn.userMessage}`)\n\n      for (const tc of turn.toolCalls) {\n        console.log(`    ${YELLOW}[Tool]${RESET}  ${tc.name}(${JSON.stringify(tc.args)})`)\n        console.log(`           ${DIM}→ ${JSON.stringify(tc.result)}${RESET}`)\n      }\n\n      console.log(`    ${CYAN}[Agent]${RESET} ${turn.agentMessage}`)\n      console.log()\n    }\n    console.log(`    ${DIM}── end transcript ──${RESET}`)\n  }\n}\n","import { writeFile, mkdir } from 'node:fs/promises'\nimport path from 'node:path'\nimport type {\n  RunResult,\n  ScenarioResult,\n  ComparisonScenarioResult,\n  ComparisonRunResult,\n} from '../runner.js'\nimport type { Reporter } from './types.js'\n\nexport interface JsonReporterOptions {\n  outputDir?: string\n  filename?: string\n}\n\nexport class JsonReporter implements Reporter {\n  private outputDir: string\n  private filename: string\n\n  constructor(options: JsonReporterOptions = {}) {\n    this.outputDir = options.outputDir ?? '.agentest'\n    this.filename = options.filename ?? 'results.json'\n  }\n\n  async onRunEnd(result: RunResult): Promise<void> {\n    const output = this.serialize(result)\n    await this.writeOutput(output)\n  }\n\n  async onComparisonRunEnd(result: ComparisonRunResult): Promise<void> {\n    const output = this.serializeComparison(result)\n    await this.writeOutput(output)\n  }\n\n  private async writeOutput(output: object): Promise<void> {\n    const resolvedDir = path.resolve(this.outputDir)\n    const cwd = process.cwd()\n    if (resolvedDir !== cwd && !resolvedDir.startsWith(cwd + path.sep)) {\n      throw new Error(\n        `JSON reporter output directory \"${this.outputDir}\" resolves outside the working directory`,\n      )\n    }\n    await mkdir(resolvedDir, { recursive: true })\n    const filePath = path.resolve(resolvedDir, this.filename)\n    if (!filePath.startsWith(resolvedDir + path.sep)) {\n      throw new Error(\n        `JSON reporter filename \"${this.filename}\" resolves outside the output directory`,\n      )\n    }\n    await writeFile(filePath, JSON.stringify(output, null, 2))\n    console.log(`Results written to ${filePath}`)\n  }\n\n  private serialize(result: RunResult): object {\n    return {\n      summary: {\n        passed: result.passed,\n        totalScenarios: result.totalScenarios,\n        passedScenarios: result.passedScenarios,\n        failedScenarios: result.failedScenarios,\n      },\n      scenarios: result.scenarioResults.map((sr) => this.serializeScenario(sr)),\n    }\n  }\n\n  private serializeComparison(result: ComparisonRunResult): object {\n    const perAgent: Record<string, object> = {}\n    for (const [agentName, agentResult] of result.perAgent) {\n      perAgent[agentName] = {\n        passed: agentResult.passed,\n        passedScenarios: agentResult.passedScenarios,\n        failedScenarios: agentResult.failedScenarios,\n      }\n    }\n\n    return {\n      mode: 'comparison',\n      agents: result.agents,\n      summary: {\n        passed: result.passed,\n        perAgent,\n      },\n      scenarios: result.scenarioComparisons.map((sc) => ({\n        name: sc.scenario.name,\n        agents: sc.agentResults.map((ar) => ({\n          agent: ar.agentName,\n          ...this.serializeScenario(ar.scenarioResult),\n        })),\n      })),\n    }\n  }\n\n  private serializeScenario(sr: ScenarioResult): object {\n    return {\n      name: sr.scenario.name,\n      passed: sr.passed,\n      conversations: sr.simulation.conversations.map((conv) => {\n        const evaluation = sr.evaluations.get(conv.conversationId)\n        const trajectory = sr.trajectoryResults.get(conv.conversationId)\n\n        return {\n          conversationId: conv.conversationId,\n          error: conv.error ?? null,\n          turns: conv.turns.map((t) => ({\n            turnIndex: t.turnIndex,\n            userMessage: t.userMessage,\n            agentMessage: t.agentMessage,\n            toolCalls: t.toolCalls,\n          })),\n          evaluation: evaluation\n            ? {\n                turnEvaluations: evaluation.turnEvaluations.map((te) => ({\n                  turnIndex: te.turnIndex,\n                  metrics: te.metrics,\n                })),\n                goalCompletion: evaluation.goalCompletion,\n              }\n            : null,\n          trajectory: trajectory ?? null,\n          perTurnTrajectory:\n            sr.perTurnTrajectoryResults.get(conv.conversationId)?.map((r) => ({\n              turnIndex: r.turnIndex,\n              ...r.result,\n            })) ?? null,\n        }\n      }),\n      errors: sr.errors,\n    }\n  }\n}\n","import { appendFileSync } from 'node:fs'\nimport type {\n  ScenarioResult,\n  RunResult,\n  ComparisonScenarioResult,\n  ComparisonRunResult,\n} from '../runner.js'\nimport type { Reporter } from './types.js'\nimport type { UniqueError } from '../../evaluator/errorDetection.js'\nimport { computeMetricAverages, computeThresholdViolations } from '../../evaluator/scoring.js'\n\n/** Escape a string for safe use inside a Markdown table cell. */\nfunction escapeMarkdown(text: string): string {\n  return text.replace(/\\|/g, '\\\\|').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/`/g, '\\\\`')\n}\n\nconst SEVERITY_ICONS: Record<string, string> = {\n  critical: '🔴',\n  high: '🟠',\n  medium: '🟡',\n  low: '⚪',\n}\n\nexport class GitHubActionsReporter implements Reporter {\n  private summaryFile: string | undefined\n  private thresholds: Record<string, number>\n\n  constructor(thresholds?: Record<string, number>) {\n    this.summaryFile = process.env.GITHUB_STEP_SUMMARY\n    this.thresholds = thresholds ?? {}\n  }\n\n  onScenarioEnd(result: ScenarioResult): void {\n    this.emitAnnotationsForScenario(result)\n  }\n\n  onRunEnd(result: RunResult): void {\n    this.emitRunAnnotations(result)\n    this.writeSummary(this.buildStandardSummary(result))\n  }\n\n  onComparisonScenarioEnd(comparison: ComparisonScenarioResult): void {\n    for (const { agentName, scenarioResult } of comparison.agentResults) {\n      this.emitAnnotationsForScenario(scenarioResult, agentName)\n    }\n  }\n\n  onComparisonRunEnd(result: ComparisonRunResult): void {\n    for (const [agentName, agentResult] of result.perAgent) {\n      if (!agentResult.passed) {\n        this.annotate(\n          'error',\n          `Agent \"${agentName}\" failed`,\n          `${agentResult.failedScenarios}/${agentResult.totalScenarios} scenarios failed`,\n        )\n      }\n    }\n    this.writeSummary(this.buildComparisonSummary(result))\n  }\n\n  // --- Annotations ---\n\n  private emitAnnotationsForScenario(result: ScenarioResult, agentLabel?: string): void {\n    const prefix = agentLabel ? `[${agentLabel}] ` : ''\n\n    if (!result.passed) {\n      this.annotate(\n        'error',\n        `${prefix}Scenario failed: ${result.scenario.name}`,\n        this.getFailureReason(result),\n      )\n    }\n\n    for (const error of result.errors) {\n      const level = this.severityToLevel(error.severity)\n      this.annotate(\n        level,\n        `${prefix}${error.label}`,\n        `${error.explanation} (${error.occurrences.length} occurrence(s))`,\n      )\n    }\n  }\n\n  private emitRunAnnotations(result: RunResult): void {\n    for (const sr of result.scenarioResults) {\n      const violations = this.getThresholdViolations(sr)\n      for (const v of violations) {\n        this.annotate(\n          'warning',\n          `Threshold violation: ${v.metric}`,\n          `Average ${v.avg.toFixed(2)} < threshold ${v.threshold} in \"${sr.scenario.name}\"`,\n        )\n      }\n    }\n  }\n\n  private annotate(level: 'error' | 'warning' | 'notice', title: string, message: string): void {\n    const safeTitle = title.replace(/%/g, '%25').replace(/\\r/g, '%0D').replace(/\\n/g, '%0A')\n    const safeMessage = message.replace(/%/g, '%25').replace(/\\r/g, '%0D').replace(/\\n/g, '%0A')\n    console.log(`::${level} title=${safeTitle}::${safeMessage}`)\n  }\n\n  private severityToLevel(severity: string): 'error' | 'warning' | 'notice' {\n    if (severity === 'critical' || severity === 'high') return 'error'\n    if (severity === 'medium') return 'warning'\n    return 'notice'\n  }\n\n  // --- Step Summary ---\n\n  private writeSummary(markdown: string): void {\n    if (!this.summaryFile) return\n    try {\n      appendFileSync(this.summaryFile, markdown)\n    } catch {\n      console.warn('Failed to write to $GITHUB_STEP_SUMMARY')\n    }\n  }\n\n  private buildStandardSummary(result: RunResult): string {\n    const icon = result.passed ? '✅' : '❌'\n    const lines: string[] = [\n      `## Agentest Results`,\n      '',\n      `**${result.passedScenarios}/${result.totalScenarios} scenarios passed** ${icon}`,\n      '',\n    ]\n\n    // Scenario table\n    const metricNames = this.collectMetricNames(result.scenarioResults)\n    const headers = ['Scenario', 'Status', 'Conversations', ...metricNames]\n    lines.push(`| ${headers.join(' | ')} |`)\n    lines.push(`| ${headers.map(() => '---').join(' | ')} |`)\n\n    for (const sr of result.scenarioResults) {\n      const status = sr.passed ? '✅ Pass' : '❌ Fail'\n      const convCount = sr.simulation.conversations.length\n      const erroredCount = sr.simulation.conversations.filter((c) => c.error).length\n      const convStatus =\n        erroredCount > 0 ? `${convCount - erroredCount}/${convCount}` : `${convCount}/${convCount}`\n\n      const avgScores = computeMetricAverages(sr.evaluations.values())\n      const scoreCells = metricNames.map((m) => {\n        const val = avgScores[m]\n        return val != null ? val.toFixed(1) : '—'\n      })\n\n      lines.push(\n        `| ${escapeMarkdown(sr.scenario.name)} | ${status} | ${convStatus} | ${scoreCells.join(' | ')} |`,\n      )\n    }\n\n    // Errors\n    const allErrors = result.scenarioResults.flatMap((sr) => sr.errors)\n    if (allErrors.length > 0) {\n      lines.push('', '### Errors', '')\n      lines.push('| Severity | Error | Occurrences |')\n      lines.push('| --- | --- | --- |')\n      for (const err of allErrors) {\n        const icon = SEVERITY_ICONS[err.severity] ?? '⚪'\n        lines.push(\n          `| ${icon} ${err.severity} | ${escapeMarkdown(err.label)} | ${err.occurrences.length} |`,\n        )\n      }\n    }\n\n    // Threshold violations\n    const allViolations = result.scenarioResults.flatMap((sr) =>\n      this.getThresholdViolations(sr).map((v) => ({ ...v, scenario: sr.scenario.name })),\n    )\n    if (allViolations.length > 0) {\n      lines.push('', '### Threshold Violations', '')\n      lines.push('| Scenario | Metric | Average | Threshold |')\n      lines.push('| --- | --- | --- | --- |')\n      for (const v of allViolations) {\n        lines.push(\n          `| ${escapeMarkdown(v.scenario)} | ${escapeMarkdown(v.metric)} | ${v.avg.toFixed(2)} | ${v.threshold} |`,\n        )\n      }\n    }\n\n    lines.push('')\n    return lines.join('\\n')\n  }\n\n  private buildComparisonSummary(result: ComparisonRunResult): string {\n    const icon = result.passed ? '✅' : '❌'\n    const lines: string[] = [`## Agentest Comparison Results ${icon}`, '']\n\n    // Collect all metric names across all agents\n    const metricNames = new Set<string>()\n    for (const agentResult of result.perAgent.values()) {\n      for (const sr of agentResult.scenarioResults) {\n        for (const name of Object.keys(computeMetricAverages(sr.evaluations.values()))) {\n          metricNames.add(name)\n        }\n      }\n    }\n    const metrics = [...metricNames]\n\n    // Agent summary table\n    const headers = ['Agent', 'Scenarios Passed', ...metrics]\n    lines.push(`| ${headers.join(' | ')} |`)\n    lines.push(`| ${headers.map(() => '---').join(' | ')} |`)\n\n    for (const agentName of result.agents) {\n      const agentResult = result.perAgent.get(agentName)!\n      const statusIcon = agentResult.passed ? '✅' : '❌'\n\n      // Compute global averages across all scenarios for this agent\n      const globalSums: Record<string, number> = {}\n      const globalCounts: Record<string, number> = {}\n      for (const sr of agentResult.scenarioResults) {\n        const avgs = computeMetricAverages(sr.evaluations.values())\n        for (const [m, v] of Object.entries(avgs)) {\n          globalSums[m] = (globalSums[m] ?? 0) + v\n          globalCounts[m] = (globalCounts[m] ?? 0) + 1\n        }\n      }\n\n      const scoreCells = metrics.map((m) => {\n        if (globalCounts[m]) return (globalSums[m] / globalCounts[m]).toFixed(1)\n        return '—'\n      })\n\n      lines.push(\n        `| ${escapeMarkdown(agentName)} | ${agentResult.passedScenarios}/${agentResult.totalScenarios} ${statusIcon} | ${scoreCells.join(' | ')} |`,\n      )\n    }\n\n    // Per-scenario breakdown\n    lines.push('', '### Per-Scenario Breakdown', '')\n    for (const sc of result.scenarioComparisons) {\n      lines.push(`#### ${escapeMarkdown(sc.scenario.name)}`, '')\n      const scHeaders = ['Agent', 'Status', ...metrics]\n      lines.push(`| ${scHeaders.join(' | ')} |`)\n      lines.push(`| ${scHeaders.map(() => '---').join(' | ')} |`)\n\n      for (const { agentName, scenarioResult } of sc.agentResults) {\n        const status = scenarioResult.passed ? '✅' : '❌'\n        const avgs = computeMetricAverages(scenarioResult.evaluations.values())\n        const cells = metrics.map((m) => (avgs[m] != null ? avgs[m].toFixed(1) : '—'))\n        lines.push(`| ${escapeMarkdown(agentName)} | ${status} | ${cells.join(' | ')} |`)\n      }\n      lines.push('')\n    }\n\n    return lines.join('\\n')\n  }\n\n  // --- Helpers ---\n\n  private collectMetricNames(scenarioResults: ScenarioResult[]): string[] {\n    const names = new Set<string>()\n    for (const sr of scenarioResults) {\n      for (const name of Object.keys(computeMetricAverages(sr.evaluations.values()))) {\n        names.add(name)\n      }\n    }\n    return [...names]\n  }\n\n  private getThresholdViolations(sr: ScenarioResult) {\n    return computeThresholdViolations(sr.evaluations.values(), this.thresholds)\n  }\n\n  private getFailureReason(result: ScenarioResult): string {\n    const reasons: string[] = []\n\n    const erroredConvs = result.simulation.conversations.filter((c) => c.error)\n    if (erroredConvs.length > 0) {\n      reasons.push(`${erroredConvs.length} conversation(s) errored`)\n    }\n\n    const failedTrajectories = [...result.trajectoryResults.values()].filter((t) => !t.matched)\n    if (failedTrajectories.length > 0) {\n      reasons.push(`${failedTrajectories.length} trajectory assertion(s) failed`)\n    }\n\n    const failedPerTurn = [...result.perTurnTrajectoryResults.values()].flatMap((rs) =>\n      rs.filter((r) => !r.result.matched),\n    )\n    if (failedPerTurn.length > 0) {\n      reasons.push(`${failedPerTurn.length} per-turn trajectory assertion(s) failed`)\n    }\n\n    if (result.errors.length > 0) {\n      const critical = result.errors.filter(\n        (e) => e.severity === 'critical' || e.severity === 'high',\n      )\n      if (critical.length > 0) {\n        reasons.push(`${critical.length} critical/high error(s)`)\n      }\n    }\n\n    const violations = this.getThresholdViolations(result)\n    if (violations.length > 0) {\n      reasons.push(`${violations.length} threshold violation(s)`)\n    }\n\n    return reasons.length > 0 ? reasons.join('; ') : 'Unknown failure reason'\n  }\n}\n"]}