import { withOverallScores } from "./statistics.js"; import type { BenchmarkResult, ModelSummary } from "./types.js"; const percent = (value: number | null | undefined) => value == null ? "N/A" : `${(value * 100).toFixed(1)}%`; const number = (value: number | undefined) => (value ?? 0).toFixed(1); const formatDuration = (milliseconds: number | undefined) => { const value = milliseconds ?? 0; return value >= 1000 ? `${(value / 1000).toFixed(1)}s` : `${value.toFixed(0)}ms`; }; const money = (value: number | undefined) => `$${(value ?? 0).toFixed(6)}`; const runMoney = (value: number) => `$${value.toFixed(2)}`; const metricNumber = (value: number | null | undefined) => value == null ? "N/A" : value.toFixed(1); function benchmarkRounds(summary: ModelSummary): number { const taskCount = summary.tasks?.length ?? 0; return taskCount > 0 && summary.count > 0 ? summary.count / taskCount : Math.max(1, summary.settings?.runs ?? 1); } export function costPerBenchmarkRun(summary: ModelSummary): number { return summary.totalCost / benchmarkRounds(summary); } export function perfPerDollar(summary: ModelSummary): number | null { const cost = costPerBenchmarkRun(summary); return cost > 0 ? (summary.passRate * 100) / cost : null; } function reportTitle(profileName: string): string { return profileName.toLowerCase() === "simplebench" ? "SimpleBench" : `ModelBench — ${profileName}`; } function escapeHtml(value: unknown): string { return String(value) .replaceAll("&", "&") .replaceAll("<", "<") .replaceAll(">", ">") .replaceAll('"', """) .replaceAll("'", "'"); } function configurationLabel(summary: ModelSummary): string { return `${summary.model.provider}/${summary.model.id} [thinking:${summary.settings?.reasoning ?? "unknown"}, temp:${summary.settings?.temperature ?? "?"}, max:${summary.settings?.maxTokens ?? "?"}]`; } function summaryRows(summaries: BenchmarkResult["models"]): string[][] { return summaries.map((summary, index) => [ `C${index + 1}`, String(summary.tasks?.length ?? 0), summary.verificationTestsTotal == null ? "N/A" : `${summary.verificationTestsPassed ?? 0}/${summary.verificationTestsTotal}`, percent(summary.passRate), number(summary.meanScore), number(summary.overallScore), percent(summary.consistencyRate), `${formatDuration(summary.meanLatencyMs)}/${formatDuration(summary.p95LatencyMs)}`, number(summary.meanOutputTokens), runMoney(costPerBenchmarkRun(summary)), metricNumber(perfPerDollar(summary)), number(summary.meanOutputTokensPerSecond), percent(summary.errorRate), ]); } export function renderComparisonTable(summaries: BenchmarkResult["models"], totalCost?: number): string { const headers = ["Cfg", "Tasks", "Tests", "Pass", "Score", "Overall", "Stable", "Mean/P95", "Out tok/Q", "$/run", "Perf/$", "Out tok/s", "Errors"]; const rows = summaryRows(withOverallScores(summaries)); const widths = headers.map((header, column) => Math.max(header.length, ...rows.map((row) => row[column]?.length ?? 0))); const formatRow = (row: string[]) => `| ${row.map((value, column) => value.padEnd(widths[column] ?? value.length)).join(" | ")} |`; const separator = `|-${widths.map((width) => "-".repeat(width)).join("-+-")}-|`; const legend = summaries.map((summary, index) => `C${index + 1}: ${configurationLabel(summary)}`); const calculatedTotal = totalCost ?? summaries.reduce((sum, summary) => sum + (summary.totalCost ?? 0), 0); const totalAttempts = summaries.reduce((sum, summary) => sum + summary.count, 0); const roundsPerConfig = summaries.length > 0 ? Math.max(...summaries.map(benchmarkRounds)) : 1; const tasksPerConfig = summaries[0]?.tasks?.length ?? 0; const footer = [ `Rounds per model: ${roundsPerConfig}`, `Tasks per model: ${tasksPerConfig}`, `Total attempts: ${totalAttempts}`, `Total bench cost: ${money(calculatedTotal)}`, "Perf/$ = pass-rate percentage / cost for one complete benchmark run", "Overall = 60% pass + 20% relative cost efficiency + 20% relative latency efficiency", "", ...legend, ]; return [formatRow(headers), separator, ...rows.map(formatRow), ...footer].join("\n"); } function htmlSummaryTable(result: BenchmarkResult): string { const rows = withOverallScores(result.models).map((summary, index) => ` C${index + 1} ${escapeHtml(configurationLabel(summary))} ${summary.tasks?.length ?? 0} ${summary.verificationTestsTotal == null ? "N/A" : `${summary.verificationTestsPassed ?? 0}/${summary.verificationTestsTotal}`} ${percent(summary.passRate)} ${number(summary.meanScore)} ${number(summary.overallScore)} ${percent(summary.consistencyRate)} ${formatDuration(summary.meanLatencyMs)} / ${formatDuration(summary.p95LatencyMs)} ${number(summary.meanOutputTokens)} ${runMoney(costPerBenchmarkRun(summary))} ${metricNumber(perfPerDollar(summary))} ${number(summary.meanOutputTokensPerSecond)} ${percent(summary.errorRate)} `).join("\n"); return `${rows}
CfgConfigurationTasksTestsPass rateScoreOverallStabilityMean / P95 latencyOut tok/Q$/runPerf/$Output tok/sErrors
`; } function sameRecordConfiguration(record: BenchmarkResult["records"][number], model: ModelSummary): boolean { return record.model.provider === model.model.provider && record.model.id === model.model.id && record.settings.temperature === model.settings.temperature && record.settings.maxTokens === model.settings.maxTokens && record.settings.reasoning === model.settings.reasoning; } function htmlAttempt(record: BenchmarkResult["records"][number]): string { const coding = "coding" in record ? record.coding as { verification: { exitCode: number | null; signal: string | null; stdout: string; stderr: string; durationMs: number; timedOut: boolean; testsPassed: number | null; testsTotal: number | null; error?: string }; failureCategory: string; toolTurns: number; changedFiles: string[]; outsideScopeFiles: string[]; diff: string; messages: unknown[]; } : undefined; return `
${escapeHtml(`attempt ${record.attempt}`)} — ${record.grade.passed ? "PASS" : "FAIL"}
Grade
${number(record.grade.score)} — ${escapeHtml(record.grade.details)}
Latency
${formatDuration(record.latencyMs)}
Tokens
${record.usage.input} input / ${record.usage.output} output
${record.error ? `
Error
${escapeHtml(record.error)}
` : ""} ${coding ? `
Tool turns
${coding.toolTurns}
Verification
${coding.verification.exitCode === 0 && !coding.verification.timedOut ? "passed" : "failed"} (${coding.verification.testsPassed ?? "?"}/${coding.verification.testsTotal ?? "?"} tests, ${formatDuration(coding.verification.durationMs)})
Changed files
${escapeHtml(coding.changedFiles.join(", ") || "none")}
Outside scope
${escapeHtml(coding.outsideScopeFiles.join(", ") || "none")}
Failure category
${escapeHtml(coding.failureCategory)}
` : ""}

Prompt

${escapeHtml(record.prompt)}

Output

${escapeHtml(record.output)}
${coding ? `

Verification stdout

${escapeHtml(coding.verification.stdout)}

Verification stderr

${escapeHtml(coding.verification.stderr)}

Diff summary

${escapeHtml(coding.diff || "No repository changes")}
Raw agent messages
${escapeHtml(JSON.stringify(coding.messages, null, 2))}
` : ""}
`; } function htmlTaskTable(result: BenchmarkResult): string { const rows = result.models.flatMap((model, modelIndex) => (model.tasks ?? []).map((task) => { const attempts = result.records.filter((record) => record.taskId === task.taskId && sameRecordConfiguration(record, model)); return `
C${modelIndex + 1}${escapeHtml(task.taskId)}${escapeHtml(task.tags.join(", "))}${percent(task.passRate)}${number(task.meanScore)}${formatDuration(task.meanLatencyMs)} / ${formatDuration(task.p95LatencyMs)}${percent(task.errorRate)}
${attempts.map(htmlAttempt).join("")}
`; })).join(""); return `
CfgTaskTagsPass rateScoreMean / P95 latencyErrors
${rows || `

Task summaries are unavailable in this older run artifact.

`}
`; } export function renderHtml(result: BenchmarkResult): string { return ` ${escapeHtml(reportTitle(result.profile.name))} — ${escapeHtml(result.runId)}

${escapeHtml(reportTitle(result.profile.name))}

${escapeHtml(result.profile.description)}

Run: ${escapeHtml(result.runId)}
Started: ${escapeHtml(result.startedAt)}
Finished: ${escapeHtml(result.finishedAt)}
Attempts: ${result.records.length}
Rounds per model: ${result.models.length > 0 ? Math.max(...result.models.map(benchmarkRounds)) : 1}
Tasks per model: ${result.models[0]?.tasks?.length ?? 0}

Configuration comparison

Total bench cost: ${money(result.totalCost ?? result.models.reduce((sum, model) => sum + (model.totalCost ?? 0), 0))}

${htmlSummaryTable(result)}

Pass rate and score measure graded task quality. Perf/$ is the pass-rate percentage divided by the cost of one complete benchmark run; $/run normalizes repeated rounds. Overall is a within-run comparison: 60% pass rate, 20% relative cost efficiency, and 20% relative latency efficiency. It is not comparable across separate runs. Stability is the share of tasks whose repeated attempts agreed (N/A when tasks were run once). Output tok/Q, output tok/s, and latency measure efficiency; the benchmark total cost is shown above.

Per-task capability and precision

${htmlTaskTable(result)}

Method

\n`; }