#!/usr/bin/env python3 """Generate an HTML report for description optimization runs.""" from __future__ import annotations import argparse import html import json import sys from pathlib import Path def _score_class(correct: int, total: int) -> str: if total <= 0: return "score-bad" ratio = correct / total if ratio >= 0.8: return "score-good" if ratio >= 0.5: return "score-ok" return "score-bad" def _aggregate_runs(results: list[dict]) -> tuple[int, int]: correct = 0 total = 0 for result in results: runs = int(result.get("runs", 0)) triggers = int(result.get("triggers", 0)) total += runs if result.get("should_trigger", True): correct += triggers else: correct += runs - triggers return correct, total def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str: """Render the optimization history as a standalone HTML page.""" history = data.get("history", []) train_queries: list[dict] = [] test_queries: list[dict] = [] if history: first = history[0] for item in first.get("train_results", first.get("results", [])): train_queries.append( {"query": item["query"], "should_trigger": item.get("should_trigger", True)} ) for item in first.get("test_results", []) or []: test_queries.append( {"query": item["query"], "should_trigger": item.get("should_trigger", True)} ) if history: if test_queries: best_iteration = max( history, key=lambda item: ( (item.get("test_passed") or 0) / max(item.get("test_total") or 1, 1), (item.get("train_passed") or 0) / max(item.get("train_total") or 1, 1), -len(item.get("description", "")), ), ).get("iteration") else: best_iteration = max( history, key=lambda item: ( (item.get("train_passed") or item.get("passed", 0)) / max(item.get("train_total") or item.get("total", 1), 1), -len(item.get("description", "")), ), ).get("iteration") else: best_iteration = None refresh_tag = ' \n' if auto_refresh else "" title_prefix = html.escape(f"{skill_name} - ") if skill_name else "" html_parts = [ "\n", "\n", "\n", ' \n', refresh_tag, f" {title_prefix}Description Optimization\n", " \n", "\n", "\n", f"

{title_prefix}Description Optimization

\n", '
\n', "

Original: " + html.escape(data.get("original_description", "N/A")) + "

\n", "

Best: " + html.escape(data.get("best_description", "N/A")) + "

\n", "

Exit reason: " + html.escape(data.get("exit_reason", "unknown")) + "

\n", "

Best score: " + html.escape(data.get("best_score", "N/A")) + "

\n", "

Iterations: " + html.escape(str(data.get("iterations_run", 0))) + " | Train: " + html.escape(str(data.get("train_size", "?"))) + " | Test: " + html.escape(str(data.get("test_size", "?"))) + "

\n", "
\n", '
\n', "

How to read this: each row is one description candidate. Checkmarks mean the query behaved as expected. Numbers under each mark show how many repeated runs triggered.

\n", '
\n', ' Should trigger\n', ' Should not trigger\n', ' Held-out test query\n', "
\n", "
\n", '
\n', " \n", " \n", " \n", " \n", " \n", " \n", " \n", ] for query_info in train_queries: polarity_class = "query-positive" if query_info["should_trigger"] else "query-negative" html_parts.append( f" \n" ) for query_info in test_queries: polarity_class = "query-positive" if query_info["should_trigger"] else "query-negative" html_parts.append( f" \n" ) html_parts.extend( [ " \n", " \n", " \n", ] ) for item in history: train_results = item.get("train_results", item.get("results", [])) test_results = item.get("test_results", []) or [] train_by_query = {result["query"]: result for result in train_results} test_by_query = {result["query"]: result for result in test_results} train_correct, train_total = _aggregate_runs(train_results) test_correct, test_total = _aggregate_runs(test_results) row_class = "best-row" if item.get("iteration") == best_iteration else "" html_parts.append( f" \n" f" \n" f" \n" f" \n" f" \n" ) for query_info in train_queries: result = train_by_query.get(query_info["query"], {}) passed = bool(result.get("pass")) triggers = result.get("triggers", 0) runs = result.get("runs", 0) icon = "PASS" if passed else "FAIL" html_parts.append( " \n" ) for query_info in test_queries: result = test_by_query.get(query_info["query"], {}) passed = bool(result.get("pass")) triggers = result.get("triggers", 0) runs = result.get("runs", 0) icon = "PASS" if passed else "FAIL" html_parts.append( " \n" ) html_parts.append(" \n") html_parts.extend( [ " \n", "
IterTrainTestDescription{html.escape(query_info['query'])}{html.escape(query_info['query'])}
{html.escape(str(item.get('iteration', '?')))}{train_correct}/{train_total}{test_correct}/{test_total}{html.escape(item.get('description', ''))}" + icon + f"{triggers}/{runs}" + icon + f"{triggers}/{runs}
\n", "
\n", "\n", "\n", ] ) return "".join(html_parts) def main() -> int: parser = argparse.ArgumentParser(description="Generate an HTML report from run_loop output") parser.add_argument("input", help="Path to run_loop JSON output, or - for stdin") parser.add_argument("-o", "--output", help="Write HTML to this path instead of stdout") parser.add_argument("--skill-name", default="", help="Optional skill name for the title") args = parser.parse_args() if args.input == "-": data = json.load(sys.stdin) else: data = json.loads(Path(args.input).read_text(encoding="utf-8")) html_output = generate_html(data, skill_name=args.skill_name) if args.output: Path(args.output).write_text(html_output, encoding="utf-8") print(f"Report written to {args.output}", file=sys.stderr) else: print(html_output) return 0 if __name__ == "__main__": raise SystemExit(main())