#!/usr/bin/env python3 """Generate an HTML review page for skill-creator benchmark results. Reads iteration workspace directories containing with_skill/ and baseline/ results, then produces a self-contained dark-mode HTML report with side-by-side comparison, assertion pass/fail coloring, and feedback collection. Usage: python generate_review.py /iteration-N --skill-name \ --benchmark /iteration-N/benchmark.json \ [--previous-workspace ] [--static ] Requirements: Python 3.8+, standard library only. """ from __future__ import annotations import argparse import html import json import sys from pathlib import Path from typing import Any, Dict, List, Optional def load_json(path: Path) -> Any: """Load a JSON file, returning None on failure.""" try: with open(path, "r", encoding="utf-8") as f: return json.load(f) except (OSError, json.JSONDecodeError): return None def discover_results(iteration_dir: Path) -> Dict[str, Any]: """Discover with_skill and baseline result files in an iteration directory.""" results: Dict[str, Any] = {"with_skill": [], "baseline": []} for variant in ("with_skill", "baseline"): variant_dir = iteration_dir / variant if not variant_dir.is_dir(): continue for json_file in sorted(variant_dir.glob("*.json")): data = load_json(json_file) if data is None: continue # Handle both list-of-scenarios and single-scenario dict if isinstance(data, list): results[variant].extend(data) elif isinstance(data, dict): results[variant].append(data) return results def count_assertions(scenarios: List[Dict]) -> Dict[str, int]: """Count pass/fail/total assertions across scenarios.""" counts = {"pass": 0, "fail": 0, "total": 0} for scenario in scenarios: for assertion in scenario.get("assertions", []): counts["total"] += 1 if assertion.get("passed", False): counts["pass"] += 1 else: counts["fail"] += 1 return counts def fmt_float(val: Any, decimals: int = 2) -> str: """Format a float value, returning '—' for missing data.""" if val is None: return "\u2014" try: return f"{float(val):.{decimals}f}" except (TypeError, ValueError): return "\u2014" def build_summary_section(benchmark: Optional[Dict]) -> str: """Build the summary statistics section from benchmark.json data.""" if not benchmark: return "" def stat_card(label: str, value: str, sub: str = "") -> str: sub_html = f'
{html.escape(sub)}
' if sub else "" return ( f'
' f'
{html.escape(label)}
' f'
{html.escape(value)}
' f'{sub_html}
' ) cards = [] # Pass rate ws = benchmark.get("with_skill", {}) bl = benchmark.get("baseline", {}) ws_rate = ws.get("pass_rate") bl_rate = bl.get("pass_rate") if ws_rate is not None: cards.append(stat_card("Pass Rate (with skill)", f"{fmt_float(ws_rate, 1)}%", f"baseline: {fmt_float(bl_rate, 1)}%")) # Tokens ws_tokens = ws.get("tokens", {}) bl_tokens = bl.get("tokens", {}) if ws_tokens.get("mean") is not None: cards.append(stat_card( "Tokens (with skill)", f"{fmt_float(ws_tokens.get('mean'), 0)}", f"\u00b1{fmt_float(ws_tokens.get('stddev'), 0)} | baseline: {fmt_float(bl_tokens.get('mean'), 0)}" )) # Time ws_time = ws.get("time", {}) bl_time = bl.get("time", {}) if ws_time.get("mean") is not None: cards.append(stat_card( "Time (with skill)", f"{fmt_float(ws_time.get('mean'))}s", f"\u00b1{fmt_float(ws_time.get('stddev'))}s | baseline: {fmt_float(bl_time.get('mean'))}s" )) if not cards: return "" return ( '
' '

Summary

' '
' + "".join(cards) + '
' '
' ) def build_scenario_rows(scenarios: List[Dict]) -> str: """Build HTML table rows for scenario assertions with collapsible evidence.""" if not scenarios: return 'No scenarios found' rows = [] for scenario in scenarios: name = html.escape(scenario.get("name", scenario.get("query", "Unknown"))) assertions = scenario.get("assertions", []) tokens = scenario.get("tokens") time_s = scenario.get("time") meta_parts = [] if tokens is not None: meta_parts.append(f"{tokens} tok") if time_s is not None: meta_parts.append(f"{fmt_float(time_s)}s") meta_str = " | ".join(meta_parts) if not assertions: rows.append( f'{name}' 'No assertions' ) continue for i, assertion in enumerate(assertions): label = html.escape(assertion.get("description", assertion.get("name", f"assertion-{i}"))) passed = assertion.get("passed", False) status_cls = "pass" if passed else "fail" status_text = "PASS" if passed else "FAIL" detail = html.escape(assertion.get("detail", assertion.get("message", ""))) # Collapsible evidence evidence = assertion.get("evidence", "") evidence_html = "" if evidence: esc_ev = html.escape(str(evidence)) evidence_html = ( f'
Evidence' f'
{esc_ev}
' ) scenario_cell = ( f'{name}' f'{"
" + html.escape(meta_str) + "" if meta_str and i == 0 else ""}' f'' ) if i == 0 else "" rows.append( f'{scenario_cell}' f'{label}' f'{status_text}' f'{detail}{evidence_html}' ) return "\n".join(rows) def build_comparison_section( with_skill: List[Dict], baseline: List[Dict] ) -> str: """Build side-by-side comparison HTML for with_skill vs baseline.""" ws_counts = count_assertions(with_skill) bl_counts = count_assertions(baseline) return f"""

With Skill

{ws_counts['pass']} pass {ws_counts['fail']} fail {ws_counts['total']} total
{build_scenario_rows(with_skill)}
ScenarioAssertionResultDetail

Baseline

{bl_counts['pass']} pass {bl_counts['fail']} fail {bl_counts['total']} total
{build_scenario_rows(baseline)}
ScenarioAssertionResultDetail
""" def build_previous_comparison( current_dir: Path, previous_dir: Path, current_benchmark: Optional[Dict] = None, previous_benchmark: Optional[Dict] = None, ) -> str: """Build a delta section comparing current vs previous iteration.""" curr = discover_results(current_dir) prev = discover_results(previous_dir) curr_ws = count_assertions(curr["with_skill"]) prev_ws = count_assertions(prev["with_skill"]) pass_delta = curr_ws["pass"] - prev_ws["pass"] fail_delta = curr_ws["fail"] - prev_ws["fail"] pass_sign = "+" if pass_delta >= 0 else "" fail_sign = "+" if fail_delta >= 0 else "" pass_cls = "delta-positive" if pass_delta >= 0 else "delta-negative" fail_cls = "delta-negative" if fail_delta >= 0 else "delta-positive" # Pass rate delta from benchmark data rate_html = "" if current_benchmark and previous_benchmark: curr_rate = current_benchmark.get("with_skill", {}).get("pass_rate") prev_rate = previous_benchmark.get("with_skill", {}).get("pass_rate") if curr_rate is not None and prev_rate is not None: rate_delta = curr_rate - prev_rate rate_sign = "+" if rate_delta >= 0 else "" rate_cls = "delta-positive" if rate_delta >= 0 else "delta-negative" rate_html = f"""
Pass Rate
{rate_sign}{fmt_float(rate_delta, 1)}%
{fmt_float(prev_rate, 1)}% → {fmt_float(curr_rate, 1)}%
""" return f"""

Delta vs Previous Iteration

{rate_html}
Pass
{pass_sign}{pass_delta}
{prev_ws['pass']} → {curr_ws['pass']}
Fail
{fail_sign}{fail_delta}
{prev_ws['fail']} → {curr_ws['fail']}
""" def generate_html( skill_name: str, iteration_dir: Path, benchmark: Optional[Dict] = None, previous_dir: Optional[Path] = None, previous_benchmark: Optional[Dict] = None, ) -> str: """Generate the complete HTML review page.""" results = discover_results(iteration_dir) iteration_name = iteration_dir.name summary_html = build_summary_section(benchmark) comparison_html = build_comparison_section( results["with_skill"], results["baseline"] ) delta_html = "" if previous_dir and previous_dir.is_dir(): delta_html = build_previous_comparison( iteration_dir, previous_dir, benchmark, previous_benchmark ) return f""" Benchmark Review \u2014 {html.escape(skill_name)}

Benchmark Review \u2014 {html.escape(skill_name)}

Iteration: {html.escape(iteration_name)} | Generated from: {html.escape(str(iteration_dir))}

{summary_html} {delta_html}

Side-by-Side Comparison

{comparison_html} """ def main() -> int: parser = argparse.ArgumentParser( description="Generate HTML review for skill-creator benchmark results." ) parser.add_argument( "iteration_dir", type=Path, help="Path to iteration directory (e.g. workspace/iteration-1)", ) parser.add_argument( "--skill-name", required=True, help="Name of the skill being benchmarked", ) parser.add_argument( "--benchmark", type=Path, default=None, help="Path to benchmark.json with statistics (pass_rate, tokens, time)", ) parser.add_argument( "--previous-workspace", type=Path, default=None, help="Path to previous iteration directory for delta comparison", ) parser.add_argument( "--static", type=Path, default=None, help="Output path for static HTML file (default: stdout)", ) args = parser.parse_args() if not args.iteration_dir.is_dir(): print(f"Error: {args.iteration_dir} is not a directory", file=sys.stderr) return 1 benchmark = None if args.benchmark: benchmark = load_json(args.benchmark) if benchmark is None: print(f"Warning: could not load {args.benchmark}", file=sys.stderr) previous_benchmark = None if args.previous_workspace: prev_bench_path = args.previous_workspace / "benchmark.json" previous_benchmark = load_json(prev_bench_path) html_content = generate_html( skill_name=args.skill_name, iteration_dir=args.iteration_dir, benchmark=benchmark, previous_dir=args.previous_workspace, previous_benchmark=previous_benchmark, ) if args.static: args.static.parent.mkdir(parents=True, exist_ok=True) with open(args.static, "w", encoding="utf-8") as f: f.write(html_content) print(f"Written to {args.static}", file=sys.stderr) else: print(html_content) return 0 if __name__ == "__main__": sys.exit(main())