#!/usr/bin/env python3
"""Generate an HTML review page for eval results.
Creates an interactive HTML page showing eval outputs, grading results,
and benchmark data for human review.
Usage:
python3 generate_review.py
python3 generate_review.py --static output.html
python3 generate_review.py --open
Options:
--static Generate a standalone HTML file instead of serving
--open Open the generated HTML in the default browser
--iteration Show specific iteration (default: latest)
"""
import argparse
import json
import os
import sys
import webbrowser
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
from utils import load_json, find_latest_iteration
def collect_eval_data(iteration_dir):
"""Collect all eval data from an iteration directory."""
iteration_dir = Path(iteration_dir)
evals = []
for eval_dir in sorted(iteration_dir.iterdir()):
if not eval_dir.is_dir() or not eval_dir.name.startswith("eval-"):
continue
eval_id = eval_dir.name.replace("eval-", "")
eval_data = {"id": eval_id, "configs": {}}
for config in ["with_skill", "without_skill"]:
config_dir = eval_dir / config
if not config_dir.exists():
continue
config_data = {"outputs": {}, "grading": None, "timing": None}
# Read outputs
outputs_dir = config_dir / "outputs"
if outputs_dir.exists():
for f in sorted(outputs_dir.iterdir()):
if f.is_file():
try:
config_data["outputs"][f.name] = f.read_text()[:5000]
except (UnicodeDecodeError, PermissionError):
config_data["outputs"][f.name] = "(binary file)"
# Read grading
grading_path = config_dir / "grading.json"
if grading_path.exists():
config_data["grading"] = load_json(grading_path)
# Read timing
timing_path = config_dir / "timing.json"
if timing_path.exists():
config_data["timing"] = load_json(timing_path)
eval_data["configs"][config] = config_data
evals.append(eval_data)
# Read benchmark
benchmark = None
benchmark_path = iteration_dir / "benchmark.json"
if benchmark_path.exists():
benchmark = load_json(benchmark_path)
return evals, benchmark
def generate_html(evals, benchmark, iteration_num, workspace_dir):
"""Generate the HTML review page."""
# Escape JSON for embedding in HTML
evals_json = json.dumps(evals, indent=2)
benchmark_json = json.dumps(benchmark, indent=2) if benchmark else "null"
html = f"""
Skill Eval Review - Iteration {iteration_num}
Skill Eval Review — Iteration {iteration_num}
{workspace_dir}
Outputs
Benchmark
"""
return html
def main():
parser = argparse.ArgumentParser(description="Generate HTML eval review page")
parser.add_argument("workspace", help="Path to skill workspace directory")
parser.add_argument("--static", help="Output path for standalone HTML file")
parser.add_argument("--open", action="store_true", help="Open in browser")
parser.add_argument("--iteration", type=int, help="Specific iteration to show")
args = parser.parse_args()
workspace = Path(args.workspace)
if not workspace.exists():
print(f"Error: {workspace} not found", file=sys.stderr)
sys.exit(1)
# Find iteration
if args.iteration:
iteration_num = args.iteration
else:
iteration_num = find_latest_iteration(workspace)
if iteration_num == 0:
print("Error: No iterations found in workspace", file=sys.stderr)
sys.exit(1)
iteration_dir = workspace / f"iteration-{iteration_num}"
if not iteration_dir.exists():
print(f"Error: {iteration_dir} not found", file=sys.stderr)
sys.exit(1)
# Collect data
evals, benchmark = collect_eval_data(iteration_dir)
if not evals:
print("Error: No eval data found", file=sys.stderr)
sys.exit(1)
# Generate HTML
html = generate_html(evals, benchmark, iteration_num, str(workspace))
# Output
output_path = args.static or str(workspace / "review.html")
Path(output_path).write_text(html)
print(f"Generated: {output_path}")
if args.open:
webbrowser.open(f"file://{os.path.abspath(output_path)}")
if __name__ == "__main__":
main()