/**
* HTML Evaluator - Browser-based HTML/JavaScript evaluation
*
* Uses Playwright for real browser rendering, screenshot capture,
* and console error detection.
*/
import { BaseEvaluator, EvaluationResult } from '../base-evaluator';
import * as fs from 'fs';
import * as path from 'path';
// Playwright types (optional dependency)
type Browser = {
newPage: () => Promise;
close: () => Promise;
};
type Page = {
setContent: (html: string) => Promise;
waitForLoadState: (state: string, options?: { timeout: number }) => Promise;
screenshot: (options: { path: string; fullPage: boolean }) => Promise;
on: (event: string, callback: (msg: unknown) => void) => void;
};
type ConsoleMessage = {
type: () => string;
text: () => string;
};
type Playwright = {
chromium: {
launch: (options: { headless: boolean }) => Promise;
};
};
export interface HTMLEvaluationResult extends EvaluationResult {
details: {
renders: boolean;
errors: string[];
warnings: string[];
screenshot?: string;
render_time_ms: number;
has_doctype: boolean;
has_required_tags: boolean;
structure_issues: string[];
[key: string]: unknown;
};
}
/**
* HTML/JavaScript evaluator using Playwright for browser testing.
*
* Scoring:
* - Renders successfully: 50 points
* - No console errors: 30 points
* - Fast render (<3s): 20 points
*/
export class HTMLEvaluator extends BaseEvaluator {
private headless: boolean;
private outputDir: string;
private playwrightAvailable: boolean = false;
private playwright: Playwright | null = null;
constructor(headless = true, outputDir = 'output/screenshots') {
super();
this.headless = headless;
this.outputDir = outputDir;
this.checkPlaywright();
}
private async checkPlaywright(): Promise {
try {
// Dynamic import to make playwright optional
// eslint-disable-next-line @typescript-eslint/no-var-requires
const pw = require('playwright') as Playwright;
this.playwright = pw;
this.playwrightAvailable = true;
} catch {
this.playwrightAvailable = false;
// Silently fail - playwright is optional
}
}
getLanguage(): string {
return 'html';
}
getFileExtension(): string {
return 'html';
}
/**
* Extract HTML code from response (handles markdown code blocks).
*/
private extractHTML(response: string): string {
// Try to extract from HTML code block
const htmlMatch = response.match(/```html\s*([\s\S]*?)```/i);
if (htmlMatch) {
return htmlMatch[1].trim();
}
// Try generic code block
const codeMatch = response.match(/```\s*([\s\S]*?)```/);
if (codeMatch) {
return codeMatch[1].trim();
}
// Return as-is if no code block
return response.trim();
}
/**
* Validate HTML structure statically.
*/
private validateStructure(html: string): {
score: number;
has_doctype: boolean;
has_required_tags: boolean;
issues: string[];
} {
let score = 0;
const issues: string[] = [];
// DOCTYPE check (25 points)
const hasDoctype = html.toUpperCase().includes('`);
}
}
score += (foundTags / requiredTags.length) * 25;
// Basic structure validation (50 points)
// Check for matching tags
const openTags = (html.match(/<[a-z][^>]*>/gi) || []).length;
if (openTags > 0) {
score += 50;
} else {
issues.push('No HTML tags found');
}
return {
score: Math.round(score),
has_doctype: hasDoctype,
has_required_tags: foundTags === requiredTags.length,
issues,
};
}
/**
* Evaluate HTML using browser (if Playwright available) or static validation.
*/
async evaluate(
code: string,
testName: string,
_prompt: string,
_expected?: string
): Promise {
const html = this.extractHTML(code);
const structureResult = this.validateStructure(html);
const result: HTMLEvaluationResult = {
score: 0,
passed: false,
feedback: [],
details: {
renders: false,
errors: [],
warnings: [],
render_time_ms: 0,
has_doctype: structureResult.has_doctype,
has_required_tags: structureResult.has_required_tags,
structure_issues: structureResult.issues,
},
};
// Add structure feedback
if (structureResult.has_doctype) {
result.feedback.push({
level: 'success',
message: '✅ DOCTYPE declaration present',
});
} else {
result.feedback.push({
level: 'warning',
message: '⚠️ Missing DOCTYPE declaration',
});
}
if (structureResult.has_required_tags) {
result.feedback.push({
level: 'success',
message: '✅ Required HTML tags present (html, head, body)',
});
} else {
result.feedback.push({
level: 'warning',
message: '⚠️ Missing some required HTML tags',
});
}
// Try browser evaluation if Playwright is available
if (this.playwrightAvailable && this.playwright) {
try {
const browserResult = await this.evaluateInBrowser(html, testName);
result.details.renders = browserResult.renders;
result.details.errors = browserResult.errors;
result.details.warnings = browserResult.warnings;
result.details.screenshot = browserResult.screenshot;
result.details.render_time_ms = browserResult.render_time_ms;
// Calculate score based on browser evaluation
let browserScore = 0;
// Renders successfully: 50 points
if (browserResult.renders) {
browserScore += 50;
result.feedback.push({
level: 'success',
message: '✅ HTML renders successfully in browser',
});
} else {
result.feedback.push({
level: 'error',
message: '❌ HTML failed to render in browser',
});
}
// No console errors: 30 points
if (browserResult.errors.length === 0) {
browserScore += 30;
result.feedback.push({
level: 'success',
message: '✅ No console errors',
});
} else if (browserResult.errors.length <= 2) {
browserScore += 15;
result.feedback.push({
level: 'warning',
message: `⚠️ ${browserResult.errors.length} console error(s)`,
details: browserResult.errors.join('; '),
});
} else {
result.feedback.push({
level: 'error',
message: `❌ ${browserResult.errors.length} console errors`,
details: browserResult.errors.slice(0, 3).join('; '),
});
}
// Fast render: 20 points
if (browserResult.render_time_ms > 0 && browserResult.render_time_ms < 1000) {
browserScore += 20;
result.feedback.push({
level: 'success',
message: `✅ Fast render time: ${browserResult.render_time_ms}ms`,
});
} else if (browserResult.render_time_ms < 3000) {
browserScore += 10;
result.feedback.push({
level: 'info',
message: `📊 Render time: ${browserResult.render_time_ms}ms`,
});
}
if (browserResult.screenshot) {
result.feedback.push({
level: 'info',
message: `📸 Screenshot saved: ${browserResult.screenshot}`,
});
}
result.score = browserScore;
} catch (error) {
// Fall back to structure-only score
result.score = Math.round(structureResult.score * 0.5); // 50% weight for structure only
result.feedback.push({
level: 'warning',
message: `⚠️ Browser evaluation failed: ${error instanceof Error ? error.message : String(error)}`,
});
}
} else {
// Structure-only evaluation (50% weight)
result.score = Math.round(structureResult.score * 0.5);
result.feedback.push({
level: 'info',
message: '📊 Using static HTML validation (Playwright not available)',
});
}
result.passed = result.score >= 70;
result.overall_score = result.score;
return result;
}
/**
* Evaluate HTML in a real browser using Playwright.
*/
private async evaluateInBrowser(
html: string,
testName: string
): Promise<{
renders: boolean;
errors: string[];
warnings: string[];
screenshot?: string;
render_time_ms: number;
}> {
if (!this.playwright) {
throw new Error('Playwright not available');
}
const errors: string[] = [];
const warnings: string[] = [];
let renders = false;
let renderTime = 0;
let screenshotPath: string | undefined;
const browser = await this.playwright.chromium.launch({
headless: this.headless,
});
try {
const page = await browser.newPage();
// Capture console messages
page.on('console', (msg: unknown) => {
const consoleMsg = msg as ConsoleMessage;
if (consoleMsg.type && consoleMsg.type() === 'error') {
errors.push(consoleMsg.text());
} else if (consoleMsg.type && consoleMsg.type() === 'warning') {
warnings.push(consoleMsg.text());
}
});
page.on('pageerror', (err: unknown) => {
errors.push(String(err));
});
// Load HTML and measure time
const startTime = Date.now();
await page.setContent(html);
await page.waitForLoadState('networkidle', { timeout: 5000 });
renderTime = Date.now() - startTime;
renders = true;
// Take screenshot
fs.mkdirSync(this.outputDir, { recursive: true });
screenshotPath = path.join(this.outputDir, `${testName}.png`);
await page.screenshot({ path: screenshotPath, fullPage: true });
} catch (error) {
errors.push(`Browser error: ${error instanceof Error ? error.message : String(error)}`);
} finally {
await browser.close();
}
return {
renders,
errors,
warnings,
screenshot: screenshotPath,
render_time_ms: renderTime,
};
}
}