/**
* TestDriver MCP App - displays screenshots with overlays for action results
*/
import {
App,
applyDocumentTheme,
applyHostFonts,
applyHostStyleVariables,
type McpUiHostContext,
} from "@modelcontextprotocol/ext-apps";
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
import { ReadResourceResultSchema } from "@modelcontextprotocol/sdk/types.js";
import "./mcp-app.css";
// DOM elements
const mainEl = document.querySelector(".main") as HTMLElement;
const containerEl = document.getElementById("screenshot-container") as HTMLDivElement;
const screenshotEl = document.getElementById("screenshot") as HTMLImageElement;
const overlaysEl = document.getElementById("overlays") as HTMLDivElement;
const actionStatusEl = document.getElementById("action-status") as HTMLSpanElement;
const sessionInfoEl = document.getElementById("session-info") as HTMLSpanElement;
const loadingOverlayEl = document.getElementById("loading-overlay") as HTMLDivElement;
const loadingTextEl = loadingOverlayEl.querySelector(".loading-text") as HTMLSpanElement;
// Create target info element dynamically
const targetInfoEl = document.createElement("div");
targetInfoEl.id = "target-info";
targetInfoEl.className = "target-info hidden";
// Track screenshot natural dimensions for coordinate scaling
let screenshotNaturalWidth = 0;
let screenshotNaturalHeight = 0;
// Zoom state - disabled, always show full view
let isZoomed = false;
const ZOOM_LEVEL = 2.0; // 2x zoom (not used when zoom disabled)
// Types for tool result data
interface ToolResultData {
action?: string;
success?: boolean;
imageUrl?: string; // Data URL with cropped image from find() response
screenshotResourceUri?: string; // Resource URI to fetch screenshot blob
croppedImageResourceUri?: string; // Resource URI to fetch cropped image from find operations
element?: {
description?: string;
x?: number;
y?: number;
centerX?: number;
centerY?: number;
width?: number;
height?: number;
confidence?: number;
ref?: string;
};
clickPosition?: { x: number; y: number; centerX?: number; centerY?: number };
scrollDirection?: string;
assertion?: string;
text?: string;
execResult?: string;
error?: string;
session?: {
id?: string;
expiresIn?: number;
};
debuggerUrl?: string;
sessionId?: string;
duration?: number;
}
// Store session info globally for display
let currentDebuggerUrl: string | null = null;
/**
* Extract structured data from tool result
*/
function extractData(result: CallToolResult): ToolResultData {
return (result.structuredContent as ToolResultData) ?? {};
}
/**
* Show loading state with optional custom message
*/
function showLoading(message = "Waiting for screenshot...") {
loadingTextEl.textContent = message;
loadingOverlayEl.classList.remove("hidden");
}
/**
* Hide loading state
*/
function hideLoading() {
loadingOverlayEl.classList.add("hidden");
}
/**
* Apply host context (theme, styles, safe areas, container dimensions)
*/
function handleHostContextChanged(ctx: McpUiHostContext) {
if (ctx.theme) {
applyDocumentTheme(ctx.theme);
}
if (ctx.styles?.variables) {
applyHostStyleVariables(ctx.styles.variables);
}
if (ctx.styles?.css?.fonts) {
applyHostFonts(ctx.styles.css.fonts);
}
if (ctx.safeAreaInsets) {
mainEl.style.paddingTop = `${ctx.safeAreaInsets.top}px`;
mainEl.style.paddingRight = `${ctx.safeAreaInsets.right}px`;
mainEl.style.paddingBottom = `${ctx.safeAreaInsets.bottom}px`;
mainEl.style.paddingLeft = `${ctx.safeAreaInsets.left}px`;
}
// Handle container dimensions to fit the entire image in the frame
const containerDimensions = (ctx as any).containerDimensions;
if (containerDimensions) {
// Handle height
if ("height" in containerDimensions) {
// Fixed height: fill the container
document.documentElement.style.height = "100vh";
mainEl.style.height = "100%";
} else if ("maxHeight" in containerDimensions && containerDimensions.maxHeight) {
// Flexible with max: let content determine size, up to max
document.documentElement.style.maxHeight = `${containerDimensions.maxHeight}px`;
mainEl.style.maxHeight = "100%";
}
// Handle width
if ("width" in containerDimensions) {
// Fixed width: fill the container
document.documentElement.style.width = "100vw";
mainEl.style.width = "100%";
} else if ("maxWidth" in containerDimensions && containerDimensions.maxWidth) {
// Flexible with max: let content determine size, up to max
document.documentElement.style.maxWidth = `${containerDimensions.maxWidth}px`;
mainEl.style.maxWidth = "100%";
}
}
}
/**
* Scale coordinates from screenshot natural size to displayed size
*/
function scaleCoord(value: number, naturalSize: number, displayedSize: number): number {
if (naturalSize === 0) return value;
return (value / naturalSize) * displayedSize;
}
/**
* Apply zoom transform to center on a point
*/
function applyZoom(centerX: number, centerY: number, zoom: boolean) {
if (!zoom) {
containerEl.style.transform = "none";
containerEl.classList.remove("zoomed");
return;
}
const displayedWidth = screenshotEl.clientWidth;
const displayedHeight = screenshotEl.clientHeight;
// Scale the target coordinates to displayed size
const scaledX = scaleCoord(centerX, screenshotNaturalWidth, displayedWidth);
const scaledY = scaleCoord(centerY, screenshotNaturalHeight, displayedHeight);
// Calculate translation to center the point
// After zoom, we need to translate so the point ends up in the center of the container
const containerWidth = containerEl.parentElement?.clientWidth || displayedWidth;
const containerHeight = containerEl.parentElement?.clientHeight || displayedHeight;
// The point's position after scaling
const scaledPointX = scaledX * ZOOM_LEVEL;
const scaledPointY = scaledY * ZOOM_LEVEL;
// Translation needed to center the point
const translateX = (containerWidth / 2) - scaledPointX;
const translateY = (containerHeight / 2) - scaledPointY;
containerEl.style.transform = `scale(${ZOOM_LEVEL}) translate(${translateX / ZOOM_LEVEL}px, ${translateY / ZOOM_LEVEL}px)`;
containerEl.classList.add("zoomed");
}
/**
* Reset zoom to show full screenshot
*/
function resetZoom() {
containerEl.style.transform = "none";
containerEl.classList.remove("zoomed");
}
/**
* Add overlays after screenshot loads (so we know dimensions)
* Uses requestAnimationFrame to ensure layout is computed
*/
function addOverlays(data: ToolResultData) {
// Use requestAnimationFrame to ensure the image has been laid out
requestAnimationFrame(() => {
overlaysEl.innerHTML = "";
const displayedWidth = screenshotEl.clientWidth;
const displayedHeight = screenshotEl.clientHeight;
console.info("addOverlays:", {
action: data.action,
hasElement: !!data.element,
hasClickPosition: !!data.clickPosition,
displayedWidth,
displayedHeight,
naturalWidth: screenshotNaturalWidth,
naturalHeight: screenshotNaturalHeight
});
// Skip if dimensions aren't ready yet
if (displayedWidth === 0 || displayedHeight === 0) {
console.warn("addOverlays: Dimensions not ready, retrying...");
// Retry after a short delay
setTimeout(() => addOverlays(data), 50);
return;
}
// Track the focal point for zoom
let focalX: number | undefined;
let focalY: number | undefined;
// Add element target overlay for 'find' and 'find_and_click' actions
// The cropped image is always centered on the found element, so position target at image center
const showElementTarget = (data.action === "find" || data.action === "find_and_click" || data.action === "findall") && data.element;
if (showElementTarget) {
const target = document.createElement("div");
target.className = "element-target";
// Position at center of displayed image (cropped image is already centered on element)
target.style.left = `${displayedWidth / 2}px`;
target.style.top = `${displayedHeight / 2}px`;
// Add crosshair lines
const crosshairH = document.createElement("div");
crosshairH.className = "crosshair-h";
target.appendChild(crosshairH);
const crosshairV = document.createElement("div");
crosshairV.className = "crosshair-v";
target.appendChild(crosshairV);
// Add label
const label = document.createElement("div");
label.className = "element-label";
label.textContent = data.element?.description || "Element";
if (data.element?.confidence) {
label.textContent += ` (${Math.round(data.element.confidence * 100)}%)`;
}
target.appendChild(label);
overlaysEl.appendChild(target);
// Set focal point for zoom at image center
focalX = screenshotNaturalWidth / 2;
focalY = screenshotNaturalHeight / 2;
console.info("addOverlays: Added element target at center");
}
// Add click marker overlay for click actions (uses full screenshot, not cropped)
// Use centerX/centerY if available (where the click actually happens), fallback to x/y
if (data.clickPosition) {
const clickX = data.clickPosition.centerX ?? data.clickPosition.x;
const clickY = data.clickPosition.centerY ?? data.clickPosition.y;
if (clickX !== undefined && clickY !== undefined && screenshotNaturalWidth > 0) {
const marker = document.createElement("div");
marker.className = "click-marker";
const scaledX = scaleCoord(clickX, screenshotNaturalWidth, displayedWidth);
const scaledY = scaleCoord(clickY, screenshotNaturalHeight, displayedHeight);
marker.style.left = `${scaledX}px`;
marker.style.top = `${scaledY}px`;
// Add ripple effect
const ripple = document.createElement("div");
ripple.className = "click-ripple";
marker.appendChild(ripple);
overlaysEl.appendChild(marker);
console.info("addOverlays: Added click marker at", { clickX, clickY, scaledX, scaledY });
// Set focal point for zoom (click position takes priority if no element)
if (focalX === undefined) {
focalX = clickX;
focalY = clickY;
}
}
}
// Add scroll indicator (doesn't need scaling - centered)
if (data.scrollDirection) {
const arrow = document.createElement("div");
arrow.className = `scroll-indicator scroll-${data.scrollDirection}`;
arrow.textContent = data.scrollDirection === "up" ? "↑" :
data.scrollDirection === "down" ? "↓" :
data.scrollDirection === "left" ? "←" : "→";
overlaysEl.appendChild(arrow);
}
// Always show full view (zoom disabled)
resetZoom();
delete containerEl.dataset.focalX;
delete containerEl.dataset.focalY;
});
}
/**
* Render screenshot and overlays
* Fetches screenshot via HTTP from localhost server (enabled by CSP connectDomains)
* This keeps base64 data out of AI context - only a small URL is passed
*/
function renderResult(data: ToolResultData) {
// Clear previous overlays
overlaysEl.innerHTML = "";
// Update action status immediately
const actionName = data.action || "unknown";
const statusIcon = data.success ? "✓" : "✗";
const statusClass = data.success ? "success" : "error";
let statusText = `${statusIcon} ${actionName}`;
if (data.duration) {
statusText += ` (${data.duration}ms)`;
}
if (data.assertion) {
statusText += `: "${data.assertion}"`;
}
if (data.text && data.action === "type") {
statusText += `: "${data.text}"`;
}
if (data.error) {
statusText += ` - ${data.error}`;
}
actionStatusEl.textContent = statusText;
actionStatusEl.className = statusClass;
// Store debugger URL from session_start
if (data.debuggerUrl) {
currentDebuggerUrl = data.debuggerUrl;
}
// Update session info with debugger link
if (data.session) {
const expiresIn = data.session.expiresIn ? Math.round(data.session.expiresIn / 1000) : 0;
sessionInfoEl.innerHTML = "";
if (currentDebuggerUrl) {
const link = document.createElement("a");
link.href = currentDebuggerUrl;
link.target = "_blank";
link.rel = "noopener noreferrer";
link.textContent = `${expiresIn}s remaining`;
link.className = "debugger-link";
link.title = `Open debugger: ${currentDebuggerUrl}`;
sessionInfoEl.appendChild(link);
} else {
sessionInfoEl.textContent = `${expiresIn}s remaining`;
}
sessionInfoEl.className = expiresIn < 30 ? "warning" : "";
} else if (currentDebuggerUrl) {
// No session data but we have a debugger URL - show it
sessionInfoEl.innerHTML = "";
const link = document.createElement("a");
link.href = currentDebuggerUrl;
link.target = "_blank";
link.rel = "noopener noreferrer";
link.textContent = "Open Debugger";
link.className = "debugger-link";
link.title = currentDebuggerUrl;
sessionInfoEl.appendChild(link);
} else if (data.action === "session_start") {
sessionInfoEl.textContent = "Session started";
}
// Update target info for find/find_and_click actions
if (data.element && (data.action === "find" || data.action === "find_and_click")) {
const el = data.element;
let targetHtml = `Target: "${el.description || "Element"}"`;
if (el.centerX !== undefined && el.centerY !== undefined) {
targetHtml += ` (${Math.round(el.centerX)}, ${Math.round(el.centerY)})`;
}
if (el.confidence !== undefined) {
const confidencePercent = Math.round(el.confidence * 100);
targetHtml += ` ${confidencePercent}%`;
}
targetInfoEl.innerHTML = targetHtml;
targetInfoEl.classList.remove("hidden");
} else {
targetInfoEl.classList.add("hidden");
}
// Load cropped image from find() response (data URL)
if (data.imageUrl) {
showLoading("Loading image...");
screenshotEl.onerror = () => {
console.error("Image failed to load");
screenshotEl.alt = "Image failed to load";
containerEl.style.display = "none";
hideLoading();
};
screenshotEl.onload = () => {
console.info("Image loaded:", screenshotEl.naturalWidth, "x", screenshotEl.naturalHeight);
// Store natural dimensions
screenshotNaturalWidth = screenshotEl.naturalWidth;
screenshotNaturalHeight = screenshotEl.naturalHeight;
// Show the container and add overlays now that we know dimensions
containerEl.style.display = "block";
addOverlays(data);
// Hide loading state
hideLoading();
};
screenshotEl.src = data.imageUrl;
screenshotEl.style.display = "block";
} else {
// No image available - just show status without visual
screenshotEl.style.display = "none";
containerEl.style.display = "none";
hideLoading();
}
}
// 1. Create app instance
const app = new App({ name: "TestDriver Screenshot", version: "1.0.0" });
/**
* Fetch screenshot blob from resource URI and convert to data URL
*/
async function fetchScreenshotFromResource(resourceUri: string): Promise {
try {
console.info("Fetching screenshot from resource:", resourceUri);
const result = await app.request(
{ method: "resources/read", params: { uri: resourceUri } },
ReadResourceResultSchema,
);
const content = result.contents[0];
if (!content || !("blob" in content)) {
console.error("Resource did not contain blob data");
return null;
}
// Convert base64 blob to data URL
const dataUrl = `data:${content.mimeType || "image/png"};base64,${content.blob}`;
console.info("Screenshot fetched successfully, blob length:", content.blob.length);
return dataUrl;
} catch (error) {
console.error("Failed to fetch screenshot resource:", error);
return null;
}
}
// 2. Register handlers BEFORE connecting
app.onteardown = async () => {
console.info("TestDriver app being torn down");
return {};
};
app.ontoolinput = (params) => {
console.info("Received tool input:", params);
const toolArgs = params.arguments;
// Build a readable summary from the arguments
const summaryParts: string[] = [];
if (toolArgs) {
// Show key params based on what's present
if (toolArgs.description) summaryParts.push(`"${toolArgs.description}"`);
if (toolArgs.text) summaryParts.push(`"${toolArgs.text}"`);
if (toolArgs.url) summaryParts.push(`${toolArgs.url}`);
if (toolArgs.direction) summaryParts.push(`${toolArgs.direction}`);
if (toolArgs.assertion) summaryParts.push(`"${toolArgs.assertion}"`);
if (toolArgs.task) summaryParts.push(`"${toolArgs.task}"`);
if (toolArgs.keys) summaryParts.push(`[${(toolArgs.keys as string[]).join("+")}]`);
if (toolArgs.type) summaryParts.push(`${toolArgs.type}`);
}
const actionSummary = summaryParts.length > 0 ? summaryParts.join(" ") : "action";
// Show loading state and hide screenshot (to avoid broken image during load)
actionStatusEl.textContent = `Running ${actionSummary}...`;
actionStatusEl.className = "loading";
containerEl.style.display = "none";
showLoading(`Running ${actionSummary}...`);
};
app.ontoolresult = async (result) => {
console.info("Received tool result:", result);
console.info("structuredContent:", result.structuredContent);
const data = extractData(result);
console.info("Extracted data keys:", Object.keys(data));
console.info("Has imageUrl:", !!data.imageUrl);
console.info("Has screenshotResourceUri:", !!data.screenshotResourceUri);
console.info("Has croppedImageResourceUri:", !!data.croppedImageResourceUri);
// If a screenshot or cropped image resource URI is provided, fetch the image from it
const resourceUri = data.screenshotResourceUri || data.croppedImageResourceUri;
if (resourceUri && !data.imageUrl) {
showLoading("Fetching image...");
const imageUrl = await fetchScreenshotFromResource(resourceUri);
if (imageUrl) {
data.imageUrl = imageUrl;
}
}
renderResult(data);
};
app.ontoolcancelled = (params) => {
console.info("Tool cancelled:", params.reason);
actionStatusEl.textContent = `Cancelled: ${params.reason}`;
actionStatusEl.className = "error";
hideLoading();
};
app.onerror = (error) => {
console.error("App error:", error);
actionStatusEl.textContent = `Error: ${error}`;
actionStatusEl.className = "error";
hideLoading();
};
app.onhostcontextchanged = handleHostContextChanged;
// 3. Connect to host
app.connect().then(() => {
const ctx = app.getHostContext();
if (ctx) {
handleHostContextChanged(ctx);
}
});
// Insert target info element after screenshot wrapper
const screenshotWrapper = document.querySelector(".screenshot-wrapper");
if (screenshotWrapper && screenshotWrapper.parentNode) {
screenshotWrapper.parentNode.insertBefore(targetInfoEl, screenshotWrapper.nextSibling);
}
// Zoom toggle disabled - always show full view