export type MidsceneRecorderEventType = | 'click' | 'drag' | 'scroll' | 'input' | 'navigation' | 'setViewport' | 'keydown'; export type MidsceneRecorderSourceKind = | 'studio-preview' | 'unsupported' | (string & {}); export type MidsceneRecorderPlatformId = | 'web' | 'android' | 'ios' | 'computer' | 'harmony' | (string & {}); export interface MidsceneRecorderElementRect { left?: number; top?: number; width?: number; height?: number; x?: number; y?: number; } export interface MidsceneRecorderPageInfo { width: number; height: number; } /** * A screenshot stored outside the recording event payload. * * Recorder events are persisted in the Studio renderer. Keeping full data * URLs there makes long recordings retain every screenshot in the renderer * heap, so screenshot bytes live in the Playground run directory instead. */ export interface MidsceneRecorderScreenshotAssetRef { id: string; mimeType: string; bytes: number; } export interface MidsceneRecorderEvent { type: MidsceneRecorderEventType; source?: MidsceneRecorderSourceKind; actionType?: string; rawPayload?: Record; url?: string; title?: string; value?: string; elementRect?: MidsceneRecorderElementRect; pageInfo: MidsceneRecorderPageInfo; screenshotBefore?: string; screenshotAfter?: string; /** The single screenshot retained for AI description and Markdown export. */ screenshotAsset?: MidsceneRecorderScreenshotAssetRef; semantic?: MidsceneRecorderSemantic; elementDescription?: string; descriptionLoading?: boolean; screenshotWithBox?: string; timestamp: number; hashId: string; mergedHashIds?: string[]; } export type MidsceneRecorderSemanticSource = | 'aiDescribe' | 'recorderAI' | 'heuristic'; export type MidsceneRecorderSemanticStatus = 'pending' | 'ready' | 'failed'; export type MidsceneRecorderSemanticConfidence = 'high' | 'medium' | 'low'; export interface MidsceneRecorderSemanticAiDescribe { verifyPrompt: boolean; verifyPassed?: boolean; deepLocate?: boolean; centerDistance?: number; expectedCenter?: [number, number]; actualCenter?: [number, number]; annotatedScreenshotPath?: string; } export interface MidsceneRecorderSemantic { source: MidsceneRecorderSemanticSource; status: MidsceneRecorderSemanticStatus; elementDescription?: string; replayInstruction?: string; actionSummary?: string; confidence?: MidsceneRecorderSemanticConfidence; error?: string; aiDescribe?: MidsceneRecorderSemanticAiDescribe; fallbackFrom?: MidsceneRecorderSemantic; } export interface MidsceneRecorderSemanticAction { type: MidsceneRecorderEventType; actionType?: string; value?: string; url?: string; scrollDestinationDescription?: string; } export interface MidsceneRecorderTarget { platformId: MidsceneRecorderPlatformId; deviceId?: string; label?: string; values: Record; } export interface MidsceneRecorderGeneratedCode { markdown?: string; yaml?: string; playwright?: string; updatedAt?: number; } export interface MidsceneRecorderMarkdownScreenshotAsset { eventIndex: number; eventHashId: string; eventType: MidsceneRecorderEventType; relativePath: string; dataUrl: string; base64Data: string; mimeType: string; } export interface MidsceneRecorderMarkdownScreenshotOptions { baseDir?: string; maxScreenshots?: number; } export const DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS = 20; function isMidsceneRecorderPendingDescription(value?: string) { return value?.trim() === 'AI is analyzing element...'; } export function getMidsceneRecorderSemantic( event: Pick, ) { return event.semantic; } function getRecorderPointerActionVerb(actionType?: string) { switch (actionType) { case 'Tap': return 'Tap'; case 'DoubleClick': return 'Double click'; case 'LongPress': return 'Long press'; case 'RightClick': return 'Right click'; default: return 'Click'; } } function getRecorderDragActionVerb(actionType?: string) { switch (actionType) { case 'Swipe': return 'Swipe'; case 'DragAndDrop': return 'Drag'; default: return 'Drag'; } } export function buildMidsceneRecorderReplayInstruction( event: MidsceneRecorderSemanticAction, elementDescription: string, ) { switch (event.type) { case 'navigation': if (event.actionType === 'Stop') { return 'Stop loading the current page.'; } if (event.actionType === 'GoBack') { return 'Go back in the browser.'; } if (event.actionType === 'GoForward') { return 'Go forward in the browser.'; } if (event.actionType === 'Reload') { return 'Reload the current page.'; } if (event.actionType === 'NavigationChanged' && event.url) { return `Wait for navigation to complete at \`${event.url}\`.`; } return event.url ? `Navigate to \`${event.url}\`.` : `Navigate using ${elementDescription}.`; case 'scroll': return event.scrollDestinationDescription ? `Scroll the page/region with description "${elementDescription}" by value "${event.value || 'down'}" until "${event.scrollDestinationDescription}" is visible.` : `Scroll the page/region with description "${elementDescription}" by value "${event.value || 'down'}".`; case 'drag': { const verb = getRecorderDragActionVerb(event.actionType); return `${verb} through the area described as "${elementDescription}".`; } case 'input': return `Input "${event.value || ''}" into the element described as "${elementDescription}".`; case 'keydown': return `Press "${event.value || 'the recorded key'}" on the element described as "${elementDescription}".`; default: { const verb = getRecorderPointerActionVerb(event.actionType); if (verb === 'Long press') { return `${verb} the element described as "${elementDescription}".`; } return `${verb} on the element described as "${elementDescription}".`; } } } export function buildMidsceneRecorderActionSummary( event: MidsceneRecorderSemanticAction, elementDescription: string, ) { switch (event.type) { case 'navigation': if (event.actionType === 'Stop') { return 'Stop page loading'; } if (event.actionType === 'GoBack') { return 'Go back'; } if (event.actionType === 'GoForward') { return 'Go forward'; } if (event.actionType === 'Reload') { return 'Reload page'; } if (event.actionType === 'NavigationChanged' && event.url) { return `Wait for navigation to complete at ${event.url}`; } return event.url ? `Navigate to ${event.url}` : 'Navigate'; case 'scroll': return event.scrollDestinationDescription ? `Scroll ${elementDescription} toward ${event.scrollDestinationDescription}` : `Scroll ${elementDescription}`; case 'drag': return `${getRecorderDragActionVerb(event.actionType)} ${elementDescription}`; case 'input': return `Input into ${elementDescription}`; case 'keydown': return `Press ${event.value || 'key'} on ${elementDescription}`; default: return `${getRecorderPointerActionVerb(event.actionType)} ${elementDescription}`; } } export function getMidsceneRecorderEventDescription( event: MidsceneRecorderEvent, ) { const semantic = getMidsceneRecorderSemantic(event); if ( semantic?.actionSummary && !isMidsceneRecorderPendingDescription(semantic.actionSummary) ) { return semantic.actionSummary; } if ( semantic?.elementDescription && !isMidsceneRecorderPendingDescription(semantic.elementDescription) ) { return semantic.elementDescription; } if ( semantic?.replayInstruction && !isMidsceneRecorderPendingDescription(semantic.replayInstruction) ) { return semantic.replayInstruction; } if (event.type === 'navigation' && event.url) { return `Navigate to ${event.url}`; } if (event.value) { return event.actionType ? `${event.actionType} ${event.value}` : event.value; } if ( event.elementRect?.x !== undefined && event.elementRect?.y !== undefined ) { const prefix = event.actionType || event.type; return `${prefix} (${Math.round(event.elementRect.x)}, ${Math.round( event.elementRect.y, )})`; } return event.actionType || event.type; } export function getMidsceneRecorderScreenshotsForLLM( events: MidsceneRecorderEvent[], maxScreenshots = 1, ) { return selectRecorderScreenshotCandidates( getRecorderScreenshotCandidates(events), maxScreenshots, ).map((candidate) => candidate.screenshot); } export function sanitizeMidsceneRecorderFileName(value: string) { return ( value .trim() .replace(/[\\/:*?"<>|]+/g, '-') .replace(/\s+/g, '-') .replace(/-+/g, '-') .replace(/^-|-$/g, '') .toLowerCase() || 'midscene-recording' ); } function normalizeMarkdownAssetBaseDir(baseDir?: string) { const value = (baseDir || './screenshots').replace(/\/+$/g, ''); if (value.startsWith('./') || value.startsWith('../')) { return value; } return `./${value}`; } function padEventIndex(index: number) { return String(index + 1).padStart(3, '0'); } function parseScreenshotDataUrl(value: string): | { dataUrl: string; base64Data: string; mimeType: string; extension: string; } | undefined { const dataUrlMatch = value.match( /^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/, ); if (dataUrlMatch) { const mimeType = dataUrlMatch[1]; const extension = mimeType.includes('jpeg') ? 'jpg' : mimeType.split('/')[1]?.replace(/[^a-zA-Z0-9]/g, '') || 'png'; return { dataUrl: value, base64Data: dataUrlMatch[2], mimeType, extension, }; } if (/^[a-zA-Z0-9+/=\s]+$/.test(value) && value.trim().length > 0) { const base64Data = value.replace(/\s+/g, ''); return { dataUrl: `data:image/png;base64,${base64Data}`, base64Data, mimeType: 'image/png', extension: 'png', }; } return undefined; } function getRecorderEventScreenshot(event: MidsceneRecorderEvent) { return ( event.screenshotWithBox || event.screenshotAfter || event.screenshotBefore ); } function hasCoordinateFallback(event: MidsceneRecorderEvent) { const semantic = getMidsceneRecorderSemantic(event); return ( !semantic?.elementDescription && event.elementRect?.x !== undefined && event.elementRect?.y !== undefined ); } function shouldIncludeMarkdownScreenshot( event: MidsceneRecorderEvent, eventIndex: number, lastEventIndex: number, ) { const semantic = getMidsceneRecorderSemantic(event); return ( eventIndex === 0 || eventIndex === lastEventIndex || event.type === 'navigation' || event.type === 'scroll' || event.type === 'input' || Boolean(event.screenshotWithBox) || !semantic?.elementDescription || hasCoordinateFallback(event) ); } interface RecorderScreenshotCandidate { event: MidsceneRecorderEvent; eventIndex: number; screenshot: string; } function getRecorderScreenshotCandidatePriority( candidate: RecorderScreenshotCandidate, firstEventIndex: number, lastEventIndex: number, ) { const event = candidate.event; let priority = 0; if (candidate.eventIndex === firstEventIndex) { priority += 100; } if (candidate.eventIndex === lastEventIndex) { priority += 95; } if (event.type === 'navigation') { priority += 80; } if (event.screenshotWithBox) { priority += 70; } const semantic = getMidsceneRecorderSemantic(event); if ( semantic?.source === 'heuristic' || semantic?.confidence === 'low' || semantic?.error ) { priority += 60; } if (event.type === 'input' || event.type === 'scroll') { priority += 40; } if (!semantic?.elementDescription || hasCoordinateFallback(event)) { priority += 30; } return priority; } function selectEvenlyDistributedCandidates< T extends RecorderScreenshotCandidate, >(candidates: T[], count: number) { if (count <= 0) { return []; } if (candidates.length <= count) { return candidates; } if (count === 1) { return [candidates[Math.floor((candidates.length - 1) / 2)]]; } return Array.from({ length: count }, (_, index) => { const candidateIndex = Math.round( (index * (candidates.length - 1)) / (count - 1), ); return candidates[candidateIndex]; }); } function selectRecorderScreenshotCandidates< T extends RecorderScreenshotCandidate, >(candidates: T[], maxScreenshots: number): T[] { if (maxScreenshots <= 0 || candidates.length === 0) { return []; } if (candidates.length <= maxScreenshots) { return candidates; } const selected = new Map(); const firstEventIndex = candidates[0].eventIndex; const lastEventIndex = candidates[candidates.length - 1].eventIndex; const addCandidate = (candidate: T | undefined) => { if (!candidate || selected.size >= maxScreenshots) { return; } selected.set(candidate.eventIndex, candidate); }; const addEvenly = (pool: T[]) => { const remaining = maxScreenshots - selected.size; if (remaining <= 0) { return; } const unselected = pool.filter( (candidate) => !selected.has(candidate.eventIndex), ); for (const candidate of selectEvenlyDistributedCandidates( unselected, remaining, )) { addCandidate(candidate); } }; addCandidate(candidates[0]); addCandidate(candidates[candidates.length - 1]); addEvenly( candidates.filter( (candidate) => getRecorderScreenshotCandidatePriority( candidate, firstEventIndex, lastEventIndex, ) >= 60, ), ); addEvenly( candidates.filter( (candidate) => getRecorderScreenshotCandidatePriority( candidate, firstEventIndex, lastEventIndex, ) >= 40, ), ); addEvenly(candidates); return Array.from(selected.values()).sort( (left, right) => left.eventIndex - right.eventIndex, ); } function getRecorderScreenshotCandidates( events: MidsceneRecorderEvent[], ): RecorderScreenshotCandidate[] { const candidates: RecorderScreenshotCandidate[] = []; const seenScreenshots = new Set(); const lastEventIndex = events.length - 1; for (let eventIndex = 0; eventIndex < events.length; eventIndex += 1) { const event = events[eventIndex]; if (!shouldIncludeMarkdownScreenshot(event, eventIndex, lastEventIndex)) { continue; } const screenshot = getRecorderEventScreenshot(event); if (!screenshot || seenScreenshots.has(screenshot)) { continue; } seenScreenshots.add(screenshot); candidates.push({ event, eventIndex, screenshot, }); } return candidates; } export function createMidsceneRecorderMarkdownScreenshotAssets( events: MidsceneRecorderEvent[], options: MidsceneRecorderMarkdownScreenshotOptions = {}, ): MidsceneRecorderMarkdownScreenshotAsset[] { const baseDir = normalizeMarkdownAssetBaseDir(options.baseDir); const maxScreenshots = options.maxScreenshots ?? DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS; const candidates: Array< RecorderScreenshotCandidate & { parsedScreenshot: NonNullable>; } > = []; for (const candidate of getRecorderScreenshotCandidates(events)) { const parsedScreenshot = parseScreenshotDataUrl(candidate.screenshot); if (parsedScreenshot) { candidates.push({ ...candidate, parsedScreenshot, }); } } return selectRecorderScreenshotCandidates(candidates, maxScreenshots).map( ({ event, eventIndex, parsedScreenshot }) => { const safeType = event.type.replace(/[^a-zA-Z0-9-]/g, '-'); const fileName = `event-${padEventIndex(eventIndex)}-${safeType}.${parsedScreenshot.extension}`; return { eventIndex, eventHashId: event.hashId, eventType: event.type, relativePath: `${baseDir}/${fileName}`, dataUrl: parsedScreenshot.dataUrl, base64Data: parsedScreenshot.base64Data, mimeType: parsedScreenshot.mimeType, }; }, ); } function scalarToYaml(value: string | number | boolean) { return JSON.stringify(value); } export function stringifyMidsceneRecorderTargetBlock( target: MidsceneRecorderTarget, ) { const lines = [`${target.platformId}:`]; const values = Object.entries(target.values); if (values.length === 0) { lines.push(' {}'); return lines.join('\n'); } for (const [key, value] of values) { lines.push(` ${key}: ${scalarToYaml(value)}`); } return lines.join('\n'); }