/** * video-analyzer.ts * * Converts a screen-recording video into an ordered list of Playwright web steps. * * The analysis pipeline has five stages: * 1. Frame extraction — ffmpeg splits the video into PNG frames at the requested FPS. * 2. Deduplication — identical frames (same MD5 hash) are removed so the same screen * state is not processed twice. * 3. Motion scoring — consecutive PNG frames are compared pixel-by-pixel (sampling every * 4th pixel for speed) to produce a 0-1 motion score per frame. * 4. OCR — tesseract reads the on-screen text from action frames and every * 3rd frame so that field labels and button names can be identified. * 5. Step inference — heuristic rules map OCR text + motion score into typed WebSteps * (navigate / fill / click / scroll / assert). * * The resulting WebStep list is returned to the video-to-test MCP tool, which passes it * to the AI model to generate a Playwright test spec. * * External dependencies: * - ffmpeg — must be installed on the host (e.g. `brew install ffmpeg`). * - tesseract — must be installed on the host (e.g. `brew install tesseract`). * * Role in the overall flow: * pwpilot_video_to_test → VideoAnalyzer.analyze() → WebStep[] → code-generation prompt */ /** * Metadata and OCR text for a single extracted video frame. */ export interface VideoFrame { /** Absolute path to the PNG file on disk. */ framePath: string; /** Zero-based position of this frame in the deduplicated frame sequence. */ index: number; /** * Fraction of pixels that changed compared to the previous frame (0–1). * A score above 0.02 means the screen changed enough to indicate a user action. */ motionScore: number; /** Raw text extracted from the frame by tesseract OCR. Empty if OCR was skipped. */ ocrText: string; /** * True when this frame had significant motion or is the first/last frame. * Action frames are always OCR-processed and included in step inference. */ isActionFrame: boolean; } /** * A single inferred user action extracted from the video, ready to be turned * into a Playwright test step. */ export interface WebStep { /** Unique identifier for this step, e.g. "STEP-001". */ stepId: string; /** The type of Playwright action this step represents. */ action: 'navigate' | 'click' | 'fill' | 'check' | 'select' | 'scroll' | 'assert' | 'wait'; /** Human-readable description of what happens in this step. */ description: string; /** The target element label or URL (not set for scroll/assert steps). */ target?: string; /** The value to type or the URL to navigate to (only for fill/navigate steps). */ value?: string; /** * Suggested Playwright assertion strings that can be placed after this step * to verify the expected outcome (e.g. `expect(page.getByText('...')).toBeVisible()`). */ assertionHints: string[]; /** * The first 300 characters of OCR text from the frame where this step was detected. * Provides the AI model with page context when writing the test spec. */ ocrContext: string; /** * How confident the heuristic is that this is a real action (0–1). * Higher values mean more signal (e.g. URL in OCR = 0.85, button click = 0.78). */ confidence: number; } /** * The complete result of analyzing a video file. * Returned by VideoAnalyzer.analyze() and forwarded to the code-generation prompt. */ export interface VideoAnalysisResult { /** Absolute path to the original video file. */ videoPath: string; /** Directory where extracted PNG frames were saved. */ framesDir: string; /** Number of unique (deduplicated) frames that were analyzed. */ totalFrames: number; /** Number of frames classified as action frames (high motion or boundary frames). */ actionFrameCount: number; /** * Ordered list of inferred Playwright steps, one per detected user action. * This is the primary output used for test generation. */ webSteps: WebStep[]; /** * Raw OCR text for every frame that had non-empty OCR output. * Used by the AI model for additional context. */ screenTexts: string[]; /** * The 20 most frequently appearing text lines across all frames. * These often correspond to persistent UI elements like navigation labels and headings. */ dominantTexts: string[]; } /** * Analyzes a screen-recording video and infers an ordered sequence of Playwright * web steps (navigate, click, fill, scroll, assert) from its frames. * * Requires ffmpeg and tesseract to be installed on the host machine. */ export declare class VideoAnalyzer { /** * Run the full video analysis pipeline on a single recording. * * The five stages are: * 1. Extract PNG frames from the video using ffmpeg. * 2. Remove duplicate frames (identical MD5 hash). * 3. Compute per-frame motion scores by comparing consecutive PNG pixel data. * 4. Run tesseract OCR on action frames and every 3rd frame. * 5. Apply heuristics to classify frames into typed WebSteps. * * @param videoPath - Path to the video file (.mp4, .mov, .webm, etc.). * @param outDir - Directory where the extracted frames sub-folder will be created. * @param fps - How many frames per second to extract from the video. Defaults to 1. * Higher values produce more frames and more detail but increase processing time. * @returns A VideoAnalysisResult containing the inferred steps and supporting metadata. * @throws Error if the video file does not exist. * @throws Error if ffmpeg is not installed or fails. */ analyze(videoPath: string, outDir: string, fps?: number): Promise; /** * Use ffmpeg to extract individual PNG frames from a video file. * * Frames are written to `/frame_0001.png`, `frame_0002.png`, etc. * The `-y` flag overwrites any existing frames from a previous run. * * @param videoPath - Absolute path to the source video. * @param framesDir - Directory where the PNG frames will be written. * @param fps - Frames per second to extract. * @throws Error with an installation hint if ffmpeg is not found or fails. */ private extractFrames; /** * Remove duplicate frames from a list of PNG file paths. * * Reads each file, computes its MD5 hash, and discards any frame whose hash * was already seen. This eliminates long static sections of the video where * the screen does not change. * * @param paths - Sorted list of absolute PNG file paths. * @returns A filtered list with only unique frames (first occurrence kept). */ private deduplicate; /** * Compute a motion score for each frame relative to its predecessor. * * The score is the fraction of pixels that changed by more than a threshold * (combined R+G+B delta > 30). Every 4th pixel is sampled for speed. * If the two frames have different dimensions (e.g. after a window resize), * a score of 0.5 is assigned. * * The first frame always gets a score of 0 because there is no previous frame. * * @param paths - Ordered list of deduplicated PNG file paths. * @returns An array of motion scores (0–1), one per frame, in the same order. */ private scoreMotion; /** * Run tesseract OCR on a single PNG image and return the extracted text. * * Tesseract stderr output is suppressed (`2>/dev/null`) to avoid polluting * the MCP server log. If tesseract is not installed or fails for any reason, * an empty string is returned rather than throwing. * * @param imagePath - Absolute path to the PNG file to read. * @returns The OCR-extracted text, trimmed of leading/trailing whitespace. * Returns an empty string if OCR fails. */ private runOcr; /** * Classify action frames into typed Playwright WebSteps using heuristic rules. * * The rules are applied in priority order for each action frame: * 1. Navigate — a URL pattern appears in OCR that was not in the previous frame. * 2. Fill — a form-field label pattern appears and the frame has medium motion. * 3. Click — a known button label (Submit, Login, Save, etc.) appears with high motion. * 4. Scroll — very high motion but no form/button change; new content lines are visible. * 5. Assert — stable screen with at least 2 new text lines compared to the previous frame. * * Each step includes suggested assertion hints derived from the text that appeared * after the action so the AI model can write `expect(...)` calls. * * @param frames - The full list of VideoFrame objects (includes non-action frames). * @returns Ordered list of inferred WebSteps, one per detected user action. */ private buildWebSteps; /** * Identify the most frequently appearing text lines across all OCR-processed frames. * * Lines that appear often are likely persistent UI elements such as navigation * labels, headings, or footers. These are useful context for the AI model when * it writes assertions and descriptions. * * @param screenTexts - Array of OCR text strings, one per frame. * @returns The top 20 most frequent non-empty text lines, sorted by frequency descending. */ private extractDominantTexts; } //# sourceMappingURL=video-analyzer.d.ts.map