/** * virtual-cursor.ts — A visible, agent-controlled cursor overlay. * * The browser can't move the OS cursor (security), but it can render a fake * cursor on top of the page and dispatch real DOM events at its position. * This is actually BETTER than a native cursor for demos: * - Visible (the user sees what the agent's doing) * - Animatable (smooth sweeps, pulse highlights) * - Recordable (frame-by-frame) * - Lives in OUR DOM so we can draw trails, labels, scopes * * Complements dom.ts — dom_mutate click is invisible/instant; virtual_cursor * is visible/deliberate (good for demos, screen-recordings, tutorials, * autonomous workflows the user is watching). * * Tools: * vcursor_show — mount the overlay (once per page) * vcursor_hide — remove the overlay * vcursor_move — move to (x,y) or to a selector, optionally animated * vcursor_click — click at current position (real DOM event at target) * vcursor_type — type text into the currently-focused element * vcursor_highlight — pulse a ring at current position (demo emphasis) * vcursor_trail — enable/disable motion trail * vcursor_status — current position, visible, target element under cursor */ import { tool } from '@strands-agents/sdk' import { z } from 'zod' // ───── Constants + global registry ───── const OVERLAY_ID = 'careless-vcursor-overlay' const CURSOR_ID = 'careless-vcursor-cursor' declare global { interface Window { __carelessVCursors?: Map } } interface VCursorState { id: string overlay: HTMLDivElement cursor: SVGSVGElement label: HTMLDivElement x: number y: number color: string styleName: CursorStyleName trailEnabled: boolean trailPoints: Array<{ x: number; y: number; ts: number }> animationFrame: number | null } // Default palette for auto-spawned cursors (cycles through as you spawn). // 'default' always gets mint green; others get assigned next-in-ring. const DEFAULT_PALETTE = [ '#00e5a0', // mint (default) '#ff6b6b', // coral '#88ddff', // sky '#ffcc00', // gold '#c8a8ff', // lavender '#ff9f43', // orange '#52d681', // leaf '#ff5dc8', // pink ] function registry(): Map { if (!window.__carelessVCursors) { window.__carelessVCursors = new Map() } return window.__carelessVCursors } function state(id: string = 'default'): VCursorState | null { return registry().get(id) ?? null } function allStates(): VCursorState[] { return Array.from(registry().values()) } // Pick a color for a new cursor id. 'default' always mint; others cycle. function pickColorFor(id: string, override?: string): string { if (override) return override if (id === 'default') return DEFAULT_PALETTE[0] const used = new Set(allStates().map((s) => s.color)) // First unused palette slot for (const c of DEFAULT_PALETTE) { if (!used.has(c)) return c } // Palette exhausted — hash the id to a hue let h = 0 for (let i = 0; i < id.length; i++) h = (h * 31 + id.charCodeAt(i)) & 0xffff return `hsl(${h % 360}, 70%, 55%)` } // ───── Overlay mount / unmount ───── // ───── Cursor style variants ───── // Users/agents can swap the pointer glyph for demos. Each style is an SVG // fragment rendered inside the 28x28 viewBox. Colors can be overridden. type CursorStyleName = 'arrow' | 'hand' | 'crosshair' | 'thinking' | 'text' | 'forbidden' const CURSOR_STYLES: Record string> = { // Default rounded arrow arrow: (c) => ` `, // Pointing hand (for "click me") hand: (c) => ` `, // Crosshair (for "precision target") crosshair: (c) => ` `, // Thinking spinner (animated in CSS via rotating class) thinking: (c) => ` `, // I-beam (for "about to type") text: (c) => ` `, // No-entry / forbidden forbidden: (c) => ` `, } function applyCursorStyle(st: VCursorState, style: CursorStyleName, color: string) { st.styleName = style st.color = color st.cursor.innerHTML = CURSOR_STYLES[style](color) st.cursor.setAttribute('data-style', style) // Update label border + (if present) id tag background to the new color st.label.style.borderColor = color const tag = st.overlay.querySelector(`[data-cursor-tag="${st.id}"]`) if (tag) tag.style.background = color } function mountOverlay( id: string = 'default', opts: { color?: string; style?: CursorStyleName; x?: number; y?: number } = {} ): VCursorState { const existing = state(id) if (existing) { // Respect color/style overrides on re-mount if (opts.color && opts.color !== existing.color) { existing.color = opts.color applyCursorStyle(existing, existing.styleName, existing.color) } if (opts.style && opts.style !== existing.styleName) { existing.styleName = opts.style applyCursorStyle(existing, existing.styleName, existing.color) } return existing } const color = pickColorFor(id, opts.color) const styleName: CursorStyleName = opts.style ?? 'arrow' const overlay = document.createElement('div') overlay.id = `${OVERLAY_ID}-${id}` overlay.setAttribute('data-cursor-id', id) Object.assign(overlay.style, { position: 'fixed', top: '0', left: '0', width: '100vw', height: '100vh', pointerEvents: 'none', zIndex: '2147483646', }) document.body.appendChild(overlay) // Cursor glyph const cursor = document.createElementNS('http://www.w3.org/2000/svg', 'svg') cursor.id = `${CURSOR_ID}-${id}` cursor.setAttribute('data-cursor-id', id) cursor.setAttribute('width', '28') cursor.setAttribute('height', '28') cursor.setAttribute('viewBox', '0 0 28 28') Object.assign(cursor.style, { position: 'absolute', left: '0', top: '0', transform: 'translate(-50%, -50%)', filter: 'drop-shadow(0 2px 4px rgba(0,0,0,0.35))', transition: 'left 0.25s cubic-bezier(.4,0,.2,1), top 0.25s cubic-bezier(.4,0,.2,1)', }) cursor.innerHTML = CURSOR_STYLES[styleName](color) cursor.setAttribute('data-style', styleName) overlay.appendChild(cursor) // Floating label — uses the cursor color for the border so multi-cursor scenes // are visually distinguishable at a glance. const label = document.createElement('div') label.id = `${OVERLAY_ID}-${id}-label` Object.assign(label.style, { position: 'absolute', left: '0', top: '0', transform: 'translate(12px, -18px)', background: 'rgba(8, 22, 22, 0.88)', color: '#e8fff3', font: '500 11px/1.35 ui-sans-serif, system-ui, sans-serif', padding: '3px 8px', borderRadius: '6px', backdropFilter: 'blur(6px)', WebkitBackdropFilter: 'blur(6px)', border: `1px solid ${color}`, whiteSpace: 'nowrap', pointerEvents: 'none', opacity: '0', transition: 'opacity 0.2s, left 0.25s cubic-bezier(.4,0,.2,1), top 0.25s cubic-bezier(.4,0,.2,1)', }) overlay.appendChild(label) const st: VCursorState = { id, overlay, cursor, label, color, styleName, x: opts.x ?? window.innerWidth / 2, y: opts.y ?? window.innerHeight / 2, trailEnabled: false, trailPoints: [], animationFrame: null, } registry().set(id, st) setPos(st, st.x, st.y) // For non-default cursors, show an identity tag above the cursor permanently // so the user can tell which agent is which. if (id !== 'default') { const tag = document.createElement('div') tag.id = `${OVERLAY_ID}-${id}-tag` Object.assign(tag.style, { position: 'absolute', left: '0', top: '0', transform: 'translate(-50%, -36px)', background: color, color: '#081616', font: '600 10px/1 ui-sans-serif, system-ui, sans-serif', padding: '3px 6px', borderRadius: '4px', whiteSpace: 'nowrap', pointerEvents: 'none', letterSpacing: '0.02em', boxShadow: '0 2px 6px rgba(0,0,0,0.35)', transition: 'left 0.25s cubic-bezier(.4,0,.2,1), top 0.25s cubic-bezier(.4,0,.2,1)', }) tag.textContent = id tag.setAttribute('data-cursor-tag', id) overlay.appendChild(tag) } return st } function unmountOverlay(id: string = 'default') { const st = state(id) if (!st) return if (st.animationFrame != null) cancelAnimationFrame(st.animationFrame) st.overlay.remove() registry().delete(id) } function unmountAll() { for (const id of Array.from(registry().keys())) unmountOverlay(id) } function setPos(st: VCursorState, x: number, y: number) { st.x = x st.y = y st.cursor.style.left = `${x}px` st.cursor.style.top = `${y}px` st.label.style.left = `${x}px` st.label.style.top = `${y}px` const tag = st.overlay.querySelector(`[data-cursor-tag="${st.id}"]`) if (tag) { tag.style.left = `${x}px` tag.style.top = `${y}px` } if (st.trailEnabled) { st.trailPoints.push({ x, y, ts: Date.now() }) const cutoff = Date.now() - 2000 while (st.trailPoints.length > 40 || (st.trailPoints[0] && st.trailPoints[0].ts < cutoff)) { st.trailPoints.shift() } redrawTrail(st) } } function redrawTrail(st: VCursorState) { let trailSvg = st.overlay.querySelector('#careless-vcursor-trail') if (!trailSvg) { trailSvg = document.createElementNS('http://www.w3.org/2000/svg', 'svg') trailSvg.id = 'careless-vcursor-trail' Object.assign(trailSvg.style, { position: 'absolute', left: '0', top: '0', width: '100%', height: '100%', pointerEvents: 'none', }) st.overlay.insertBefore(trailSvg, st.cursor) } trailSvg.innerHTML = '' const pts = st.trailPoints if (pts.length < 2) return const d = pts.map((p, i) => `${i === 0 ? 'M' : 'L'} ${p.x} ${p.y}`).join(' ') const path = document.createElementNS('http://www.w3.org/2000/svg', 'path') path.setAttribute('d', d) path.setAttribute('fill', 'none') path.setAttribute('stroke', st.color) path.setAttribute('stroke-width', '2') path.setAttribute('stroke-linecap', 'round') path.setAttribute('stroke-linejoin', 'round') path.setAttribute('opacity', '0.45') trailSvg.appendChild(path) } function pulseRing(st: VCursorState, color?: string, radius: number = 24) { const ringColor = color ?? st.color const ring = document.createElement('div') Object.assign(ring.style, { position: 'absolute', left: `${st.x}px`, top: `${st.y}px`, width: `${radius * 2}px`, height: `${radius * 2}px`, borderRadius: '50%', border: `2px solid ${ringColor}`, transform: 'translate(-50%, -50%) scale(0.3)', opacity: '0.9', pointerEvents: 'none', transition: 'transform 0.6s cubic-bezier(.2,.9,.2,1), opacity 0.6s', }) st.overlay.appendChild(ring) requestAnimationFrame(() => { ring.style.transform = 'translate(-50%, -50%) scale(1.4)' ring.style.opacity = '0' }) setTimeout(() => ring.remove(), 700) } function setLabel(st: VCursorState, text: string | null, durationMs?: number) { if (!text) { st.label.style.opacity = '0' return } st.label.textContent = text st.label.style.opacity = '1' if (durationMs && durationMs > 0) { window.setTimeout(() => { st.label.style.opacity = '0' }, durationMs) } } // Move smoothly via rAF (for finer control than CSS transitions). function animateMove(st: VCursorState, toX: number, toY: number, durationMs: number): Promise { return new Promise((resolve) => { if (st.animationFrame != null) cancelAnimationFrame(st.animationFrame) const fromX = st.x const fromY = st.y const start = performance.now() // Temporarily disable CSS transition so rAF drives it const prevCursorTransition = st.cursor.style.transition const prevLabelTransition = st.label.style.transition st.cursor.style.transition = 'none' st.label.style.transition = 'none' const step = (now: number) => { const t = Math.min(1, (now - start) / durationMs) // Ease-in-out cubic for natural feel const e = t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2 const cx = fromX + (toX - fromX) * e const cy = fromY + (toY - fromY) * e setPos(st, cx, cy) if (t < 1) { st.animationFrame = requestAnimationFrame(step) } else { st.animationFrame = null st.cursor.style.transition = prevCursorTransition st.label.style.transition = prevLabelTransition resolve() } } st.animationFrame = requestAnimationFrame(step) }) } // Hit-test: find the element under the virtual cursor. function elementUnder(st: VCursorState): Element | null { // Hide overlay momentarily so elementFromPoint returns the real target. const prev = st.overlay.style.display st.overlay.style.display = 'none' const el = document.elementFromPoint(st.x, st.y) st.overlay.style.display = prev return el } function elementSummary(el: Element | null): Record | null { if (!el) return null const rect = (el as HTMLElement).getBoundingClientRect() return { tag: el.tagName.toLowerCase(), id: (el as HTMLElement).id || undefined, classes: typeof el.className === 'string' ? el.className.split(/\s+/).filter(Boolean) : undefined, text: (el.textContent || '').trim().slice(0, 120) || undefined, rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height }, } } // ───── Tools ───── export const vcursorShowTool = tool({ name: 'vcursor_show', description: 'Mount a virtual cursor overlay on the page. The cursor is visible, animatable, and agent-controlled. ' + 'Multi-cursor: pass a unique `cursor_id` to spawn additional cursors (each with its own color, trail, position). ' + 'Each non-default cursor gets a small identity tag above it so the user can tell sibling agents apart. ' + 'If the cursor_id already exists, the call is idempotent (updates color/style if provided).', inputSchema: z.object({ cursor_id: z.string().optional().describe('Unique id (default: "default"). Use "researcher", "coder", etc. to drive multiple cursors.'), x: z.number().optional().describe('Initial X (CSS px from viewport left). Default: center.'), y: z.number().optional().describe('Initial Y. Default: center.'), color: z.string().optional().describe('CSS color for this cursor. Default: auto-picked from palette so it stays distinct from other cursors.'), style: z.enum(['arrow', 'hand', 'crosshair', 'thinking', 'text', 'forbidden']).optional().describe('Initial glyph. Default: arrow.'), label: z.string().optional(), trail: z.boolean().optional().describe('Enable motion trail (default false).'), }), callback: (input) => { try { const id = input.cursor_id ?? 'default' const st = mountOverlay(id, { color: input.color, style: input.style, x: input.x, y: input.y, }) if (input.x !== undefined || input.y !== undefined) { setPos(st, input.x ?? st.x, input.y ?? st.y) } if (input.label) setLabel(st, input.label) if (input.trail !== undefined) st.trailEnabled = input.trail return JSON.stringify({ status: 'success', cursor_id: id, x: st.x, y: st.y, color: st.color, style: st.styleName, mounted: true, total_cursors: allStates().length, }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorHideTool = tool({ name: 'vcursor_hide', description: 'Unmount one or all virtual cursors. Pass cursor_id to hide a specific cursor, or all:true to clear all.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which cursor to hide. Default: "default".'), all: z.boolean().optional().describe('Hide all cursors (overrides cursor_id).'), }), callback: (input) => { try { if (input.all) { const ids = allStates().map((s) => s.id) unmountAll() return JSON.stringify({ status: 'success', unmounted: ids, remaining: 0 }) } const id = input.cursor_id ?? 'default' const existed = state(id) != null unmountOverlay(id) return JSON.stringify({ status: 'success', cursor_id: id, unmounted: existed, remaining: allStates().length, }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorMoveTool = tool({ name: 'vcursor_move', description: 'Move the virtual cursor. Either pass {x, y} in viewport px, or pass {selector} to move to an element\'s center. ' + 'Set animate:true for a smooth sweep (default) or animate:false for instant teleport. ' + 'Auto-mounts the overlay if not already shown.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), x: z.number().optional(), y: z.number().optional(), selector: z.string().optional().describe('Move to the CENTER of this element\'s bounding rect'), nth: z.number().optional().describe('Nth match if the selector has multiple'), animate: z.boolean().optional().describe('Default true'), duration_ms: z.number().optional().describe('Animation duration, default 500'), label: z.string().optional().describe('Show a floating label at the new position'), label_duration_ms: z.number().optional().describe('Auto-hide label after N ms. 0 = keep.'), scroll_into_view: z.boolean().optional().describe('If targeting selector, scroll it into view first (default true)'), }), callback: async (input) => { try { const st = mountOverlay(input.cursor_id) let tx: number | undefined = input.x let ty: number | undefined = input.y if (input.selector) { const nodes = Array.from(document.querySelectorAll(input.selector)) if (nodes.length === 0) { return JSON.stringify({ status: 'error', error: `No elements matched: ${input.selector}` }) } const el = nodes[input.nth ?? 0] as HTMLElement if (!el) return JSON.stringify({ status: 'error', error: `nth=${input.nth} out of range (${nodes.length})` }) if (input.scroll_into_view !== false) { el.scrollIntoView({ behavior: 'smooth', block: 'center', inline: 'center' }) // Let the scroll settle so rect reads are stable. await new Promise((r) => setTimeout(r, 250)) } const rect = el.getBoundingClientRect() tx = rect.left + rect.width / 2 ty = rect.top + rect.height / 2 } if (tx === undefined || ty === undefined) { return JSON.stringify({ status: 'error', error: 'Provide either {x, y} or {selector}' }) } if (input.label) setLabel(st, input.label, input.label_duration_ms ?? 2500) else setLabel(st, null) const animate = input.animate !== false if (animate) { await animateMove(st, tx, ty, input.duration_ms ?? 500) } else { setPos(st, tx, ty) } const under = elementUnder(st) return JSON.stringify({ status: 'success', x: st.x, y: st.y, animated: animate, duration_ms: animate ? (input.duration_ms ?? 500) : 0, under: elementSummary(under), }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorClickTool = tool({ name: 'vcursor_click', description: 'Click at the virtual cursor\'s current position. Dispatches a real MouseEvent on the element under the cursor ' + '(via elementFromPoint). Pulses a ring for visual feedback. Supports single/double click + modifier keys.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), button: z.enum(['left', 'middle', 'right']).optional().describe('Default left'), double: z.boolean().optional().describe('Fire dblclick instead of click'), ctrlKey: z.boolean().optional(), shiftKey: z.boolean().optional(), altKey: z.boolean().optional(), metaKey: z.boolean().optional(), }), callback: (input) => { try { const st = state(input.cursor_id) if (!st) return JSON.stringify({ status: 'error', error: 'Cursor not shown. Call vcursor_show first.' }) const target = elementUnder(st) if (!target) return JSON.stringify({ status: 'error', error: 'No element under cursor' }) const btn = input.button === 'middle' ? 1 : input.button === 'right' ? 2 : 0 const eventType = input.double ? 'dblclick' : 'click' const eventInit: MouseEventInit = { bubbles: true, cancelable: true, view: window, clientX: st.x, clientY: st.y, button: btn, buttons: 1 << btn, ctrlKey: !!input.ctrlKey, shiftKey: !!input.shiftKey, altKey: !!input.altKey, metaKey: !!input.metaKey, } // Fire mousedown + mouseup + click to mimic real interaction target.dispatchEvent(new MouseEvent('mousedown', eventInit)) target.dispatchEvent(new MouseEvent('mouseup', eventInit)) const clickEv = new MouseEvent(eventType, eventInit) const prevented = !target.dispatchEvent(clickEv) pulseRing(st, input.button === 'right' ? '#ff6b6b' : '#00e5a0') return JSON.stringify({ status: 'success', event: eventType, x: st.x, y: st.y, target: elementSummary(target), default_prevented: prevented, }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorTypeTool = tool({ name: 'vcursor_type', description: 'Type text into the element under the virtual cursor (must be input/textarea/contenteditable). ' + 'Uses the React-compatible native value setter + fires input/change events so frameworks notice.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), text: z.string(), per_char_delay_ms: z.number().optional().describe('Visible typing animation. Default 0 (instant).'), clear: z.boolean().optional().describe('Clear existing value before typing'), submit: z.boolean().optional().describe('Fire Enter keydown after typing'), }), callback: async (input) => { try { const st = state(input.cursor_id) if (!st) return JSON.stringify({ status: 'error', error: 'Cursor not shown. Call vcursor_show first.' }) let target = elementUnder(st) as HTMLElement | null if (!target) return JSON.stringify({ status: 'error', error: 'No element under cursor' }) // Climb if we hit a non-editable child of an input container let attempts = 0 while (target && attempts < 5 && !(target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement || (target as HTMLElement).isContentEditable)) { target = target.parentElement attempts++ } if (!target) return JSON.stringify({ status: 'error', error: 'No editable element near cursor' }) target.focus() const setInputValue = (el: HTMLInputElement | HTMLTextAreaElement, v: string) => { const proto = Object.getPrototypeOf(el) const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set if (setter) setter.call(el, v) else (el as unknown as { value: string }).value = v el.dispatchEvent(new Event('input', { bubbles: true })) } const isInput = target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement const current = isInput ? (target as HTMLInputElement).value : target.textContent || '' const base = input.clear ? '' : current if ((input.per_char_delay_ms ?? 0) > 0) { for (let i = 0; i < input.text.length; i++) { const chunk = base + input.text.slice(0, i + 1) if (isInput) { setInputValue(target as HTMLInputElement, chunk) } else { target.textContent = chunk target.dispatchEvent(new Event('input', { bubbles: true })) } await new Promise((r) => setTimeout(r, input.per_char_delay_ms)) } } else { const full = base + input.text if (isInput) { setInputValue(target as HTMLInputElement, full) } else { target.textContent = full target.dispatchEvent(new Event('input', { bubbles: true })) } } target.dispatchEvent(new Event('change', { bubbles: true })) if (input.submit) { target.dispatchEvent(new KeyboardEvent('keydown', { bubbles: true, cancelable: true, key: 'Enter', code: 'Enter', })) } return JSON.stringify({ status: 'success', typed_chars: input.text.length, target: elementSummary(target), submitted: !!input.submit, }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorHighlightTool = tool({ name: 'vcursor_highlight', description: 'Pulse a highlight ring at the cursor\'s current position (or at a selector). Visual emphasis during narration / demos. ' + 'Does NOT click.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), selector: z.string().optional().describe('If set, move cursor there first'), nth: z.number().optional(), color: z.string().optional().describe('CSS color, default mint green'), radius: z.number().optional().describe('Ring radius in px, default 24'), repeat: z.number().optional().describe('Pulse N times (default 1)'), interval_ms: z.number().optional().describe('Gap between pulses, default 280ms'), label: z.string().optional(), }), callback: async (input) => { try { const st = mountOverlay(input.cursor_id) if (input.selector) { const nodes = Array.from(document.querySelectorAll(input.selector)) const el = nodes[input.nth ?? 0] as HTMLElement if (!el) return JSON.stringify({ status: 'error', error: `No element for selector ${input.selector}` }) const rect = el.getBoundingClientRect() await animateMove(st, rect.left + rect.width / 2, rect.top + rect.height / 2, 350) } if (input.label) setLabel(st, input.label, 2000) const n = Math.max(1, input.repeat ?? 1) const gap = input.interval_ms ?? 280 for (let i = 0; i < n; i++) { pulseRing(st, input.color || '#00e5a0', input.radius ?? 24) if (i < n - 1) await new Promise((r) => setTimeout(r, gap)) } return JSON.stringify({ status: 'success', x: st.x, y: st.y, pulses: n }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorTrailTool = tool({ name: 'vcursor_trail', description: 'Enable/disable the motion trail behind the virtual cursor. Useful for demo recordings.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), enabled: z.boolean(), }), callback: (input) => { const st = state(input.cursor_id) if (!st) return JSON.stringify({ status: 'error', error: 'Cursor not shown' }) st.trailEnabled = input.enabled if (!input.enabled) { st.trailPoints = [] const trailSvg = st.overlay.querySelector('#careless-vcursor-trail') if (trailSvg) trailSvg.innerHTML = '' } return JSON.stringify({ status: 'success', trail_enabled: st.trailEnabled }) }, }) export const vcursorStatusTool = tool({ name: 'vcursor_status', description: 'Report the virtual cursor state: mounted, position, element under cursor, trail config.', inputSchema: z.object({}), callback: () => { const st = state() if (!st) { return JSON.stringify({ status: 'success', mounted: false }) } const under = elementUnder(st) return JSON.stringify({ status: 'success', mounted: true, x: st.x, y: st.y, trail_enabled: st.trailEnabled, trail_points: st.trailPoints.length, under: elementSummary(under), viewport: { w: window.innerWidth, h: window.innerHeight }, }) }, }) export const vcursorDragTool = tool({ name: 'vcursor_drag', description: 'Drag from the cursor\'s current position (or a start selector) to a destination. Dispatches a full ' + 'mousedown → mousemove(*N) → mouseup sequence plus pointer/drag events, so sliders, reordering, canvas ' + 'drawing, and drag-and-drop UIs all react. The cursor animates the path so the user sees it.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), from_selector: z.string().optional().describe('Start point. If omitted, drags from current cursor position.'), from_nth: z.number().optional(), to_x: z.number().optional().describe('Destination X in viewport px'), to_y: z.number().optional().describe('Destination Y'), to_selector: z.string().optional().describe('Destination element (center of bounding rect)'), to_nth: z.number().optional(), dx: z.number().optional().describe('Relative delta X (from current position). Overrides to_x/to_y.'), dy: z.number().optional().describe('Relative delta Y'), duration_ms: z.number().optional().describe('Total drag time, default 700'), steps: z.number().optional().describe('Intermediate mousemove count, default 24 (smooth). Higher = more granular events.'), button: z.enum(['left', 'middle', 'right']).optional(), hold_ms: z.number().optional().describe('Extra hold at the destination before releasing (for apps that need settle time). Default 0.'), use_pointer_events: z.boolean().optional().describe('Also fire pointerdown/pointermove/pointerup (default true)'), use_drag_events: z.boolean().optional().describe('Also fire dragstart/dragover/drop (for native HTML5 DnD). Default false — can interfere with mousemove-based UIs.'), }), callback: async (input) => { try { const st = mountOverlay(input.cursor_id) // Resolve start position if (input.from_selector) { const nodes = Array.from(document.querySelectorAll(input.from_selector)) const el = nodes[input.from_nth ?? 0] as HTMLElement | undefined if (!el) return JSON.stringify({ status: 'error', error: `No element matched from_selector: ${input.from_selector}` }) el.scrollIntoView({ behavior: 'smooth', block: 'center' }) await new Promise((r) => setTimeout(r, 200)) const r = el.getBoundingClientRect() setPos(st, r.left + r.width / 2, r.top + r.height / 2) } const startX = st.x const startY = st.y // Resolve end position let tx: number, ty: number if (input.dx !== undefined || input.dy !== undefined) { tx = startX + (input.dx ?? 0) ty = startY + (input.dy ?? 0) } else if (input.to_selector) { const nodes = Array.from(document.querySelectorAll(input.to_selector)) const el = nodes[input.to_nth ?? 0] as HTMLElement | undefined if (!el) return JSON.stringify({ status: 'error', error: `No element matched to_selector: ${input.to_selector}` }) const r = el.getBoundingClientRect() tx = r.left + r.width / 2 ty = r.top + r.height / 2 } else if (input.to_x !== undefined && input.to_y !== undefined) { tx = input.to_x ty = input.to_y } else { return JSON.stringify({ status: 'error', error: 'Provide dx/dy, to_x/to_y, or to_selector' }) } const btn = input.button === 'middle' ? 1 : input.button === 'right' ? 2 : 0 const startTarget = elementUnder(st) if (!startTarget) return JSON.stringify({ status: 'error', error: 'No element at drag start' }) const duration = Math.max(50, input.duration_ms ?? 700) const steps = Math.max(2, input.steps ?? 24) const usePointer = input.use_pointer_events !== false const useDrag = !!input.use_drag_events const fire = (el: Element, type: string, x: number, y: number) => { const init: MouseEventInit = { bubbles: true, cancelable: true, view: window, clientX: x, clientY: y, screenX: x, screenY: y, button: btn, buttons: 1 << btn, } el.dispatchEvent(new MouseEvent(type, init)) } const firePointer = (el: Element, type: string, x: number, y: number) => { if (!usePointer) return try { const init: PointerEventInit = { bubbles: true, cancelable: true, view: window, clientX: x, clientY: y, pointerType: 'mouse', isPrimary: true, button: btn, buttons: 1 << btn, } el.dispatchEvent(new PointerEvent(type, init)) } catch { /* older browsers: skip pointer events */ } } const fireDrag = (el: Element, type: string, x: number, y: number) => { if (!useDrag) return try { const ev = new DragEvent(type, { bubbles: true, cancelable: true, clientX: x, clientY: y }) el.dispatchEvent(ev) } catch { /* ignore */ } } // mousedown / pointerdown at start firePointer(startTarget, 'pointerdown', startX, startY) fire(startTarget, 'mousedown', startX, startY) fireDrag(startTarget, 'dragstart', startX, startY) pulseRing(st, '#ffcc00', 18) // Disable CSS transition during drag so rAF controls motion const prevCursorTransition = st.cursor.style.transition const prevLabelTransition = st.label.style.transition st.cursor.style.transition = 'none' st.label.style.transition = 'none' // Move in small steps, firing mousemove/pointermove each frame let lastTarget: Element | null = startTarget const stepDelay = duration / steps for (let i = 1; i <= steps; i++) { const t = i / steps // Ease-in-out cubic const e = t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2 const cx = startX + (tx - startX) * e const cy = startY + (ty - startY) * e setPos(st, cx, cy) const under = elementUnder(st) if (under) { firePointer(under, 'pointermove', cx, cy) fire(under, 'mousemove', cx, cy) if (useDrag) { fireDrag(under, 'dragover', cx, cy) if (under !== lastTarget) { if (lastTarget) fireDrag(lastTarget, 'dragleave', cx, cy) fireDrag(under, 'dragenter', cx, cy) } } lastTarget = under } // Also fire on the original target (some sliders track events on the thumb) firePointer(startTarget, 'pointermove', cx, cy) fire(startTarget, 'mousemove', cx, cy) if (i < steps) await new Promise((r) => setTimeout(r, stepDelay)) } st.cursor.style.transition = prevCursorTransition st.label.style.transition = prevLabelTransition if ((input.hold_ms ?? 0) > 0) await new Promise((r) => setTimeout(r, input.hold_ms)) // mouseup / pointerup at destination, plus drop const dropTarget = elementUnder(st) || lastTarget || startTarget firePointer(dropTarget, 'pointerup', tx, ty) fire(dropTarget, 'mouseup', tx, ty) if (useDrag) { fireDrag(dropTarget, 'drop', tx, ty) fireDrag(startTarget, 'dragend', tx, ty) } pulseRing(st, '#00e5a0', 22) return JSON.stringify({ status: 'success', from: { x: startX, y: startY, target: elementSummary(startTarget) }, to: { x: tx, y: ty, target: elementSummary(dropTarget) }, duration_ms: duration, steps, used_pointer_events: usePointer, used_drag_events: useDrag, }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorHoverTool = tool({ name: 'vcursor_hover', description: 'Hover over an element (or the element at the current cursor position). Dispatches the full mouseenter / ' + 'mouseover / pointermove sequence AND keeps firing mousemove at interval_ms for hold_ms so CSS :hover + JS ' + 'hover handlers reveal tooltips, submenus, hidden buttons. Returns when the hover period ends.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), selector: z.string().optional().describe('Element to hover. If omitted, uses current cursor position.'), nth: z.number().optional(), x: z.number().optional().describe('Hover at explicit (x,y) instead'), y: z.number().optional(), hold_ms: z.number().optional().describe('How long to hold the hover state, default 800'), interval_ms: z.number().optional().describe('Re-fire mousemove every N ms during hold (keeps :hover CSS active, default 120)'), animate_to: z.boolean().optional().describe('Animate the cursor to the target first (default true)'), }), callback: async (input) => { try { const st = mountOverlay(input.cursor_id) let tx = st.x let ty = st.y if (input.selector) { const nodes = Array.from(document.querySelectorAll(input.selector)) const el = nodes[input.nth ?? 0] as HTMLElement | undefined if (!el) return JSON.stringify({ status: 'error', error: `No element matched: ${input.selector}` }) el.scrollIntoView({ behavior: 'smooth', block: 'center' }) await new Promise((r) => setTimeout(r, 200)) const r = el.getBoundingClientRect() tx = r.left + r.width / 2 ty = r.top + r.height / 2 } else if (input.x !== undefined && input.y !== undefined) { tx = input.x ty = input.y } if (input.animate_to !== false && (tx !== st.x || ty !== st.y)) { await animateMove(st, tx, ty, 350) } else { setPos(st, tx, ty) } const target = elementUnder(st) if (!target) return JSON.stringify({ status: 'error', error: 'No element under target position' }) const fire = (el: Element, type: string, x: number, y: number) => { el.dispatchEvent(new MouseEvent(type, { bubbles: true, cancelable: true, view: window, clientX: x, clientY: y, relatedTarget: null, })) } const firePointer = (el: Element, type: string, x: number, y: number) => { try { el.dispatchEvent(new PointerEvent(type, { bubbles: true, cancelable: true, view: window, clientX: x, clientY: y, pointerType: 'mouse', isPrimary: true, })) } catch { /* ignore */ } } // Enter sequence — walk up from deepest so each ancestor gets mouseenter/pointerenter const chain: Element[] = [] let cur: Element | null = target while (cur && cur !== document.body) { chain.push(cur); cur = cur.parentElement } // mouseenter/pointerenter are non-bubbling → dispatch on each; mouseover bubbles for (const el of chain.slice().reverse()) { firePointer(el, 'pointerenter', tx, ty) fire(el, 'mouseenter', tx, ty) } fire(target, 'mouseover', tx, ty) firePointer(target, 'pointerover', tx, ty) const holdMs = input.hold_ms ?? 800 const tick = Math.max(30, input.interval_ms ?? 120) const start = performance.now() while (performance.now() - start < holdMs) { const under = elementUnder(st) || target fire(under, 'mousemove', tx, ty) firePointer(under, 'pointermove', tx, ty) await new Promise((r) => setTimeout(r, tick)) } pulseRing(st, '#88ddff', 20) return JSON.stringify({ status: 'success', x: tx, y: ty, hold_ms: holdMs, target: elementSummary(target), hovered_chain_depth: chain.length, }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorScrollTool = tool({ name: 'vcursor_scroll', description: 'Scroll the page or the scrollable container under the cursor. Fires real wheel events so infinite-scroll / ' + 'lazy-load UIs react. You can scroll by pixel delta, by viewport pages, to a target selector, or to the top/bottom. ' + 'Auto-detects the nearest scrollable ancestor if the element under the cursor isn\'t itself scrollable.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), dy: z.number().optional().describe('Vertical scroll delta in px (positive = down). Typical: 300-600 per "tick".'), dx: z.number().optional().describe('Horizontal delta in px'), pages: z.number().optional().describe('Scroll N viewport pages (positive=down). Overrides dy.'), to: z.enum(['top', 'bottom', 'left', 'right']).optional().describe('Scroll container to edge'), selector: z.string().optional().describe('Scroll a specific container (not auto-detected)'), nth: z.number().optional(), to_selector: z.string().optional().describe('Scroll until this element is in view (centered). Uses scrollIntoView.'), smooth: z.boolean().optional().describe('Use smooth behavior (default true)'), steps: z.number().optional().describe('For smooth simulation without native smooth: break into N steps'), wait_ms: z.number().optional().describe('Wait after scroll completes (lets lazy-load settle), default 150'), }), callback: async (input) => { try { const st = mountOverlay(input.cursor_id) // Resolve scroll container let container: Element | Window | null = null let containerEl: Element | null = null if (input.selector) { const nodes = Array.from(document.querySelectorAll(input.selector)) const el = nodes[input.nth ?? 0] as HTMLElement | undefined if (!el) return JSON.stringify({ status: 'error', error: `No element matched: ${input.selector}` }) container = el containerEl = el } else { // Find nearest scrollable ancestor of element under cursor let cur = elementUnder(st) as HTMLElement | null while (cur && cur !== document.body && cur !== document.documentElement) { const style = getComputedStyle(cur) const canScroll = (cur.scrollHeight > cur.clientHeight && /(auto|scroll|overlay)/.test(style.overflowY)) || (cur.scrollWidth > cur.clientWidth && /(auto|scroll|overlay)/.test(style.overflowX)) if (canScroll) { container = cur; containerEl = cur; break } cur = cur.parentElement } if (!container) { container = window containerEl = document.documentElement } } // to_selector — scrollIntoView if (input.to_selector) { const nodes = Array.from(document.querySelectorAll(input.to_selector)) const el = nodes[0] as HTMLElement | undefined if (!el) return JSON.stringify({ status: 'error', error: `No element matched to_selector: ${input.to_selector}` }) el.scrollIntoView({ behavior: input.smooth !== false ? 'smooth' : 'auto', block: 'center', inline: 'center' }) await new Promise((r) => setTimeout(r, input.wait_ms ?? 400)) const rect = el.getBoundingClientRect() return JSON.stringify({ status: 'success', action: 'scrolled_to_element', target: elementSummary(el), final_rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height }, }) } // Compute delta let dy = input.dy ?? 0 let dx = input.dx ?? 0 if (input.pages !== undefined) { const vh = containerEl === document.documentElement ? window.innerHeight : (containerEl as HTMLElement).clientHeight dy = input.pages * vh * 0.9 } if (input.to) { const el = containerEl as HTMLElement if (input.to === 'top') dy = -(container === window ? window.scrollY : el.scrollTop) else if (input.to === 'bottom') { const max = container === window ? document.documentElement.scrollHeight - window.innerHeight - window.scrollY : el.scrollHeight - el.clientHeight - el.scrollTop dy = max } else if (input.to === 'left') dx = -(container === window ? window.scrollX : el.scrollLeft) else if (input.to === 'right') { const max = container === window ? document.documentElement.scrollWidth - window.innerWidth - window.scrollX : el.scrollWidth - el.clientWidth - el.scrollLeft dx = max } } if (dy === 0 && dx === 0) { return JSON.stringify({ status: 'error', error: 'Provide dy/dx, pages, to, or to_selector' }) } // Fire wheel event at cursor position (for JS handlers that listen) const target = elementUnder(st) || document.body target.dispatchEvent(new WheelEvent('wheel', { bubbles: true, cancelable: true, view: window, clientX: st.x, clientY: st.y, deltaX: dx, deltaY: dy, deltaMode: 0, })) // Perform actual scroll const smooth = input.smooth !== false const steps = Math.max(1, input.steps ?? 1) if (steps === 1) { if (container === window) { window.scrollBy({ left: dx, top: dy, behavior: smooth ? 'smooth' : 'auto' }) } else { (container as Element).scrollBy({ left: dx, top: dy, behavior: smooth ? 'smooth' : 'auto' }) } } else { const sy = dy / steps const sx = dx / steps for (let i = 0; i < steps; i++) { if (container === window) window.scrollBy({ left: sx, top: sy, behavior: 'auto' }) else (container as Element).scrollBy({ left: sx, top: sy, behavior: 'auto' }) await new Promise((r) => setTimeout(r, 16)) } } await new Promise((r) => setTimeout(r, input.wait_ms ?? 150)) const finalY = container === window ? window.scrollY : (containerEl as HTMLElement).scrollTop const finalX = container === window ? window.scrollX : (containerEl as HTMLElement).scrollLeft return JSON.stringify({ status: 'success', container: container === window ? 'window' : elementSummary(containerEl), scrolled_by: { dx, dy }, final_scroll: { x: finalX, y: finalY }, smooth, }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorKeysTool = tool({ name: 'vcursor_keys', description: 'Send keyboard events to the element under the cursor (or document.activeElement). Supports single keys ' + '(Enter, Tab, Escape, ArrowDown, Backspace, PageDown, Home, End...), modifier combos (Ctrl+A, Cmd+K, Shift+Tab), ' + 'and key sequences. Fires keydown + keypress + keyup with correct key/code/keyCode so even legacy JS handlers fire.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), keys: z.union([z.string(), z.array(z.string())]).describe( 'Key name or combo ("Enter", "Tab", "Ctrl+A", "Cmd+K", "ArrowDown"), OR array of those for a sequence.' ), target_selector: z.string().optional().describe('Element to send keys to. Default: element under cursor, or document.activeElement.'), target_nth: z.number().optional(), per_key_delay_ms: z.number().optional().describe('Delay between keys in a sequence, default 60'), repeat: z.number().optional().describe('Send the (single) key N times. Ignored when keys is an array.'), hold_ms: z.number().optional().describe('For each key, delay between keydown and keyup (default 0 = same tick)'), }), callback: async (input) => { try { const st = state(input.cursor_id) // Resolve target let target: Element | null = null if (input.target_selector) { const nodes = Array.from(document.querySelectorAll(input.target_selector)) target = nodes[input.target_nth ?? 0] ?? null if (!target) return JSON.stringify({ status: 'error', error: `No element matched: ${input.target_selector}` }) } else if (st) { target = elementUnder(st) ?? document.activeElement } else { target = document.activeElement } if (!target) target = document.body // Focus it (some handlers only fire on focused element) if (target instanceof HTMLElement) { try { target.focus({ preventScroll: true }) } catch { /* noop */ } } // Map common key names to KeyboardEvent shape const keyMap: Record = { Enter: { key: 'Enter', code: 'Enter', keyCode: 13 }, Tab: { key: 'Tab', code: 'Tab', keyCode: 9 }, Escape: { key: 'Escape', code: 'Escape', keyCode: 27 }, Esc: { key: 'Escape', code: 'Escape', keyCode: 27 }, Backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 }, Delete: { key: 'Delete', code: 'Delete', keyCode: 46 }, Space: { key: ' ', code: 'Space', keyCode: 32 }, ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 }, ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 }, ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 }, ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', keyCode: 39 }, Home: { key: 'Home', code: 'Home', keyCode: 36 }, End: { key: 'End', code: 'End', keyCode: 35 }, PageUp: { key: 'PageUp', code: 'PageUp', keyCode: 33 }, PageDown: { key: 'PageDown', code: 'PageDown', keyCode: 34 }, } for (let i = 1; i <= 12; i++) { keyMap['F' + i] = { key: 'F' + i, code: 'F' + i, keyCode: 111 + i } } const parseKeyCombo = (combo: string) => { const parts = combo.split('+').map((p) => p.trim()).filter(Boolean) const mods = { ctrlKey: false, shiftKey: false, altKey: false, metaKey: false } let base = '' for (const p of parts) { const low = p.toLowerCase() if (low === 'ctrl' || low === 'control') mods.ctrlKey = true else if (low === 'shift') mods.shiftKey = true else if (low === 'alt' || low === 'option' || low === 'opt') mods.altKey = true else if (low === 'meta' || low === 'cmd' || low === 'command' || low === 'win') mods.metaKey = true else base = p } if (!base) throw new Error(`No base key in combo: ${combo}`) const mapped = keyMap[base] if (mapped) return { ...mods, ...mapped } // Single character if (base.length === 1) { const upper = base.toUpperCase() return { ...mods, key: mods.shiftKey ? upper : base.toLowerCase(), code: /^[a-zA-Z]$/.test(base) ? 'Key' + upper : 'Digit' + base, keyCode: upper.charCodeAt(0), } } // Unknown multi-char key — pass through as-is return { ...mods, key: base, code: base, keyCode: 0 } } const sendKey = async (combo: string) => { const spec = parseKeyCombo(combo) const init: KeyboardEventInit = { bubbles: true, cancelable: true, view: window, key: spec.key, code: spec.code, ctrlKey: spec.ctrlKey, shiftKey: spec.shiftKey, altKey: spec.altKey, metaKey: spec.metaKey, } // keyCode is deprecated but some legacy handlers need it const down = new KeyboardEvent('keydown', init) Object.defineProperty(down, 'keyCode', { get: () => spec.keyCode }) Object.defineProperty(down, 'which', { get: () => spec.keyCode }) const defaultPrevented = !target!.dispatchEvent(down) // keypress only for printable keys (legacy, some apps still listen) if (spec.key.length === 1 && !spec.ctrlKey && !spec.metaKey) { const press = new KeyboardEvent('keypress', init) Object.defineProperty(press, 'keyCode', { get: () => spec.keyCode }) target!.dispatchEvent(press) } if ((input.hold_ms ?? 0) > 0) await new Promise((r) => setTimeout(r, input.hold_ms)) const up = new KeyboardEvent('keyup', init) Object.defineProperty(up, 'keyCode', { get: () => spec.keyCode }) Object.defineProperty(up, 'which', { get: () => spec.keyCode }) target!.dispatchEvent(up) return defaultPrevented } const sequence = Array.isArray(input.keys) ? input.keys : (input.repeat ? Array(Math.max(1, input.repeat)).fill(input.keys) : [input.keys]) const delay = input.per_key_delay_ms ?? 60 const results: Array<{ key: string; default_prevented: boolean }> = [] for (let i = 0; i < sequence.length; i++) { const prevented = await sendKey(sequence[i]) results.push({ key: sequence[i], default_prevented: prevented }) if (i < sequence.length - 1) await new Promise((r) => setTimeout(r, delay)) } if (st) pulseRing(st, '#c8a8ff', 16) return JSON.stringify({ status: 'success', keys_sent: results, target: elementSummary(target), }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorStyleTool = tool({ name: 'vcursor_style', description: 'Swap the virtual cursor\'s visual style. Useful for demos: use "hand" before clicking, "thinking" during a long ' + 'operation, "text" before typing, "crosshair" when aiming precisely, "forbidden" to indicate blocked action.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), style: z.enum(['arrow', 'hand', 'crosshair', 'thinking', 'text', 'forbidden']), color: z.string().optional().describe('CSS color for the glyph (default #00e5a0)'), }), callback: (input) => { try { const st = mountOverlay(input.cursor_id) applyCursorStyle(st, input.style, input.color || '#00e5a0') return JSON.stringify({ status: 'success', style: input.style, color: input.color || '#00e5a0' }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorInspectTool = tool({ name: 'vcursor_inspect', description: 'Deep-read the element under the cursor (or by selector). Returns tag, id, classes, text, rect, computed visibility, ' + 'disabled state, aria attributes, input value, form context, and a suggested selector you can re-use. ' + 'Saves a hop vs. falling back to dom_query after every move.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), selector: z.string().optional().describe('Inspect this element instead of the one under the cursor'), nth: z.number().optional(), include_computed_style: z.boolean().optional().describe('Include selected computed CSS properties (default false — adds cost)'), style_props: z.array(z.string()).optional().describe('Which computed style properties to include (default: display, visibility, opacity, color, background-color, cursor)'), }), callback: (input) => { try { let target: Element | null = null if (input.selector) { const nodes = Array.from(document.querySelectorAll(input.selector)) target = (nodes[input.nth ?? 0] as Element) ?? null if (!target) return JSON.stringify({ status: 'error', error: `No element matched: ${input.selector}` }) } else { const st = state(input.cursor_id) if (!st) return JSON.stringify({ status: 'error', error: 'Cursor not shown and no selector provided' }) target = elementUnder(st) if (!target) return JSON.stringify({ status: 'error', error: 'No element under cursor' }) } const el = target as HTMLElement const rect = el.getBoundingClientRect() const style = getComputedStyle(el) const visible = rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none' && parseFloat(style.opacity) > 0.01 // Collect aria-* and data-* attributes const aria: Record = {} const data: Record = {} for (const attr of Array.from(el.attributes)) { if (attr.name.startsWith('aria-')) aria[attr.name] = attr.value else if (attr.name.startsWith('data-')) data[attr.name] = attr.value } // Form / input details let formInfo: Record | undefined if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement) { formInfo = { type: (el as HTMLInputElement).type, name: (el as HTMLInputElement).name, value: (el as HTMLInputElement).value, checked: (el as HTMLInputElement).checked, required: (el as HTMLInputElement).required, disabled: (el as HTMLInputElement).disabled, readonly: (el as HTMLInputElement).readOnly, placeholder: (el as HTMLInputElement).placeholder, form_id: el.form?.id, label: el.labels && el.labels.length > 0 ? el.labels[0].textContent?.trim() : undefined, } } else if (el instanceof HTMLButtonElement || el.getAttribute('role') === 'button') { formInfo = { type: (el as HTMLButtonElement).type, disabled: (el as HTMLButtonElement).disabled, text: el.textContent?.trim().slice(0, 200), } } // Suggest a stable selector: id > data-testid > unique class combo > tag + nth-of-type let suggestedSelector: string | undefined if (el.id) { suggestedSelector = `#${CSS.escape(el.id)}` } else if (el.getAttribute('data-testid')) { suggestedSelector = `[data-testid="${el.getAttribute('data-testid')}"]` } else if (el.getAttribute('aria-label')) { suggestedSelector = `[aria-label="${el.getAttribute('aria-label')}"]` } else if (typeof el.className === 'string' && el.className.trim()) { const classes = el.className.trim().split(/\s+/).filter(c => !c.includes(':')).slice(0, 3) if (classes.length > 0) suggestedSelector = el.tagName.toLowerCase() + '.' + classes.map(c => CSS.escape(c)).join('.') } const info: Record = { status: 'success', tag: el.tagName.toLowerCase(), id: el.id || undefined, classes: typeof el.className === 'string' ? el.className.split(/\s+/).filter(Boolean) : [], text: el.textContent?.trim().slice(0, 200) || undefined, href: el.getAttribute('href') || undefined, rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height }, visible, in_viewport: rect.bottom > 0 && rect.right > 0 && rect.top < window.innerHeight && rect.left < window.innerWidth, role: el.getAttribute('role') || undefined, tabindex: el.getAttribute('tabindex') || undefined, aria: Object.keys(aria).length > 0 ? aria : undefined, data: Object.keys(data).length > 0 ? data : undefined, form: formInfo, focused: document.activeElement === el, suggested_selector: suggestedSelector, } if (input.include_computed_style) { const props = input.style_props ?? ['display', 'visibility', 'opacity', 'color', 'background-color', 'cursor', 'pointer-events'] const computed: Record = {} for (const p of props) computed[p] = style.getPropertyValue(p) info.computed_style = computed } return JSON.stringify(info) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorWaitForStableTool = tool({ name: 'vcursor_wait_for_stable', description: 'Wait until a target element stops moving (e.g. dropdown finished animating in). Polls the bounding rect ' + 'every poll_ms and returns when N consecutive polls yield the same rect — OR times out. Use BEFORE a click ' + 'on an element that just animated in, to avoid clicking where it used to be.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), selector: z.string().describe('Element to watch'), nth: z.number().optional(), poll_ms: z.number().optional().describe('Poll interval, default 80'), stable_polls: z.number().optional().describe('Consecutive stable polls required, default 3'), timeout_ms: z.number().optional().describe('Max wait, default 3000'), tolerance_px: z.number().optional().describe('Allowed pixel drift between polls, default 1'), require_visible: z.boolean().optional().describe('Also wait for visibility (display/opacity), default true'), }), callback: async (input) => { try { const timeout = input.timeout_ms ?? 3000 const poll = Math.max(16, input.poll_ms ?? 80) const need = Math.max(1, input.stable_polls ?? 3) const tol = Math.max(0, input.tolerance_px ?? 1) const requireVisible = input.require_visible !== false const start = performance.now() let lastRect: DOMRect | null = null let stableCount = 0 let lastEl: Element | null = null const rectsEqual = (a: DOMRect, b: DOMRect) => Math.abs(a.x - b.x) <= tol && Math.abs(a.y - b.y) <= tol && Math.abs(a.width - b.width) <= tol && Math.abs(a.height - b.height) <= tol while (performance.now() - start < timeout) { const nodes = Array.from(document.querySelectorAll(input.selector)) const el = nodes[input.nth ?? 0] as HTMLElement | undefined if (!el) { await new Promise((r) => setTimeout(r, poll)) continue } lastEl = el const rect = el.getBoundingClientRect() const style = getComputedStyle(el) const visible = rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none' && parseFloat(style.opacity) > 0.01 if (requireVisible && !visible) { stableCount = 0 lastRect = null await new Promise((r) => setTimeout(r, poll)) continue } if (lastRect && rectsEqual(lastRect, rect)) { stableCount++ if (stableCount >= need) { return JSON.stringify({ status: 'success', stable: true, waited_ms: Math.round(performance.now() - start), rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height }, target: elementSummary(el), }) } } else { stableCount = 1 } lastRect = rect await new Promise((r) => setTimeout(r, poll)) } return JSON.stringify({ status: 'error', stable: false, error: `Timed out after ${timeout}ms without ${need} consecutive stable polls`, last_rect: lastRect ? { x: lastRect.x, y: lastRect.y, w: lastRect.width, h: lastRect.height } : null, last_target: elementSummary(lastEl), }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorScriptTool = tool({ name: 'vcursor_script', description: 'Run a sequence of virtual cursor actions as a single atomic macro. Each step is one of: ' + '{action:"move",...}, {action:"click",...}, {action:"type",...}, {action:"drag",...}, {action:"hover",...}, ' + '{action:"scroll",...}, {action:"keys",...}, {action:"highlight",...}, {action:"wait",ms:N}, ' + '{action:"wait_for",selector:"..."}, {action:"style",style:"..."}, {action:"label",text:"..."}. ' + 'Steps run sequentially; a failed step stops execution unless continue_on_error is true. Returns each step result.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which virtual cursor to act on. Default: "default". Use a unique id like "researcher" or "coder" to drive multiple cursors simultaneously (each with own color + trail + position).'), steps: z.array(z.record(z.string(), z.unknown())).describe('Array of step objects with an "action" field.'), continue_on_error: z.boolean().optional().describe('If true, failed steps are logged but execution continues'), label: z.string().optional().describe('Optional macro label shown as floating label at start'), }), callback: async (input) => { const st = mountOverlay(input.cursor_id) if (input.label) setLabel(st, input.label, 3000) const results: Array> = [] const runOne = async (step: Record): Promise> => { const action = step.action as string const args = { ...step } delete args.action // Propagate parent script's cursor_id if the step didn't specify one. // This lets you write [{action:'move',selector:'.foo'}, {action:'click'}] // once with a top-level cursor_id:'researcher' and have both hit that cursor. if (input.cursor_id && args.cursor_id === undefined) { args.cursor_id = input.cursor_id } const call = async (tool: typeof vcursorMoveTool, payload: unknown) => { const raw = await ((tool as any).callback as (x: unknown) => unknown | Promise)(payload) try { return JSON.parse(String(raw)) } catch { return { status: 'success', raw: String(raw) } } } switch (action) { case 'move': return call(vcursorMoveTool, args) case 'click': return call(vcursorClickTool, args) case 'type': return call(vcursorTypeTool, args) case 'highlight': return call(vcursorHighlightTool, args) case 'drag': return call(vcursorDragTool, args) case 'hover': return call(vcursorHoverTool, args) case 'scroll': return call(vcursorScrollTool, args) case 'keys': return call(vcursorKeysTool, args) case 'inspect': return call(vcursorInspectTool, args) case 'style': return call(vcursorStyleTool, args) case 'wait': { const ms = (args.ms as number) ?? 200 await new Promise((r) => setTimeout(r, ms)) return { status: 'success', action: 'wait', ms } } case 'wait_for': { return call(vcursorWaitForStableTool, args) } case 'label': { setLabel(st, (args.text as string) || null, (args.duration_ms as number) ?? 2000) return { status: 'success', action: 'label', text: args.text } } default: throw new Error(`Unknown action: ${action}`) } } for (let i = 0; i < input.steps.length; i++) { const step = input.steps[i] try { const res = await runOne(step) results.push({ step: i, action: step.action, ...res }) const status = (res as { status?: string }).status if (status === 'error' && !input.continue_on_error) { return JSON.stringify({ status: 'error', error: `Step ${i} (${step.action}) failed — stopping. Use continue_on_error:true to keep going.`, completed: i, total: input.steps.length, results, }) } } catch (err: unknown) { results.push({ step: i, action: step.action, status: 'error', error: (err as Error).message }) if (!input.continue_on_error) { return JSON.stringify({ status: 'error', error: `Step ${i} threw: ${(err as Error).message}`, completed: i, total: input.steps.length, results, }) } } } return JSON.stringify({ status: 'success', completed: input.steps.length, total: input.steps.length, results, }) }, }) export const vcursorListTool = tool({ name: 'vcursor_list', description: 'List all currently mounted virtual cursors. Returns each cursor\'s id, position, color, style, trail state, and ' + 'the element under it. Use BEFORE driving a named cursor to confirm it exists — or to see what your sibling ' + 'agents are up to if they\'re each driving their own cursor.', inputSchema: z.object({ include_under: z.boolean().optional().describe('Include the element under each cursor (default true)'), }), callback: (input) => { try { const cursors = allStates().map((st) => { const info: Record = { id: st.id, x: st.x, y: st.y, color: st.color, style: st.styleName, trail_enabled: st.trailEnabled, trail_points: st.trailPoints.length, } if (input.include_under !== false) { info.under = elementSummary(elementUnder(st)) } return info }) return JSON.stringify({ status: 'success', total: cursors.length, cursors, palette: DEFAULT_PALETTE, }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const vcursorColorTool = tool({ name: 'vcursor_color', description: 'Recolor an existing cursor (glyph + label border + id tag). Useful for marking state changes: e.g. turn the ' + 'researcher cursor red while it\'s waiting for a paused decision, then back to its palette color when it resumes.', inputSchema: z.object({ cursor_id: z.string().optional().describe('Which cursor to recolor. Default: "default".'), color: z.string().describe('New CSS color (e.g. "#ff6b6b", "tomato", "hsl(200, 80%, 60%)").'), }), callback: (input) => { try { const id = input.cursor_id ?? 'default' const st = state(id) if (!st) return JSON.stringify({ status: 'error', error: `No cursor with id: ${id}. Use vcursor_list to see available cursors.` }) applyCursorStyle(st, st.styleName, input.color) return JSON.stringify({ status: 'success', cursor_id: id, color: st.color, style: st.styleName, }) } catch (err: unknown) { return JSON.stringify({ status: 'error', error: (err as Error).message }) } }, }) export const VIRTUAL_CURSOR_TOOLS = [ vcursorShowTool, vcursorHideTool, vcursorMoveTool, vcursorClickTool, vcursorTypeTool, vcursorHighlightTool, vcursorTrailTool, vcursorStatusTool, vcursorDragTool, vcursorHoverTool, vcursorScrollTool, vcursorKeysTool, vcursorStyleTool, vcursorInspectTool, vcursorWaitForStableTool, vcursorScriptTool, vcursorListTool, vcursorColorTool, ]