import type { AttachedBrowserBridge, AttachedBrowserDevice, BrowserAction, BrowserActionBatch, BrowserActionReceipt, BrowserDiagnosticBatch, BrowserDownload, BrowserDownloadSaveResponse, BrowserFrame, BrowserIdentity, BrowserObservation, BrowserRevision, BrowserSession, BrowserTarget, ComputerSession, InteractionPlacement, InteractionIntervention, InteractionSemanticNode, SiteAuthConnection, } from "@opengeni/sdk/interaction"; import { interactionControlFailureFromError } from "@opengeni/sdk/interaction"; import { BugIcon, ArchiveIcon, ArrowLeftIcon, ArrowRightIcon, CheckIcon, ChevronDownIcon, CircleAlertIcon, DownloadIcon, FileCheck2Icon, Globe2Icon, LoaderCircleIcon, MonitorIcon, PlusIcon, RefreshCwIcon, RotateCcwIcon, SaveIcon, UserRoundIcon, XIcon, ZapIcon, } from "lucide-react"; import { type FormEvent, type ClipboardEvent, type CompositionEvent, type KeyboardEvent, type MouseEvent, type PointerEvent, type ReactNode, type WheelEvent, useCallback, useEffect, useId, useLayoutEffect, useMemo, useRef, useState, } from "react"; import { useBrowserFrameStream, type BrowserFrameWebSocketFactory, } from "../hooks/use-browser-frame-stream"; import { useAttachedBrowsers } from "../hooks/use-attached-browsers"; import { useBrowserIdentities } from "../hooks/use-browser-identities"; import { useBrowserDownloads } from "../hooks/use-browser-downloads"; import { useBrowserSession, type BrowserFrameInputFence } from "../hooks/use-browser-session"; import { useBrowserSessions } from "../hooks/use-browser-sessions"; import { useInteractionInterventions } from "../hooks/use-interaction-interventions"; import { useSiteAuthConnections } from "../hooks/use-site-auth-connections"; import { cn } from "../lib/cn"; import { copyTextToClipboard } from "../lib/clipboard"; import { formatBytes } from "../lib/format"; import { isInteractionControlUnavailable, isSourcePlacementChangedError, } from "../lib/interaction-errors"; import { useClientConfigFlags } from "../hooks/use-client-config-flags"; import { type EmbeddedBrowserInteractionClientOverride, useEmbeddedBrowserInteraction, } from "../session-context"; import { browserKey, HUMAN_BROWSER_HOME_URL, normalizeBrowserAddress } from "./browser-input"; import { InteractionInterventionBanner } from "./interaction-intervention-banner"; import { BrowserSelectControl } from "./browser-select-control"; import { useViewerMenuDismiss } from "./use-viewer-menu-dismiss"; import { useOpenGeniText, type OpenGeniText } from "../messages/runtime"; /** Auto-ending a lost attached-device browser: bounded retries for transient failures only. */ const STALE_BROWSER_END_MAX_ATTEMPTS = 3; const STALE_BROWSER_END_RETRY_BASE_MS = 5_000; /** * Delay before retrying an automatic end of a lost attached-device browser, or null to stop. * A refusal (4xx, e.g. 409 when the server will not end it) is never retried: re-sending it on * every registry refresh looped about one request per second on prod. */ export function staleBrowserEndRetryDelayMs(cause: unknown, failedAttempts: number): number | null { const status = (cause as { status?: unknown } | null)?.status; if (typeof status === "number" && status >= 400 && status < 500) return null; if (failedAttempts >= STALE_BROWSER_END_MAX_ATTEMPTS) return null; return STALE_BROWSER_END_RETRY_BASE_MS * 2 ** (failedAttempts - 1); } export type BrowserViewerNotification = { kind: "error" | "info"; message: string; }; export type BrowserViewerProps = EmbeddedBrowserInteractionClientOverride & { /** The selected Opengeni agent/session. Peer BrowserSessions stay discoverable. */ sessionId: string; enabled?: boolean | undefined; /** Whether the Browser tab is selected. Keeps the mounted media stream warm * while pausing semantic reads behind other dock tabs. */ active?: boolean | undefined; className?: string | undefined; onNotify?: ((notification: BrowserViewerNotification) => void) | undefined; /** Tests/demos only. Production uses the browser's native WebSocket. */ webSocketFactory?: BrowserFrameWebSocketFactory | undefined; renderEmpty?: ((create: () => void, creating: boolean) => ReactNode) | undefined; /** Optional host capability for placing a human-created headed browser inside * an exact ComputerSession. Browser-only embedders still create a headed * browser; they simply omit the linked desktop resource. */ createLinkedComputer?: | (( name: string, placement?: InteractionPlacement, ) => Promise>) | undefined; /** Navigate to the exact linked ComputerSession; never a lookalike desktop. */ onOpenComputer?: ((computerSessionId: string) => void) | undefined; /** Restore and report the BrowserSession selected in this task's dock. */ initialBrowserSessionId?: string | null | undefined; onBrowserSessionIdChange?: ((browserSessionId: string | null) => void) | undefined; /** Host-owned install/setup surface for attaching an existing Chrome profile. * The machine bridge remains discoverable even when this URL is omitted. */ browserExtensionSetupUrl?: string | undefined; }; type BrowserSelection = { sessionId: string; pinned: boolean } | null; type BrowserLaunchChoice = | { kind: "clean" } | { kind: "profile"; identityId: string; baseRevisionId?: string } | { kind: "attached"; device: AttachedBrowserDevice }; // ArrayBuffer is opaque to React development prop diagnostics; a Uint8Array // exposes every image byte as an enumerable property retained in timing entries. type BrowserViewportFrame = Omit & { data: ArrayBuffer }; type PointerStart = { x: number; y: number; pointerId: number; frame: BrowserViewportFrame; }; type BrowserResumeAttempt = { operationId: string; running: boolean; terminal: boolean; error: Error | null; }; type BrowserDiagnosticsView = { browserSessionId: string; targetId: string; loading: boolean; batch: BrowserDiagnosticBatch | null; error: Error | null; }; /** * Complete browser-native session surface: workspace discovery, agent relevance, * browser/tab switching, live frames, semantic fallback, direct human input, * address navigation, diagnostics, and reconnect. Closing it never closes the browser. */ export function BrowserViewer({ sessionId, enabled = true, active = true, className, onNotify, webSocketFactory, renderEmpty, createLinkedComputer, onOpenComputer, initialBrowserSessionId, onBrowserSessionIdChange, browserExtensionSetupUrl, ...override }: BrowserViewerProps) { const t = useOpenGeniText(); const registry = useBrowserSessions({ ...override, sessionId, enabled }); const attached = useAttachedBrowsers({ ...override, enabled }); const { client: configClient } = useEmbeddedBrowserInteraction(override); // The deployment's own branded extension when it ships one. const browserExtensionName = useClientConfigFlags(configClient as never).browserExtension?.name; const onlineAttachedBridges = useMemo( () => attached.bridges.filter((bridge) => bridge.state === "online"), [attached.bridges], ); const profiles = useBrowserIdentities({ ...override, enabled, includeArchived: true }); const siteAuth = useSiteAuthConnections({ ...override, enabled: enabled && profiles.identities.length > 0, }); const createRegistryBrowser = registry.create; const refreshRegistry = registry.refresh; const loadProfileRevisions = profiles.revisions; const liveSessions = useMemo( () => registry.sessions.filter((session) => isLiveBrowser(session)), [registry.sessions], ); const attachedGenerationLoss = useMemo( () => attachedChromeGenerationLoss(registry.sessions), [registry.sessions], ); const endedGenerationLossRef = useRef(new Set()); const endStaleBrowserAttemptsRef = useRef(new Map()); const endStaleBrowser = registry.end; useEffect(() => { if (!enabled) return; for (const session of registry.sessions) { if ( session.lifecycle !== "lost" || session.failureCode !== "controller_transition_expired" || session.placement.kind !== "attached_device" ) { continue; } if (endedGenerationLossRef.current.has(session.id)) continue; endedGenerationLossRef.current.add(session.id); void endStaleBrowser(session.id).catch((cause: unknown) => { const attempts = (endStaleBrowserAttemptsRef.current.get(session.id) ?? 0) + 1; endStaleBrowserAttemptsRef.current.set(session.id, attempts); const delay = staleBrowserEndRetryDelayMs(cause, attempts); if (delay === null) return; setTimeout(() => endedGenerationLossRef.current.delete(session.id), delay); }); } }, [enabled, endStaleBrowser, registry.sessions]); const relevant = useMemo( () => registry.relevantSessions.filter((session) => isLiveBrowser(session)), [registry.relevantSessions], ); const latestRelevantBrowser = registry.relevantSessions[0]; const unavailableBrowser = latestRelevantBrowser?.lifecycle === "lost" || latestRelevantBrowser?.lifecycle === "failed" ? latestRelevantBrowser : undefined; const [selection, setSelection] = useState(() => initialBrowserSessionId ? { sessionId: initialBrowserSessionId, pinned: true } : null, ); const selectBrowserSession = useCallback( (next: BrowserSelection) => { setSelection(next); onBrowserSessionIdChange?.(next?.sessionId ?? null); }, [onBrowserSessionIdChange], ); const interventions = useInteractionInterventions({ ...override, enabled, resourceKind: "browser_session", }); const [creating, setCreating] = useState(false); const [savingProfile, setSavingProfile] = useState(false); const [baseRevisionOrdinal, setBaseRevisionOrdinal] = useState(null); const [profileHistory, setProfileHistory] = useState<{ identityId: string; loading: boolean; revisions: BrowserRevision[]; } | null>(null); const [resumeAttempt, setResumeAttempt] = useState<{ sessionId: string; attempt: BrowserResumeAttempt; } | null>(null); const resumeAttemptsRef = useRef(new Map()); const previousSessionIdRef = useRef(sessionId); const seenInterventionIdsRef = useRef(new Set()); const diagnosticRequestRef = useRef(0); const viewportFocusHandoffRef = useRef(null); const [diagnosticsView, setDiagnosticsView] = useState(null); const notifyError = useCallback( (cause: unknown, fallback: string) => { const message = cause instanceof Error ? cause.message : fallback; onNotify?.({ kind: "error", message }); }, [onNotify], ); useEffect(() => { if (previousSessionIdRef.current !== sessionId) { previousSessionIdRef.current = sessionId; setSelection( initialBrowserSessionId ? { sessionId: initialBrowserSessionId, pinned: true } : null, ); } }, [initialBrowserSessionId, sessionId]); useEffect(() => { if (!enabled || registry.loading) return; const selectedStillLive = liveSessions.some((session) => session.id === selection?.sessionId); // A manually selected workspace browser stays selected, but an unrelated // browser must never become the implicit browser for this agent. That made // a Modal session appear to own a connected Mac browser. if (selection?.pinned && selectedStillLive) return; const preferred = relevant[0] ?? null; if (!preferred) { if (selection) selectBrowserSession(null); return; } if (!selectedStillLive || !selection?.pinned) { if (selection?.sessionId !== preferred.id || selection.pinned) { selectBrowserSession({ sessionId: preferred.id, pinned: false }); } } }, [enabled, liveSessions, registry.loading, relevant, selectBrowserSession, selection]); const selectedRegistrySession = useMemo( () => liveSessions.find((session) => session.id === selection?.sessionId) ?? null, [liveSessions, selection?.sessionId], ); const interventionCounts = useMemo( () => countInterventions(interventions.interventions), [interventions.interventions], ); const selectedInterventions = useMemo( () => interventions.interventions.filter( (intervention) => intervention.resourceId === selection?.sessionId, ), [interventions.interventions, selection?.sessionId], ); const controllerReady = selectedRegistrySession?.lifecycle === "active"; const resumeSession = registry.resume; const [hasLiveFrame, setHasLiveFrame] = useState(false); const wakeBrowser = useCallback( async (session: BrowserSession, retry = false): Promise => { const current = resumeAttemptsRef.current.get(session.id); if (current?.running || (!retry && current)) return; const operationId = current && !current.terminal ? current.operationId : crypto.randomUUID(); const attempt: BrowserResumeAttempt = { operationId, running: true, terminal: false, error: null, }; resumeAttemptsRef.current.set(session.id, attempt); setResumeAttempt({ sessionId: session.id, attempt }); try { const response = await resumeSession(session.id, operationId); if (response.operation.state !== "completed") { throw Object.assign( new Error( response.operation.error?.message ?? t("browserViewer.browserCouldNotReopen2"), ), { terminal: true }, ); } resumeAttemptsRef.current.delete(session.id); setResumeAttempt((visible) => (visible?.sessionId === session.id ? null : visible)); } catch (cause) { const error = cause instanceof Error ? cause : new Error(String(cause)); const failed: BrowserResumeAttempt = { operationId, running: false, terminal: typeof cause === "object" && cause !== null && "terminal" in cause ? cause.terminal === true : false, error, }; resumeAttemptsRef.current.set(session.id, failed); setResumeAttempt({ sessionId: session.id, attempt: failed }); notifyError(error, t("browserViewer.couldNotReopenThisBrowser")); } }, [notifyError, resumeSession, t], ); useEffect(() => { if (!selectedRegistrySession) return; if (selectedRegistrySession.lifecycle === "active") { resumeAttemptsRef.current.delete(selectedRegistrySession.id); setResumeAttempt((visible) => visible?.sessionId === selectedRegistrySession.id ? null : visible, ); } }, [selectedRegistrySession]); const browser = useBrowserSession({ ...override, browserSessionId: selection?.sessionId ?? null, enabled: enabled && selection !== null && controllerReady, semanticObservationEnabled: active && !hasLiveFrame, }); useEffect(() => { if (!isSourcePlacementChangedError(browser.error, "browser_session")) return; void refreshRegistry(); }, [browser.error, refreshRegistry]); const downloads = useBrowserDownloads({ ...override, browserSessionId: selection?.sessionId ?? null, enabled: enabled && diagnosticsView !== null && controllerReady && browser.session?.capabilities.downloads === true, }); const loadBrowserDiagnostics = browser.diagnostics; useEffect(() => { for (const intervention of interventions.interventions) { if (seenInterventionIdsRef.current.has(intervention.id)) continue; seenInterventionIdsRef.current.add(intervention.id); onNotify?.({ kind: "info", message: `${interventionTitle(t, intervention)}: ${intervention.reason}`, }); } }, [interventions.interventions, onNotify, t]); const resolveIntervention = useCallback( (intervention: InteractionIntervention, outcome: "completed" | "dismissed") => { void interventions .resolve(intervention.id, { expectedVersion: intervention.version, outcome, }) .then(() => { onNotify?.({ kind: "info", message: outcome === "completed" ? t("browserViewer.agentNotifiedContinuingWork") : t("browserViewer.requestCancelled"), }); }) .catch((cause) => notifyError(cause, t("browserViewer.couldNotUpdateTheBrowser"))); }, [interventions, notifyError, onNotify, t], ); const frames = useBrowserFrameStream({ ...override, browserSessionId: selection?.sessionId ?? null, targetId: browser.selectedTarget?.id ?? null, enabled: enabled && selection !== null && controllerReady && browser.selectedTarget !== null && browser.session?.capabilities.liveFrames === true, stream: { format: "jpeg", quality: 76, maxWidth: 1_920, maxHeight: 1_200 }, ...(webSocketFactory ? { webSocketFactory } : {}), }); const receivedFrame = frameMatchesSelectedTarget( frames.frame, browser.session, browser.selectedTarget, ) ? frames.frame : null; const displayedFrame = useMemo( () => (receivedFrame ? { ...receivedFrame, data: receivedFrame.data.slice().buffer } : null), [receivedFrame], ); const frameIsLive = frames.state === "live" && displayedFrame !== null; useEffect(() => { setHasLiveFrame(frameIsLive); }, [frameIsLive]); const supportsLiveFrames = (browser.session ?? selectedRegistrySession)?.capabilities.liveFrames === true; const controlUnavailable = isInteractionControlUnavailable(browser.error); const connectionError = controlUnavailable ? browser.error : (frames.error ?? browser.error); // A managed controller can reject a stale attachment while the same browser // remains healthy. Only extension-attached Chrome requires a new browser on // connection-generation loss; an unrelated lost Chrome must not poison the // selected managed browser's recovery UI. const selectedPlacement = (browser.session ?? selectedRegistrySession)?.placement; const selectedChromeGenerationLoss = selectedPlacement?.kind === "attached_device" && isAttachedChromeGenerationLossError(connectionError); const replacementChromeDevice = selectedChromeGenerationLoss && selectedPlacement.kind === "attached_device" ? attached.devices.find((candidate) => candidate.id === selectedPlacement.deviceId) : null; const displayConnectionState = connectionError && (!frameIsLive || controlUnavailable) ? "error" : !browser.session ? "connecting" : supportsLiveFrames ? frames.state === "live" && !displayedFrame ? "connecting" : frames.state : "semantic"; const selectedProfile = useMemo( () => profiles.identities.find( (identity) => identity.id === (browser.session?.identityId ?? selectedRegistrySession?.identityId), ) ?? null, [browser.session?.identityId, profiles.identities, selectedRegistrySession?.identityId], ); const selectedProfileAuth = useMemo( () => selectedProfile ? siteAuth.connections.filter( (connection) => connection.preferredIdentityId === selectedProfile.id, ) : [], [selectedProfile, siteAuth.connections], ); useEffect(() => { diagnosticRequestRef.current += 1; setDiagnosticsView(null); }, [browser.session?.id, browser.selectedTarget?.id]); const loadDiagnostics = useCallback(() => { const browserSessionId = browser.session?.id; const targetId = browser.selectedTarget?.id; if (!browserSessionId || !targetId) return; const requestId = diagnosticRequestRef.current + 1; diagnosticRequestRef.current = requestId; setDiagnosticsView((current) => ({ browserSessionId, targetId, loading: true, batch: current?.browserSessionId === browserSessionId && current.targetId === targetId ? current.batch : null, error: null, })); void loadBrowserDiagnostics({ limit: 100 }) .then((batch) => { if (diagnosticRequestRef.current !== requestId) return; setDiagnosticsView({ browserSessionId, targetId, loading: false, batch, error: null }); }) .catch((cause) => { if (diagnosticRequestRef.current !== requestId) return; setDiagnosticsView({ browserSessionId, targetId, loading: false, batch: null, error: cause instanceof Error ? cause : new Error(String(cause)), }); }); }, [browser.selectedTarget?.id, browser.session?.id, loadBrowserDiagnostics]); const closeDiagnostics = useCallback(() => { diagnosticRequestRef.current += 1; setDiagnosticsView(null); }, []); const refreshDiagnostics = useCallback(() => { loadDiagnostics(); if (browser.session?.capabilities.downloads) void downloads.refresh(); }, [browser.session?.capabilities.downloads, downloads, loadDiagnostics]); const saveDownload = useCallback( async ( downloadId: string, destinationPath: string, overwrite: boolean, ): Promise => { const response = await downloads.saveToWorkspace(downloadId, destinationPath, { overwrite }); onNotify?.({ kind: "info", message: t("browserViewer.filenameSavedToDestinationpath", { filename: response.download.filename, destinationPath: response.destinationPath, }), }); return response; }, [downloads, onNotify, t], ); useEffect(() => { const identityId = browser.session?.identityId ?? selectedRegistrySession?.identityId; const revisionId = browser.session?.baseRevisionId ?? selectedRegistrySession?.baseRevisionId; if (!identityId) { setBaseRevisionOrdinal(null); setProfileHistory(null); return; } let disposed = false; setBaseRevisionOrdinal(null); setProfileHistory({ identityId, loading: true, revisions: [] }); void loadProfileRevisions(identityId) .then((response) => { if (!disposed) { setBaseRevisionOrdinal( revisionId ? (response.revisions.find((revision) => revision.id === revisionId)?.ordinal ?? null) : null, ); setProfileHistory({ identityId, loading: false, revisions: response.revisions }); } }) .catch(() => { if (!disposed) setProfileHistory({ identityId, loading: false, revisions: [] }); }); return () => { disposed = true; }; }, [ browser.session?.baseRevisionId, browser.session?.identityId, loadProfileRevisions, selectedRegistrySession?.baseRevisionId, selectedRegistrySession?.identityId, ]); const createBrowser = useCallback( (choice: BrowserLaunchChoice = { kind: "clean" }) => { if (creating) return; const identity = choice.kind === "profile" ? profiles.identities.find((candidate) => candidate.id === choice.identityId) : null; if (choice.kind === "profile" && (!identity || identity.status !== "active")) { onNotify?.({ kind: "error", message: t("browserViewer.thisBrowserProfileIsNo"), }); return; } const device = choice.kind === "attached" ? choice.device : null; const browserName = device?.profileLabel ?? device?.name ?? (identity ? `${identity.name} browser` : t("browserViewer.browser")); setCreating(true); void (async () => { let linkedComputer: Pick | null = null; let computerViewUnavailable = false; if (createLinkedComputer) { try { linkedComputer = await createLinkedComputer( `${browserName} desktop`, device ? { kind: "attached_device", deviceId: device.id } : undefined, ); } catch { // Browser control is independently useful on locked/headless placements. // A linked ComputerSession is an optional visual enhancement and must not // prevent the canonical BrowserSession from starting. computerViewUnavailable = true; } } const response = await createRegistryBrowser({ sessionId, name: browserName, // Human-created browsers are always ordinary visual Chromium/Chrome. // Lightpanda remains an agent-only optimization requested explicitly // through browser_open; if an agent creates one it is still visible in // the shared BrowserSession switcher. headless: false, initialUrl: HUMAN_BROWSER_HOME_URL, ...(device ? { placement: { kind: "attached_device" as const, deviceId: device.id, }, } : {}), ...(linkedComputer ? { linkedComputerSessionId: linkedComputer.id, placement: linkedComputer.placement, } : {}), ...(identity ? { identityId: identity.id, ...(choice.kind === "profile" && choice.baseRevisionId ? { baseRevisionId: choice.baseRevisionId } : {}), } : {}), }); selectBrowserSession({ sessionId: response.session.id, pinned: true }); if (computerViewUnavailable) { onNotify?.({ kind: "info", message: t("browserViewer.browserOpenedDesktopViewIs"), }); } })() .catch((cause) => notifyError(cause, t("browserViewer.couldNotOpenABrowser"))) .finally(() => setCreating(false)); }, [ createLinkedComputer, createRegistryBrowser, creating, notifyError, onNotify, selectBrowserSession, profiles.identities, sessionId, t, ], ); const saveProfileVersion = useCallback( async (newProfileName?: string): Promise => { const session = browser.session; if (!session || savingProfile || !session.capabilities.identityPublication) return false; setSavingProfile(true); try { let identity = selectedProfile; if (!identity) { const name = newProfileName?.trim() ?? ""; if (!name) return false; // First-save is intentionally two-phase: the named identity exists // before its first immutable revision. If capture/upload failed after // that create, retry into the same empty identity instead of turning // the recoverable draft into a permanent name conflict. identity = profiles.identities.find( (candidate) => candidate.status === "active" && candidate.revisionCount === 0 && candidate.name.localeCompare(name, undefined, { sensitivity: "base" }) === 0, ) ?? (await profiles.create({ name })).identity; } const response = await profiles.publish(session.id, { identityId: identity.id, expectedHeadGeneration: identity.headGeneration, advanceDefault: true, }); setBaseRevisionOrdinal(response.revision.ordinal); setProfileHistory((current) => ({ identityId: response.identity.id, loading: false, revisions: current?.identityId === response.identity.id ? [ ...current.revisions.filter((revision) => revision.id !== response.revision.id), response.revision, ] : [response.revision], })); await Promise.all([browser.refresh(), registry.refresh()]); onNotify?.({ kind: "info", message: response.outcome === "saved_as_default" ? t("browserViewer.nameVersionOrdinalSavedFor", { name: response.identity.name, ordinal: response.revision.ordinal, }) : t("browserViewer.nameVersionOrdinalSavedAnother", { name: response.identity.name, ordinal: response.revision.ordinal, }), }); return true; } catch (cause) { notifyError(cause, t("browserViewer.couldNotSaveThisBrowser")); return false; } finally { setSavingProfile(false); } }, [browser, notifyError, onNotify, profiles, registry, savingProfile, selectedProfile, t], ); const updateProfile = useCallback( async ( identity: BrowserIdentity, update: { name?: string; status?: "active" | "archived"; defaultRevisionId?: string }, ): Promise => { if (savingProfile) return false; setSavingProfile(true); try { const response = await profiles.update(identity.id, { expectedVersion: identity.version, ...update, }); if (update.defaultRevisionId) { const ordinal = profileHistory?.revisions.find( (revision) => revision.id === update.defaultRevisionId, )?.ordinal; onNotify?.({ kind: "info", message: ordinal ? t("browserViewer.nameVersionOpensByDefault", { name: response.identity.name, ordinal, }) : t("browserViewer.nameOpensByDefault", { name: response.identity.name }), }); } else if (update.status === "archived") { onNotify?.({ kind: "info", message: t("browserViewer.nameHiddenFromNewBrowsers", { name: response.identity.name }), }); } else if (update.status === "active") { onNotify?.({ kind: "info", message: `${response.identity.name} restored.` }); } return true; } catch (cause) { notifyError(cause, t("browserViewer.couldNotUpdateThisBrowser")); await profiles.refresh().catch(() => undefined); return false; } finally { setSavingProfile(false); } }, [notifyError, onNotify, profileHistory?.revisions, profiles, savingProfile, t], ); if (!enabled) return null; if (registry.loading && liveSessions.length === 0) { return ( } text={t("browserViewer.findingBrowsers")} {...(className ? { className } : {})} /> ); } if (liveSessions.length === 0) { if (renderEmpty && !unavailableBrowser) return <>{renderEmpty(() => createBrowser(), creating)}; return (
{unavailableBrowser && !attachedGenerationLoss ? ( ) : ( <>

{attachedGenerationLoss ? t("browserViewer.chromeReconnectedOpenAFresh") : t("browserViewer.noBrowserOpen")}

{attachedGenerationLoss ? t("browserViewer.thisChromeProfileIsLive") : t("browserViewer.aBrowserAppearsHereWhen")}

)} updateProfile(identity, { status: "active" })} setupUrl={browserExtensionSetupUrl} extensionName={browserExtensionName} prominent /> {registry.error ? (

{registry.error.message}

) : null}
); } return (
session.id))} selectedSessionId={selection?.sessionId ?? null} identities={profiles.identities} attachedBridges={onlineAttachedBridges} attachedDevices={attached.devices} selectedProfile={selectedProfile} baseRevisionOrdinal={baseRevisionOrdinal} profileHistory={profileHistory} authConnections={selectedProfileAuth} creating={creating} savingProfile={savingProfile} refreshing={registry.refreshing || attached.refreshing} interventionCounts={interventionCounts} onSelect={(browserSessionId) => selectBrowserSession({ sessionId: browserSessionId, pinned: true }) } onFollow={() => { const preferred = relevant[0]; if (preferred) selectBrowserSession({ sessionId: preferred.id, pinned: false }); }} onCreate={createBrowser} onSaveProfile={saveProfileVersion} onUpdateProfile={updateProfile} onOpenProfileVersion={(identityId, baseRevisionId) => createBrowser({ kind: "profile", identityId, baseRevisionId }) } onRefresh={() => void Promise.all([registry.refresh(), attached.refresh()])} setupUrl={browserExtensionSetupUrl} extensionName={browserExtensionName} /> void browser .selectTarget(intervention.targetId) .catch((cause) => notifyError(cause, t("browserViewer.couldNotOpenTheRequested"))) } onResolve={resolveIntervention} /> {!selectedRegistrySession ? ( createBrowser()} /> ) : !controllerReady ? ( void wakeBrowser(selectedRegistrySession, true)} /> ) : ( <> void browser .selectTarget(targetId) .catch((cause) => notifyError(cause, t("browserViewer.couldNotSwitchTabs"))) } onClose={(targetId) => void browser .closeTarget(targetId) .catch((cause) => notifyError(cause, t("browserViewer.couldNotCloseTheTab"))) } onOpen={() => void browser .openTarget(HUMAN_BROWSER_HOME_URL) .catch((cause) => notifyError(cause, t("browserViewer.couldNotOpenATab"))) } /> void browser .act({ type: "navigate", url }) .catch((cause) => notifyError(cause, t("browserViewer.couldNotNavigate"))) } onHistory={async (direction) => { try { const action = { type: "history", direction } as const; if (receivedFrame) await browser.actFromFrame(action, receivedFrame); else { if (!browser.observation) throw new Error("The browser page is not ready for input."); await browser.actFromObservation(action, browser.observation); } } catch (cause) { notifyError( cause, direction === "back" ? t("browserViewer.couldNotGoBack") : t("browserViewer.couldNotGoForward"), ); } }} onReload={() => { const url = browser.selectedTarget?.url; if (url) void browser .act({ type: "navigate", url }) .catch((cause) => notifyError(cause, t("browserViewer.couldNotReload"))); }} /> {browser.inputFailure ? (
{browser.inputFailure.state === "failed" ? t("browserViewer.browserInputFailed") : t("browserViewer.inputResultUnknown")}

{browser.inputFailure.error?.message ?? t("browserViewer.inspectThePageBeforeContinuing")}

) : null}
{ const focusedInput = frame && action.type === "pointer" && action.action === "click" && (action.button === undefined || action.button === "left") && frames.attachment?.focusedInputObservations === true && frames.attachment.browserSessionId === frame.browserSessionId && frames.attachment.controllerGeneration === frame.controllerGeneration && frames.attachment.targetId === frame.targetId && Date.parse(frames.attachment.expiresAt) > Date.now(); const receipt = frame ? await browser.actFromFrame( action, frame, undefined, focusedInput ? "input" : "none", ) : await browser.act(action); if (receipt.state !== "completed") { throw new Error( receipt.error?.message ?? t("browserViewer.browserInputDidNotComplete"), ); } return receipt; }} onReadClipboard={browser.readClipboard} onObserveForInput={browser.observeForInput} onSelectFromObservation={browser.actFromObservation} onReconnect={ selectedChromeGenerationLoss ? () => { if (replacementChromeDevice) { createBrowser({ kind: "attached", device: replacementChromeDevice }); } } : () => { void browser.refresh(); frames.reconnect(); } } reconnectLabel={ selectedChromeGenerationLoss && replacementChromeDevice ? t("browserViewer.openAFreshConnectedChrome") : undefined } reconnectMessage={ selectedChromeGenerationLoss ? t("browserViewer.chromeReconnectedOpenAFresh") : undefined } onError={(cause) => notifyError(cause, t("browserViewer.browserInputFailed2"))} /> {diagnosticsView ? ( void downloads.refresh()} onRefresh={refreshDiagnostics} onClose={closeDiagnostics} /> ) : null}
{ const computerSessionId = browser.session?.linkedComputerSessionId; if (computerSessionId) onOpenComputer(computerSessionId); } : undefined } onDiagnostics={diagnosticsView ? closeDiagnostics : loadDiagnostics} /> )}
); } function BrowserLifecyclePanel(props: { session: BrowserSession; attempt: BrowserResumeAttempt | null; onResume: () => void; }) { const t = useOpenGeniText(); const failed = props.attempt?.error ?? null; const label = failed ? t("browserViewer.browserCouldNotReopen") : props.session.lifecycle === "suspending" ? t("browserViewer.savingBrowserState") : props.session.lifecycle === "restoring" || props.attempt?.running ? t("browserViewer.openingBrowser") : props.session.lifecycle === "starting" ? t("browserViewer.startingBrowser") : t("browserViewer.browserIsSleeping"); const detail = failed ? failed.message : props.session.lifecycle === "suspended" ? t("browserViewer.itsPrivateStateIsSaved") : t("browserViewer.theBrowserWillReconnectHere"); return (
{failed ? ( ) : ( )}

{label}

{detail}

{failed || (props.session.lifecycle === "suspended" && !props.attempt?.running) ? ( ) : null}
); } function BrowserToolbar(props: { sessions: BrowserSession[]; relevantSessionIds: Set; selectedSessionId: string | null; attachedBridges: AttachedBrowserBridge[]; attachedDevices: AttachedBrowserDevice[]; identities: BrowserIdentity[]; selectedProfile: BrowserIdentity | null; baseRevisionOrdinal: number | null; profileHistory: { identityId: string; loading: boolean; revisions: BrowserRevision[] } | null; authConnections: SiteAuthConnection[]; creating: boolean; savingProfile: boolean; refreshing: boolean; interventionCounts: Map; onSelect: (id: string) => void; onFollow: () => void; onCreate: (choice?: BrowserLaunchChoice) => void; onSaveProfile: (newProfileName?: string) => Promise; onUpdateProfile: ( identity: BrowserIdentity, update: { name?: string; status?: "active" | "archived"; defaultRevisionId?: string }, ) => Promise; onOpenProfileVersion: (identityId: string, baseRevisionId: string) => void; onRefresh: () => void; setupUrl?: string | undefined; extensionName?: string | undefined; }) { const t = useOpenGeniText(); const detailsRef = useViewerMenuDismiss(); const selected = props.sessions.find((session) => session.id === props.selectedSessionId); const current = props.sessions.filter((session) => props.relevantSessionIds.has(session.id)); const others = props.sessions.filter((session) => !props.relevantSessionIds.has(session.id)); const choose = (id: string) => { props.onSelect(id); detailsRef.current?.removeAttribute("open"); }; return (
{selected?.name ?? t("browserViewer.browser")} {(props.interventionCounts.get(selected?.id ?? "") ?? 0) > 0 ? ( ) : null}
{current.length > 0 ? ( {t("browserViewer.followAgent")} ) : null} props.onCreate()} disabled={props.creating}> {t("browserViewer.newBrowser")}
{placementLabel(t, selected)} props.onUpdateProfile(identity, { status: "active" })} setupUrl={props.setupUrl} extensionName={props.extensionName} />
); } function BrowserUnselectedPanel(props: { unavailableBrowser: BrowserSession | undefined; peerCount: number; creating: boolean; onCreate: () => void; }) { const t = useOpenGeniText(); return (
{props.unavailableBrowser ? ( ) : (

{t("browserViewer.noBrowserForThisAgent")}

)}

{props.peerCount === 1 ? t("browserViewer.oneWorkspaceBrowserIsAvailable") : t("browserViewer.peercountWorkspaceBrowsersAreAvailable", { peerCount: props.peerCount, })}

); } function BrowserUnavailableNotice({ session }: { session: BrowserSession }) { const t = useOpenGeniText(); const reason = session.failureCode === "provider_deadline_rotation" ? t("browserViewer.theComputerRunningItReached") : session.failureCode === "source_placement_changed" ? t("browserViewer.thisChatMovedToAnother") : session.failureCode === "idle_released" ? t("browserViewer.itStoppedAfterItWent") : session.lifecycle === "failed" ? t("browserViewer.itStoppedUnexpectedly") : t("browserViewer.theConnectionToItWas"); return (

{t("browserViewer.browserUnavailable")}

{t("browserViewer.nameReasonOpenANew", { name: session.name, reason })}

); } function BrowserSessionGroup(props: { label: string; sessions: BrowserSession[]; identities: BrowserIdentity[]; selectedId: string | null; interventionCounts: Map; onSelect: (id: string) => void; }) { const t = useOpenGeniText(); if (props.sessions.length === 0) return null; return (

{props.label}

{props.sessions.map((session) => { const identity = props.identities.find((candidate) => candidate.id === session.identityId); const interventionCount = props.interventionCounts.get(session.id) ?? 0; return ( ); })}
); } function BrowserLaunchMenu(props: { attachedBridges: AttachedBrowserBridge[]; attachedDevices: AttachedBrowserDevice[]; identities: BrowserIdentity[]; creating: boolean; mutatingProfile: boolean; onCreate: (choice?: BrowserLaunchChoice) => void; onRestore: (identity: BrowserIdentity) => Promise; setupUrl?: string | undefined; extensionName?: string | undefined; prominent?: boolean; }) { const t = useOpenGeniText(); const detailsRef = useViewerMenuDismiss(); const choose = (choice?: BrowserLaunchChoice) => { detailsRef.current?.removeAttribute("open"); props.onCreate(choice); }; const restore = async (identity: BrowserIdentity) => { if (await props.onRestore(identity)) detailsRef.current?.removeAttribute("open"); }; const busy = props.creating || props.mutatingProfile; const activeIdentities = props.identities.filter((identity) => identity.status === "active"); const archivedIdentities = props.identities.filter((identity) => identity.status === "archived"); return (
{ if (busy) event.preventDefault(); }} className={cn( "cursor-pointer list-none items-center justify-center gap-1.5 text-og-fg-muted transition hover:bg-og-surface-2 hover:text-og-fg [&::-webkit-details-marker]:hidden", props.prominent ? "inline-flex h-8 rounded-og-sm border border-og-border bg-og-surface-1 px-3 text-og-control font-medium text-og-fg" : "grid size-7 place-items-center rounded-og-sm", busy && "pointer-events-none opacity-50", )} aria-label={t("browserViewer.newBrowser")} > {busy ? ( ) : ( )} {props.prominent ? t("browserViewer.newBrowser") : null} {props.prominent ? : null}
{props.attachedDevices.length > 0 || props.attachedBridges.length > 0 ? (

{t("browserViewer.connectedChrome")}

{props.attachedDevices.map((device) => ( ))} {props.attachedBridges.length > 0 ? ( props.setupUrl ? ( detailsRef.current?.removeAttribute("open")} className="flex w-full items-center gap-2 rounded-og-sm px-2 py-2 text-left transition hover:bg-og-surface-2" > {props.attachedDevices.length > 0 ? t("browserViewer.connectAnotherChromeProfile") : t("browserViewer.connectThisChromeProfile")} {t("browserViewer.machineConnectedChromeExtensionMissing")} ) : (
{t("browserViewer.chromeConnectionReady")} {t("browserViewer.installTheOpengeniBrowserExtension", { extensionName: props.extensionName ?? "Opengeni Browser", })}
) ) : null}
) : null}

0 || props.attachedBridges.length > 0) && "border-t border-og-border", )} > {t("browserViewer.openIsolatedBrowser")}

{activeIdentities.length > 0 ? (
{activeIdentities.map((identity) => ( ))}
) : null} {archivedIdentities.length > 0 ? (
{" "} {t("browserViewer.hiddenProfilesLength", { length: archivedIdentities.length })} {archivedIdentities.map((identity) => ( ))}
) : null}
); } function BrowserProfileMenu(props: { session: BrowserSession | null; identity: BrowserIdentity | null; baseRevisionOrdinal: number | null; history: { identityId: string; loading: boolean; revisions: BrowserRevision[] } | null; authConnections: SiteAuthConnection[]; saving: boolean; onSave: (newProfileName?: string) => Promise; onUpdate: ( identity: BrowserIdentity, update: { name?: string; status?: "active" | "archived"; defaultRevisionId?: string }, ) => Promise; onOpenVersion: (identityId: string, baseRevisionId: string) => void; }) { const t = useOpenGeniText(); const detailsRef = useViewerMenuDismiss(); const nameInputId = useId(); const [name, setName] = useState(""); const attached = props.session?.placement.kind === "attached_device"; const canSave = (props.session?.capabilities.identityPublication ?? false) && props.identity?.status !== "archived"; const revisions = props.identity && props.history?.identityId === props.identity.id ? [...props.history.revisions].sort((left, right) => right.ordinal - left.ordinal) : []; const save = async (newProfileName?: string) => { if (await props.onSave(newProfileName)) { setName(""); detailsRef.current?.removeAttribute("open"); } }; const openVersion = (identityId: string, baseRevisionId: string) => { detailsRef.current?.removeAttribute("open"); props.onOpenVersion(identityId, baseRevisionId); }; const versionLabel = attached ? "live" : props.identity ? props.baseRevisionOrdinal ? `v${props.baseRevisionOrdinal}` : props.session?.baseRevisionId ? "saved" : "unsaved" : "temporary"; return (
{props.saving ? ( ) : attached ? ( ) : ( )} {attached ? props.session?.name : (props.identity?.name ?? t("browserViewer.temporary"))} {attached || props.identity ? ( · {versionLabel} ) : null}
{attached ? ( ) : ( )}

{attached ? props.session?.name : (props.identity?.name ?? t("browserViewer.temporaryBrowser"))}

{attached ? t("browserViewer.thisSessionDrivesTheExisting") : props.identity?.status === "archived" ? t("browserViewer.hiddenFromNewBrowsersThis") : props.identity ? props.baseRevisionOrdinal ? t("browserViewer.startedFromVersionBaserevisionordinal", { baseRevisionOrdinal: props.baseRevisionOrdinal, }) : t("browserViewer.notSavedYet") : t("browserViewer.thisBrowserIsNotReusable")}

{props.identity && revisions.length > 0 ? (

{t("browserViewer.savedVersions2")}

{revisions.map((revision) => { const isDefault = revision.id === props.identity?.defaultRevisionId; const isCurrent = revision.ordinal === props.baseRevisionOrdinal; const materialization = revisionMaterializationSummary(t, revision); return (
); })}

{t("browserViewer.changingTheDefaultAffectsOnly")}

) : props.identity && props.history?.identityId === props.identity.id && props.history.loading ? (

{" "} {t("browserViewer.loadingSavedVersions")}

) : null} {props.identity && props.authConnections.length > 0 ? (

{t("browserViewer.loginHealth")}

{props.authConnections.map((connection) => { const health = siteAuthHealth(t, connection); return (
{connection.name} {connection.accountLabel} · {health.label}
); })}
) : null} {canSave ? ( props.identity ? ( ) : (
{ event.preventDefault(); if (name.trim()) void save(name); }} >
setName(event.currentTarget.value)} maxLength={200} placeholder={t("browserViewer.work")} className="h-8 min-w-0 flex-1 rounded-og-sm border border-og-border bg-og-bg px-2 text-og-control text-og-fg placeholder:text-og-fg-subtle focus:border-og-accent focus-visible:outline-hidden focus-visible:ring-2 focus-visible:ring-og-accent" />
) ) : (

{attached ? t("browserViewer.chromeKeepsThisProfileS") : props.identity?.status === "archived" ? t("browserViewer.restoreThisProfileBeforeSaving") : t("browserViewer.thisBrowserCannotSaveReusable")}

)} {!attached && props.identity ? ( ) : null} {canSave ? (

{t("browserViewer.savingRestartsThisBrowserSave")}

) : null}
); } function revisionMaterializationSummary( t: OpenGeniText, revision: BrowserRevision, ): { label: string } { const materializations = revision.components.map((component) => component.materialization); const placementBound = materializations.find( (materialization) => materialization.portability === "placement_bound", ); if (placementBound) { return { label: placementBound.reason ? t("browserViewer.thisMachineOnlyReason", { reason: placementBound.reason }) : t("browserViewer.thisMachineOnly"), }; } const providerBound = materializations.find( (materialization) => materialization.portability === "provider_bound", ); if (providerBound) { return { label: providerBound.reason ? t("browserViewer.providerBoundReason", { reason: providerBound.reason }) : t("browserViewer.providerBound"), }; } return { label: t("browserViewer.portableBrowserData") }; } function siteAuthHealth( t: OpenGeniText, connection: SiteAuthConnection, ): { label: string; dotClass: string; } { switch (connection.verificationState) { case "verified": return { label: t("browserViewer.verified"), dotClass: "bg-og-status-running" }; case "needs_repair": return { label: t("browserViewer.signInNeedsAttention"), dotClass: "bg-og-status-waiting" }; case "failed": return { label: t("browserViewer.verificationFailed"), dotClass: "bg-og-status-error" }; case "unknown": return { label: t("browserViewer.notCheckedYet"), dotClass: "bg-og-muted" }; } } function MenuButton(props: { children: ReactNode; onClick: () => void; disabled?: boolean }) { return ( ); } function BrowserTabs(props: { targets: BrowserTarget[]; selectedTargetId: string | null; mutating: boolean; tabControl: boolean; onSelect: (id: string) => void; onClose: (id: string) => void; onOpen: () => void; }) { const t = useOpenGeniText(); const pages = props.targets.filter((target) => target.kind === "page" || target.kind === "popup"); return (
{pages.map((target) => (
{props.tabControl ? ( ) : null}
))} {props.tabControl ? ( ) : null}
); } function BrowserAddressBar(props: { target: BrowserTarget | null; loading: boolean; onNavigate: (url: string) => void; onHistory: (direction: "back" | "forward") => Promise; onReload: () => void; }) { const t = useOpenGeniText(); const [draft, setDraft] = useState(props.target?.url ?? ""); const focusedRef = useRef(false); const historyPendingRef = useRef(false); const [historyPending, setHistoryPending] = useState(false); useEffect(() => { if (!focusedRef.current) setDraft(props.target?.url ?? ""); }, [props.target?.id, props.target?.url]); const submit = (event: FormEvent) => { event.preventDefault(); if (historyPendingRef.current) return; const normalized = normalizeBrowserAddress(draft); if (normalized) props.onNavigate(normalized); }; const navigateHistory = async (direction: "back" | "forward") => { if (!props.target || props.loading || historyPendingRef.current) return; historyPendingRef.current = true; setHistoryPending(true); try { await props.onHistory(direction); } finally { historyPendingRef.current = false; setHistoryPending(false); } }; return (
{(["back", "forward"] as const).map((direction) => ( ))}
setDraft(event.target.value)} onFocus={() => { focusedRef.current = true; }} onBlur={() => { focusedRef.current = false; }} onKeyDown={(event) => { if (event.key !== "Enter") return; event.preventDefault(); if (historyPendingRef.current) return; const normalized = normalizeBrowserAddress(draft); if (normalized) props.onNavigate(normalized); }} disabled={!props.target} className="min-w-0 flex-1 bg-transparent text-og-control text-og-fg placeholder:text-og-fg-subtle focus-visible:outline-hidden focus-visible:ring-2 focus-visible:ring-og-accent disabled:opacity-50" aria-label={t("browserViewer.address")} spellCheck={false} placeholder={t("browserViewer.searchOrEnterAddress")} />
); } function BrowserViewport(props: { focusHandoffRef: { current: string | null }; focusScope: string; frame: BrowserViewportFrame | null; connectionState: string; supportsLiveFrames: boolean; connectionError: Error | null; controlUnavailable: boolean; observation: ReturnType["observation"]; mutating: boolean; clipboardEnabled: boolean; inputBatchAttachment?: { browserSessionId: string; controllerGeneration: string; targetId: string; fencedInputBatches?: true | undefined; focusedInputObservations?: true | undefined; expiresAt: string; } | null; activityLabel?: string | undefined; onAction: ( action: BrowserAction | BrowserActionBatch, frame: BrowserFrameInputFence | null, ) => Promise; onReadClipboard: () => Promise<{ text: string }>; onObserveForInput: () => Promise; onSelectFromObservation: ( action: BrowserAction, observation: BrowserObservation, ) => Promise; onReconnect: () => void; reconnectLabel?: string | undefined; reconnectMessage?: string | undefined; onError: (cause: unknown) => void; }) { const t = useOpenGeniText(); const canvasRef = useRef(null); const inputRef = useRef(null); const [selectPopup, setSelectPopup] = useState<{ observation: BrowserObservation | null; anchor: { x: number; y: number } | null; } | null>(null); const inputSequenceRef = useRef(0); const dismissSelectPopup = useCallback(() => { inputSequenceRef.current += 1; setSelectPopup({ observation: null, anchor: null }); }, []); const composingRef = useRef(false); const pointerStartRef = useRef(null); const lastClickRef = useRef<{ at: number; x: number; y: number; frame: BrowserViewportFrame; } | null>(null); const wheelRef = useRef<{ x: number; y: number; deltaX: number; deltaY: number; frame: BrowserViewportFrame; timer: ReturnType | null; } | null>(null); const pendingTextRef = useRef<{ text: string; frame: BrowserViewportFrame | null; timer: ReturnType; } | null>(null); const actionRef = useRef(props.onAction); const readClipboardRef = useRef(props.onReadClipboard); const observeInputRef = useRef(props.onObserveForInput); const errorRef = useRef(props.onError); const actionTailRef = useRef>(Promise.resolve()); const actionQueueEpochRef = useRef(0); const queuedTypingRef = useRef<{ actions: BrowserAction[]; frame: BrowserFrameInputFence; epoch: number; } | null>(null); const queuedFrameRef = useRef(null); const currentFrameRef = useRef(props.frame); const paintedFrameRef = useRef(null); const [paintedFrame, setPaintedFrame] = useState(null); const decodingFrameRef = useRef(false); const mountedRef = useRef(true); actionRef.current = props.onAction; readClipboardRef.current = props.onReadClipboard; observeInputRef.current = props.onObserveForInput; errorRef.current = props.onError; const clearBufferedInput = useCallback(() => { dismissSelectPopup(); if (wheelRef.current?.timer) clearTimeout(wheelRef.current.timer); if (pendingTextRef.current?.timer) clearTimeout(pendingTextRef.current.timer); wheelRef.current = null; pendingTextRef.current = null; queuedTypingRef.current = null; pointerStartRef.current = null; lastClickRef.current = null; composingRef.current = false; if (inputRef.current) inputRef.current.value = ""; }, [dismissSelectPopup]); const paintQueuedFrames = useCallback(() => { if (decodingFrameRef.current) return; decodingFrameRef.current = true; void (async () => { try { while (mountedRef.current) { const frame = queuedFrameRef.current; queuedFrameRef.current = null; if (!frame) break; let objectUrl: string | null = null; try { const blob = new Blob([frame.data], { type: frame.mediaType, }); if (typeof createImageBitmap === "function") { const bitmap = await createImageBitmap(blob); try { const canvas = canvasRef.current; if ( mountedRef.current && canvas && currentFrameRef.current && sameBrowserDocument(frame, currentFrameRef.current) ) { paintCanvas(canvas, bitmap, frame.width, frame.height); paintedFrameRef.current = frame; setPaintedFrame(frame); } } finally { bitmap.close(); } continue; } objectUrl = URL.createObjectURL(blob); const image = await loadImage(objectUrl); const canvas = canvasRef.current; if ( mountedRef.current && canvas && currentFrameRef.current && sameBrowserDocument(frame, currentFrameRef.current) ) { paintCanvas(canvas, image, frame.width, frame.height); paintedFrameRef.current = frame; setPaintedFrame(frame); } } catch (cause) { if ( mountedRef.current && currentFrameRef.current && sameBrowserDocument(frame, currentFrameRef.current) ) errorRef.current(cause); } finally { if (objectUrl) URL.revokeObjectURL(objectUrl); } } } finally { decodingFrameRef.current = false; if (mountedRef.current && queuedFrameRef.current) paintQueuedFrames(); } })(); }, []); useLayoutEffect(() => { const keyboardInput = inputRef.current; const focusHandoff = props.focusHandoffRef; mountedRef.current = true; if (focusHandoff.current === props.focusScope) { keyboardInput?.focus({ preventScroll: true }); } focusHandoff.current = null; return () => { // Only a same-target replacement may inherit focus the keyboard sink owned. if (document.activeElement === keyboardInput) { focusHandoff.current = props.focusScope; } mountedRef.current = false; actionQueueEpochRef.current += 1; queuedFrameRef.current = null; clearBufferedInput(); }; }, [clearBufferedInput, props.focusHandoffRef, props.focusScope]); useLayoutEffect(() => { if (currentFrameRef.current && !props.frame) { actionQueueEpochRef.current += 1; clearBufferedInput(); } currentFrameRef.current = props.frame; if (!props.frame) { queuedFrameRef.current = null; paintedFrameRef.current = null; setPaintedFrame(null); return; } queuedFrameRef.current = props.frame; paintQueuedFrames(); }, [clearBufferedInput, paintQueuedFrames, props.frame]); const enqueue = useCallback( ( action: BrowserAction, frame: BrowserFrameInputFence | null, after?: (receipt: BrowserActionReceipt) => Promise, selectAnchor?: { x: number; y: number }, ) => { const inputSequence = ++inputSequenceRef.current; setSelectPopup({ observation: null, anchor: null }); // Detach image bytes even when the caller supplied a complete painted frame. frame = frame ? { browserSessionId: frame.browserSessionId, controllerGeneration: frame.controllerGeneration, targetId: frame.targetId, targetGeneration: frame.targetGeneration, documentGeneration: frame.documentGeneration, frameId: frame.frameId, } : null; const epoch = actionQueueEpochRef.current; const attachment = props.inputBatchAttachment; const canBatchTyping = action.type === "type" && !after && frame && attachment?.fencedInputBatches === true && attachment.browserSessionId === frame.browserSessionId && attachment.controllerGeneration === frame.controllerGeneration && attachment.targetId === frame.targetId && Date.parse(attachment.expiresAt) > Date.now(); const previous = queuedTypingRef.current; if ( canBatchTyping && frame && previous && previous.epoch === epoch && sameFrameFence(previous.frame, frame) && previous.actions.length < 16 ) { // Keep every existing text/input event boundary. Only share transport. previous.actions.push(action); return; } const typing = canBatchTyping && frame ? { actions: [action], frame, epoch } : null; // Every non-typing action is an ordering barrier, including keys and paste. queuedTypingRef.current = typing; actionTailRef.current = actionTailRef.current .catch(() => undefined) .then(async () => { if (!mountedRef.current || epoch !== actionQueueEpochRef.current) return; if (queuedTypingRef.current === typing) queuedTypingRef.current = null; const dispatchedAction = typing && typing.actions.length > 1 ? { type: "batch" as const, actions: typing.actions, fenceEachAction: true as const } : action; // Read after the epoch guard: a queued render callback can retain every // frame from that render even when its input fence contains no bytes. const receipt = await actionRef.current(dispatchedAction, frame); if (!mountedRef.current || epoch !== actionQueueEpochRef.current) return; // Alt+Down explicitly opens a native dropdown from the keyboard. // Existing controllers return focused-input metadata for clicks only, // so use the ordinary authorized read for this explicit gesture. const keyboardSelect = dispatchedAction.type === "press" && dispatchedAction.key === "Alt+ArrowDown"; if ((selectAnchor || keyboardSelect) && inputSequence === inputSequenceRef.current) { const observation = keyboardSelect ? await observeInputRef.current() : receipt.observation; if ( mountedRef.current && epoch === actionQueueEpochRef.current && inputSequence === inputSequenceRef.current && observation ) { setSelectPopup({ observation, anchor: selectAnchor ?? null }); } } await after?.(receipt); }) .catch((cause) => { if (!mountedRef.current || epoch !== actionQueueEpochRef.current) return; // Do not replay or continue input collected behind a failed request. actionQueueEpochRef.current += 1; clearBufferedInput(); errorRef.current(cause); }); }, [clearBufferedInput, props.inputBatchAttachment], ); const point = useCallback( (frame: BrowserViewportFrame, clientX: number, clientY: number) => browserPoint(canvasRef.current, frame, clientX, clientY), [], ); const flushPendingText = useCallback(() => { const pending = pendingTextRef.current; if (!pending) return; clearTimeout(pending.timer); pendingTextRef.current = null; enqueue({ type: "type", text: pending.text }, pending.frame); }, [enqueue]); const flushPendingClick = useCallback(() => { lastClickRef.current = null; }, []); const flushPendingWheel = useCallback(() => { const batch = wheelRef.current; if (!batch) return; if (batch.timer) clearTimeout(batch.timer); wheelRef.current = null; enqueue( { type: "pointer", action: "scroll", x: batch.x, y: batch.y, deltaX: batch.deltaX, deltaY: batch.deltaY, }, batch.frame, ); }, [enqueue]); const pointerDown = (event: PointerEvent) => { const frame = paintedFrameRef.current; if (!frame || event.button !== 0) return; // Keep the hidden keyboard sink focused after the browser performs the // pointerdown default action for the canvas. Without preventing that // default, Chrome immediately moves focus back to the document and all // subsequent typing/paste is silently lost. event.preventDefault(); dismissSelectPopup(); flushPendingWheel(); flushPendingText(); pointerStartRef.current = { x: event.clientX, y: event.clientY, pointerId: event.pointerId, frame, }; event.currentTarget.setPointerCapture?.(event.pointerId); inputRef.current?.focus({ preventScroll: true }); }; const pointerUp = (event: PointerEvent) => { const start = pointerStartRef.current; pointerStartRef.current = null; if (!start || start.pointerId !== event.pointerId) return; const from = point(start.frame, start.x, start.y); const to = point(start.frame, event.clientX, event.clientY); if (!from || !to) return; if (Math.hypot(event.clientX - start.x, event.clientY - start.y) > 5) { flushPendingClick(); enqueue( { type: "pointer", action: "drag", x: from.x, y: from.y, endX: to.x, endY: to.y, }, start.frame, ); return; } const now = Date.now(); const previous = lastClickRef.current; if ( previous && now - previous.at < 280 && Math.hypot(previous.x - to.x, previous.y - to.y) < 6 && sameFrameFence(previous.frame, start.frame) ) { lastClickRef.current = null; // The first click was already dispatched immediately. Tell CDP this is // the continuation (clickCount=2), so the page receives a true dblclick // without making every ordinary click wait for the double-click window. enqueue({ type: "pointer", action: "click", clickCount: 2, x: to.x, y: to.y }, start.frame); return; } lastClickRef.current = { at: now, x: to.x, y: to.y, frame: start.frame }; const viewportBounds = canvasRef.current?.parentElement?.getBoundingClientRect(); enqueue( { type: "pointer", action: "click", x: to.x, y: to.y }, start.frame, undefined, viewportBounds ? { x: event.clientX - viewportBounds.left, y: event.clientY - viewportBounds.top + 8 } : undefined, ); }; const contextMenu = (event: MouseEvent) => { event.preventDefault(); const frame = paintedFrameRef.current; if (!frame) return; flushPendingWheel(); flushPendingText(); flushPendingClick(); const at = point(frame, event.clientX, event.clientY); if (at) enqueue({ type: "pointer", action: "click", x: at.x, y: at.y, button: "right" }, frame); }; const wheel = (event: WheelEvent) => { dismissSelectPopup(); const frame = paintedFrameRef.current; if (!frame) return; const at = point(frame, event.clientX, event.clientY); if (!at) return; flushPendingText(); flushPendingClick(); event.preventDefault(); let pending = wheelRef.current; if ( pending && (!sameBrowserDocument(pending.frame, frame) || Math.hypot(pending.x - at.x, pending.y - at.y) > 6) ) { flushPendingWheel(); pending = null; } wheelRef.current = { x: at.x, y: at.y, deltaX: (pending?.deltaX ?? 0) + event.deltaX, deltaY: (pending?.deltaY ?? 0) + event.deltaY, frame, // Retain the first event's deadline so a continuous gesture keeps moving. timer: pending?.timer ?? setTimeout(flushPendingWheel, 45), }; }; const keyDown = (event: KeyboardEvent) => { // Candidate selection/editing belongs to the local IME. Forward only the // committed text; Enter must not submit the remote page during composition. // Safari can end composition before keydown while retaining keyCode 229. if ( composingRef.current || event.nativeEvent.isComposing || event.nativeEvent.keyCode === 229 ) { return; } dismissSelectPopup(); flushPendingWheel(); const command = event.metaKey || event.ctrlKey; if ( props.clipboardEnabled && command && !event.altKey && ["c", "v"].includes(event.key.toLowerCase()) ) { flushPendingText(); return; } const key = browserKey(event); if (!key) return; event.preventDefault(); flushPendingText(); flushPendingClick(); enqueue({ type: "press", key }, paintedFrameRef.current); }; const finishCopy = useCallback(async () => { const clipboard = await readClipboardRef.current(); if (!mountedRef.current) return; if (clipboard.text.length === 0) return; if (!(await copyTextToClipboard(clipboard.text))) { throw new Error(t("browserViewer.browserTextCouldNotBe")); } }, [t]); const copy = (event: ClipboardEvent) => { if (!props.clipboardEnabled) return; event.preventDefault(); flushPendingWheel(); flushPendingText(); flushPendingClick(); enqueue({ type: "clipboard", operation: "copy" }, paintedFrameRef.current, finishCopy); }; const paste = (event: ClipboardEvent) => { if (!props.clipboardEnabled) return; event.preventDefault(); flushPendingWheel(); flushPendingText(); flushPendingClick(); enqueue( { type: "clipboard", operation: "paste", text: event.clipboardData.getData("text/plain"), }, paintedFrameRef.current, ); }; const input = (value: string, nativeComposing = false) => { dismissSelectPopup(); if (composingRef.current || nativeComposing) return; if (!value) return; flushPendingWheel(); flushPendingClick(); const pending = pendingTextRef.current; if (pending && sameOptionalBrowserFrame(pending.frame, paintedFrameRef.current)) { clearTimeout(pending.timer); pending.text += value; pending.timer = setTimeout(flushPendingText, 16); } else { flushPendingText(); pendingTextRef.current = { text: value, frame: paintedFrameRef.current, timer: setTimeout(flushPendingText, 16), }; } if (inputRef.current) inputRef.current.value = ""; }; const compositionStart = () => { composingRef.current = true; }; const compositionEnd = (event: CompositionEvent) => { composingRef.current = false; input(event.currentTarget.value || event.data); }; const showCanvas = !props.controlUnavailable && props.frame !== null && paintedFrame !== null && sameBrowserDocument(props.frame, paintedFrame); return (
{ pointerStartRef.current = null; }} onContextMenu={contextMenu} onWheel={wheel} aria-label={t("browserViewer.interactiveBrowserPage")} />