import type { AttemptToolResult, AttachedBrowserDevice, AuthRun, AuthRunListResponse, AuthRunMutationResponse, BrowserAction, BrowserActionReceipt, BrowserClipboard, BrowserDiagnosticBatch, BrowserDiagnosticKind, BrowserDownload, BrowserDownloadListResponse, BrowserDownloadSaveResponse, BrowserDomReadLocator, BrowserDomReadResponse, BrowserDomSafeAttribute, BrowserIdentity, BrowserIdentityListResponse, BrowserIdentityMutationResponse, BrowserLocator, BrowserObservation, BrowserPermission, BrowserPermissionSetting, BrowserRevisionListResponse, BrowserSession, BrowserSessionMutationResponse, BrowserTarget, ComputerAction, ComputerActionReceipt, ComputerClipboard, ComputerLocator, ComputerObservation, ComputerSession, ComputerSessionMutationResponse, ComputerTarget, ComputerTargetListResponse, ExternalAuthRunRequest, ExternalAuthRunResponse, InteractionPlacement, PublishBrowserRevisionResponse, ReportAuthRunRequest, RequestHumanInteractionToolInput, RequestHumanInteractionToolOutput, SiteAuthConnection, SiteAuthConnectionListResponse, StartAuthRunRequest, UpdateBrowserIdentityRequest, VerifyAuthRunRequest, } from "@opengeni/contracts"; import type { CodemodeCallOptions, CodemodeClient } from "./index"; import { environmentCodemodeClient, type CodemodeClientProvider } from "./environment"; import { CodemodeArtifactCollection } from "./artifacts"; import { materializeCodemodeImages, type CodemodeLocalImage } from "./images"; import { callStructured, codemodeClientProvider, CodemodeToolExecutionError } from "./structured"; export { CodemodeToolExecutionError } from "./structured"; const PATH = { discover: ["interaction", "discover"], browserOpen: ["interaction", "browser", "open"], browserTabs: ["interaction", "browser", "tabs"], browserObserve: ["interaction", "browser", "observe"], browserRead: ["interaction", "browser", "read"], browserScreenshot: ["interaction", "browser", "screenshot"], browserAct: ["interaction", "browser", "act"], browserClipboard: ["interaction", "browser", "clipboard"], browserDebug: ["interaction", "browser", "debug"], browserDownloads: ["interaction", "browser", "downloads"], browserDownloadSave: ["interaction", "browser", "downloadSave"], browserAuth: ["interaction", "browser", "auth"], requestHuman: ["interaction", "requestHuman"], browserIdentity: ["interaction", "browser", "identity"], browserPublish: ["interaction", "browser", "publish"], browserLifecycle: ["interaction", "browser", "lifecycle"], computerOpen: ["interaction", "computer", "open"], computerTargets: ["interaction", "computer", "targets"], computerObserve: ["interaction", "computer", "observe"], computerClipboard: ["interaction", "computer", "clipboard"], computerAct: ["interaction", "computer", "act"], computerLifecycle: ["interaction", "computer", "lifecycle"], } as const; export type InteractionDiscovery = { browserRevision: number; computerRevision: number; identityRevision: number; attachedBrowserRevision: number; browsers: BrowserSession[]; computers: ComputerSession[]; identities: BrowserIdentity[]; /** Online machines ready for the extension; listed only when no attached * profile is connected. */ attachedBrowserBridges: Array<{ enrollmentId: string; connectedProfileCount: number }>; attachedBrowsers: Array<{ id: string; name: string; profileLabel?: string; browserName: string; platform: AttachedBrowserDevice["platform"]; state: AttachedBrowserDevice["state"]; tabCount: number; /** Capabilities this profile lacks; absent when it supports all of them. */ unsupported?: string[]; }>; }; export type CodemodeBrowserOpenOptions = { browserSessionId?: string | undefined; mode?: "reuse_or_create" | "new" | undefined; name?: string | undefined; initialUrl?: string | undefined; headless?: boolean | undefined; /** `{ kind: "attached_device" }` without a deviceId opens the person's own * connected Chrome (the only, or most recently active, profile). */ placement?: InteractionPlacement | { kind: "attached_device" } | undefined; identityId?: string | undefined; baseRevisionId?: string | undefined; linkedComputerSessionId?: string | undefined; }; export type CodemodeComputerOpenOptions = { computerSessionId?: string | undefined; mode?: "reuse_or_create" | "new" | undefined; name?: string | undefined; placement?: InteractionPlacement | undefined; }; export type BrowserActionFences = { expectedTargetGeneration?: string | undefined; expectedDocumentGeneration?: string | null | undefined; expectedFrameId?: string | null | undefined; }; export type CodemodeBrowserScreenshotOptions = { /** Capture the entire scrollable page when supported by the browser. */ fullPage?: boolean | undefined; /** JPEG quality, 1–100. Reduce it if a large image exceeds the journal bound. */ quality?: number | undefined; /** Save the still at this exact path. Existing files are never overwritten. */ saveTo?: string | undefined; }; export type CodemodeBrowserScreenshotMetadata = Readonly<{ kind: "browser_screenshot"; browserSessionId: string; targetId: string; frameId: string; targetGeneration: string; documentGeneration: string; mediaType: "image/png" | "image/jpeg"; width: number; height: number; deviceScaleFactor: number; scrollX: number; scrollY: number; capturedAt: string; fullPage: boolean; }>; export type CodemodeBrowserScreenshot = CodemodeLocalImage & Readonly<{ /** Small structured frame metadata; pixel bytes stay in the local image file. */ metadata: CodemodeBrowserScreenshotMetadata; }>; export type CodemodeBrowserAgentNode = Readonly<{ ref: string; role: string; depth: number; name?: string; description?: string; value?: string | { redacted: true; reason: string }; /** Omitted when empty. */ states?: readonly string[]; /** Omitted when empty. */ actions?: readonly string[]; }>; export type CodemodeBrowserAgentView = Readonly<{ sourceKind: "snapshot" | "diff" | "none"; nodes: readonly CodemodeBrowserAgentNode[]; /** Nodes left out by the output budget; absent when nothing was left out. */ omittedNodeCount?: number; /** The largest regions holding the left-out nodes. */ omittedRegions?: readonly Readonly<{ role: string; name?: string; count: number }>[]; removedRefCount?: number; clippedFieldCount?: number; }>; /** A tab with the identity, page, selection and fences a next call needs. */ export type CodemodeBrowserTarget = Readonly<{ id: string; kind: BrowserTarget["kind"]; title: string; url: string; selected: boolean; controllerGeneration: string; targetGeneration: string; documentGeneration: string | null; }>; export type CodemodeBrowserTargetList = Readonly<{ browserSessionId: string; targets: readonly CodemodeBrowserTarget[]; }>; /** The compact default observation. Empty, zero and default fields are * omitted; use observeFull() for the complete BrowserObservation. */ export type CodemodeBrowserAgentObservation = Readonly<{ target: CodemodeBrowserTarget; frameId?: string; viewport?: Readonly<{ width: number; height: number; deviceScaleFactor?: number; maxTouchPoints?: number; }>; focusedRef?: string; changedRegions?: BrowserObservation["changedRegions"]; diagnostics?: BrowserObservation["diagnostics"]; dialog?: NonNullable; screenshot?: NonNullable; agentView: CodemodeBrowserAgentView; }>; /** The compact default receipt; pass view "full" for the complete * BrowserActionReceipt with operation id and timestamps. */ export type CodemodeBrowserActionReceipt = Readonly<{ targetId: string; state: BrowserActionReceipt["state"]; error?: NonNullable; targets?: BrowserActionReceipt["targets"]; observation?: CodemodeBrowserAgentObservation; }>; export type CodemodeBrowserActionOptions = BrowserActionFences & { view?: "compact" | "full" | "none" | undefined; }; export type CodemodeBrowserAxReadOptions = Readonly<{ mode?: "matches" | "count" | "subtree"; ref?: string; scopeRef?: string; role?: string; nameContains?: string; textContains?: string; state?: string; action?: string; limit?: number; }>; export type CodemodeBrowserDomReadOptions = BrowserActionFences & Readonly<{ mode: "dom"; dom: | { kind: "element"; locator: BrowserDomReadLocator; attributes?: readonly BrowserDomSafeAttribute[]; maxChars?: number; } | { kind: "count"; selector: string }; }>; export type CodemodeBrowserReadOptions = | CodemodeBrowserAxReadOptions | CodemodeBrowserDomReadOptions; export type CodemodeBrowserAxReadResult = Readonly<{ browserSessionId: string; targetId: string; observationId: string; targetGeneration: string; documentGeneration: string | null; frameId: string | null; source: "accessibility"; mode: "matches" | "count" | "subtree"; scopeFound: boolean; nodes: readonly CodemodeBrowserAgentNode[]; totalMatches: number; omittedMatches: number; clippedFieldCount: number; maxNodeBytes: number; }>; export type CodemodeBrowserReadResult = | CodemodeBrowserAxReadResult | (BrowserDomReadResponse & { source: "dom" }); export type ComputerActionFences = { expectedTargetGeneration?: string | undefined; expectedObservationId?: string | null | undefined; expectedFrameId?: string | null | undefined; }; /** Authored object facade over the exact atomic Browser/Computer catalog. */ export class OpenGeniCodemode { readonly browsers: CodemodeBrowserCollection; readonly computers: CodemodeComputerCollection; readonly artifacts: CodemodeArtifactCollection; readonly auth: CodemodeAuth; constructor(client: CodemodeClient | CodemodeClientProvider = () => environmentCodemodeClient()) { const provider = codemodeClientProvider(client); this.browsers = new CodemodeBrowserCollection(provider); this.computers = new CodemodeComputerCollection(provider); this.artifacts = new CodemodeArtifactCollection(provider); this.auth = new CodemodeAuth(provider); } async discover( options: { scope?: "current_session" | "workspace" | "attached_browsers" | undefined; includeTerminal?: boolean | undefined; includeArchivedIdentities?: boolean | undefined; includeDisconnectedDevices?: boolean | undefined; } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await callStructured(this.browsers.client, PATH.discover, options, callOptions); } async requestHuman( request: RequestHumanInteractionToolInput, callOptions: CodemodeCallOptions = {}, ): Promise { return await callStructured(this.browsers.client, PATH.requestHuman, request, callOptions); } } export class CodemodeBrowserCollection { readonly identities: CodemodeBrowserIdentityCollection; constructor(readonly client: CodemodeClientProvider) { this.identities = new CodemodeBrowserIdentityCollection(client); } async list( options: { includeTerminal?: boolean | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return ( await callStructured(this.client, PATH.discover, options, callOptions) ).browsers; } async open( options: CodemodeBrowserOpenOptions = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const opened = await callStructured<{ session: BrowserSession; targets: BrowserTarget[] }>( this.client, PATH.browserOpen, options, callOptions, ); return new CodemodeBrowser(this.client, opened.session.id); } use(browserSessionId: string): CodemodeBrowser { return new CodemodeBrowser(this.client, browserSessionId); } } export class CodemodeBrowserIdentityCollection { constructor(private readonly client: CodemodeClientProvider) {} async list( options: { includeArchived?: boolean | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "list"; result: BrowserIdentityListResponse; }>(this.client, PATH.browserIdentity, { operation: "list", ...options }, callOptions); return response.result; } async get(identityId: string, callOptions: CodemodeCallOptions = {}): Promise { const response = await callStructured<{ operation: "get"; result: BrowserIdentity }>( this.client, PATH.browserIdentity, { operation: "get", identityId }, callOptions, ); return response.result; } async create( name: string, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "create"; result: BrowserIdentityMutationResponse; }>(this.client, PATH.browserIdentity, { operation: "create", name }, callOptions); return response.result; } async revisions( identityId: string, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "revisions"; result: BrowserRevisionListResponse; }>(this.client, PATH.browserIdentity, { operation: "revisions", identityId }, callOptions); return response.result; } async update( identityId: string, update: Omit, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "update"; result: BrowserIdentityMutationResponse; }>( this.client, PATH.browserIdentity, { operation: "update", identityId, ...update }, callOptions, ); return response.result; } } export class CodemodeBrowser { readonly tabs: CodemodeBrowserTabCollection; readonly auth: CodemodeBrowserAuth; readonly clipboard: CodemodeBrowserClipboard; readonly downloads: CodemodeBrowserDownloadCollection; constructor( private readonly client: CodemodeClientProvider, readonly id: string, ) { this.tabs = new CodemodeBrowserTabCollection(client, id); this.auth = new CodemodeBrowserAuth(client, id); this.clipboard = new CodemodeBrowserClipboard(client, id, this.tabs); this.downloads = new CodemodeBrowserDownloadCollection(client, id); } async refresh(callOptions: CodemodeCallOptions = {}): Promise { const result = await callStructured<{ session: BrowserSession; targets: BrowserTarget[] }>( this.client, PATH.browserOpen, { browserSessionId: this.id }, callOptions, ); return result.session; } async observe( targetId?: string, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.tabs .use(await this.tabs.resolveId(targetId, callOptions)) .observe(callOptions); } async observeFull( targetId?: string, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.tabs .use(await this.tabs.resolveId(targetId, callOptions)) .observeFull(callOptions); } async read( options: CodemodeBrowserReadOptions & { targetId?: string | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const { targetId, ...query } = options; return await this.tabs .use(await this.tabs.resolveId(targetId, callOptions)) .read(query, callOptions); } async screenshot( options: CodemodeBrowserScreenshotOptions & { targetId?: string | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.tabs .use(await this.tabs.resolveId(options.targetId, callOptions)) .screenshot(options, callOptions); } async act( action: BrowserAction, options: CodemodeBrowserActionOptions & { targetId?: string | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const targetId = await this.tabs.resolveId(options.targetId, callOptions); return await this.tabs.use(targetId).act(action, options, callOptions); } async diagnostics( targetId?: string, options: { kinds?: BrowserDiagnosticKind[] | undefined; after?: number | undefined; limit?: number | undefined; } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await callStructured( this.client, PATH.browserDebug, { browserSessionId: this.id, targetId: await this.tabs.resolveId(targetId, callOptions), ...options, }, callOptions, ); } async publish( input: { identityId: string; expectedHeadGeneration: number; advanceDefault?: boolean | undefined; }, callOptions: CodemodeCallOptions = {}, ): Promise { return await callStructured( this.client, PATH.browserPublish, { browserSessionId: this.id, ...input }, callOptions, ); } async suspend(callOptions: CodemodeCallOptions = {}): Promise { return await this.lifecycle("suspend", callOptions); } async resume(callOptions: CodemodeCallOptions = {}): Promise { return await this.lifecycle("resume", callOptions); } async end(callOptions: CodemodeCallOptions = {}): Promise { return await this.lifecycle("end", callOptions); } private async lifecycle( action: "suspend" | "resume" | "end", callOptions: CodemodeCallOptions, ): Promise { return await callStructured( this.client, PATH.browserLifecycle, { browserSessionId: this.id, action }, callOptions, ); } } export class CodemodeBrowserDownloadCollection { constructor( private readonly client: CodemodeClientProvider, private readonly browserSessionId: string, ) {} async list(callOptions: CodemodeCallOptions = {}): Promise { return await callStructured( this.client, PATH.browserDownloads, { browserSessionId: this.browserSessionId, operation: "list" }, callOptions, ); } download(downloadId: string): CodemodeBrowserDownload { return new CodemodeBrowserDownload(this.client, this.browserSessionId, downloadId); } } export class CodemodeBrowserDownload { constructor( private readonly client: CodemodeClientProvider, private readonly browserSessionId: string, readonly id: string, ) {} async get(callOptions: CodemodeCallOptions = {}): Promise { return await callStructured( this.client, PATH.browserDownloads, { browserSessionId: this.browserSessionId, operation: "get", downloadId: this.id }, callOptions, ); } async saveToWorkspace( destinationPath: string, options: { overwrite?: boolean | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await callStructured( this.client, PATH.browserDownloadSave, { browserSessionId: this.browserSessionId, downloadId: this.id, destinationPath, ...(options.overwrite === undefined ? {} : { overwrite: options.overwrite }), }, callOptions, ); } } export class CodemodeBrowserClipboard { constructor( private readonly client: CodemodeClientProvider, private readonly browserSessionId: string, private readonly tabs: CodemodeBrowserTabCollection, ) {} async read(callOptions: CodemodeCallOptions = {}): Promise { return await callStructured( this.client, PATH.browserClipboard, { browserSessionId: this.browserSessionId }, callOptions, ); } async write( text: string, options: BrowserActionFences & { targetId?: string | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.act({ type: "clipboard", operation: "write", text }, options, callOptions); } async clear( options: BrowserActionFences & { targetId?: string | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.act({ type: "clipboard", operation: "clear" }, options, callOptions); } async copy( options: BrowserActionFences & { targetId?: string | undefined; locator?: BrowserLocator | undefined; content?: "selection" | "value" | "text" | undefined; } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const { targetId, expectedTargetGeneration, expectedDocumentGeneration, expectedFrameId, ...copy } = options; return await this.act( { type: "clipboard", operation: "copy", ...copy }, { targetId, expectedTargetGeneration, expectedDocumentGeneration, expectedFrameId, }, callOptions, ); } async paste( options: BrowserActionFences & { targetId?: string | undefined; locator?: BrowserLocator | undefined; text?: string | undefined; } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const { targetId, expectedTargetGeneration, expectedDocumentGeneration, expectedFrameId, ...paste } = options; return await this.act( { type: "clipboard", operation: "paste", ...paste }, { targetId, expectedTargetGeneration, expectedDocumentGeneration, expectedFrameId, }, callOptions, ); } private async act( action: BrowserAction, options: BrowserActionFences & { targetId?: string | undefined }, callOptions: CodemodeCallOptions, ): Promise { const { targetId, ...fences } = options; return await this.tabs .use(await this.tabs.resolveId(targetId, callOptions)) .act(action, fences, callOptions); } } export class CodemodeBrowserTabCollection { constructor( private readonly client: CodemodeClientProvider, private readonly browserSessionId: string, ) {} async list(callOptions: CodemodeCallOptions = {}): Promise { return await callStructured( this.client, PATH.browserTabs, { operation: "list", browserSessionId: this.browserSessionId }, callOptions, ); } async selected(callOptions: CodemodeCallOptions = {}): Promise { return this.use(await this.resolveId(undefined, callOptions)); } use(targetId: string): CodemodeBrowserTab { return new CodemodeBrowserTab(this.client, this.browserSessionId, targetId); } async open(url?: string, callOptions: CodemodeCallOptions = {}): Promise { const before = await this.list(callOptions); const after = await callStructured( this.client, PATH.browserTabs, { operation: "open", browserSessionId: this.browserSessionId, ...(url ? { url } : {}), }, callOptions, ); const beforeIds = new Set(before.targets.map((target) => target.id)); const opened = after.targets.find((target) => !beforeIds.has(target.id)) ?? after.targets.find((target) => target.selected); if (!opened) throw new Error("Browser opened no discoverable tab"); return this.use(opened.id); } async select( targetId: string, callOptions: CodemodeCallOptions = {}, ): Promise { const after = await callStructured( this.client, PATH.browserTabs, { operation: "select", browserSessionId: this.browserSessionId, targetId }, callOptions, ); if (!after.targets.some((target) => target.id === targetId && target.selected)) { throw new Error("Browser did not select the requested tab"); } return this.use(targetId); } async close( targetId: string, callOptions: CodemodeCallOptions = {}, ): Promise { return await callStructured( this.client, PATH.browserTabs, { operation: "close", browserSessionId: this.browserSessionId, targetId }, callOptions, ); } async resolveId(targetId: string | undefined, callOptions: CodemodeCallOptions): Promise { const listed = await this.list(callOptions); if (targetId) { if (!listed.targets.some((target) => target.id === targetId)) { throw new Error(`Browser tab is unavailable: ${targetId}`); } return targetId; } const selected = listed.targets.filter((target) => target.selected); if (selected.length === 1) return selected[0]!.id; if (listed.targets.length === 1) return listed.targets[0]!.id; if (listed.targets.length === 0) throw new Error("Browser has no tabs"); throw new Error("Browser target is ambiguous; select an exact tab"); } } export class CodemodeBrowserTab { constructor( private readonly client: CodemodeClientProvider, readonly browserSessionId: string, readonly id: string, ) {} async observe(callOptions: CodemodeCallOptions = {}): Promise { return await callStructured( this.client, PATH.browserObserve, { browserSessionId: this.browserSessionId, targetId: this.id }, callOptions, ); } async observeFull(callOptions: CodemodeCallOptions = {}): Promise { return await callStructured( this.client, PATH.browserObserve, { browserSessionId: this.browserSessionId, targetId: this.id, view: "full" }, callOptions, ); } async read( options: CodemodeBrowserReadOptions = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await callStructured( this.client, PATH.browserRead, { browserSessionId: this.browserSessionId, targetId: this.id, ...options }, callOptions, ); } async screenshot( options: CodemodeBrowserScreenshotOptions = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const result = await ( await this.client() ).callPath( PATH.browserScreenshot, { browserSessionId: this.browserSessionId, targetId: this.id, ...(options.fullPage === undefined ? {} : { fullPage: options.fullPage }), ...(options.quality === undefined ? {} : { quality: options.quality }), }, callOptions, ); return await materializeBrowserScreenshot(result, options.saveTo); } async act( action: BrowserAction, fences: CodemodeBrowserActionOptions = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await callStructured( this.client, PATH.browserAct, { browserSessionId: this.browserSessionId, targetId: this.id, action, ...fences }, callOptions, ); } /** Navigate this tab. `waitUntil` also waits for that load state, which is * useful before a screenshot of a page that loads its data after `load`. */ async navigate( url: string, options: CodemodeCallOptions & { waitUntil?: "load" | "network_idle" | undefined } = {}, ) { const { waitUntil, ...callOptions } = options; const action = waitUntil ? { type: "navigate", url, waitUntil } : { type: "navigate", url }; return await this.act(action as BrowserAction, {}, callOptions); } async setPermission( permission: BrowserPermission, setting: BrowserPermissionSetting, fences: BrowserActionFences = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.act({ type: "permission", permission, setting }, fences, callOptions); } async requestHuman( reason: string, options: { kind?: "manual_login" | "mfa" | "external_action" | "confirmation" | "other"; authRunId?: string | undefined; expiresInSeconds?: number | undefined; } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const observation = await this.observe(callOptions); return await callStructured( this.client, PATH.requestHuman, { operation: "request", resourceKind: "browser_session", resourceId: this.browserSessionId, targetId: this.id, expectedControllerGeneration: observation.target.controllerGeneration, expectedTargetGeneration: observation.target.targetGeneration, expectedDocumentGeneration: observation.target.documentGeneration, kind: options.kind ?? "other", reason, ...(options.authRunId ? { authRunId: options.authRunId } : {}), ...(options.expiresInSeconds === undefined ? {} : { expiresInSeconds: options.expiresInSeconds }), }, callOptions, ); } getByRole(role: string, options: { name?: string; exact?: boolean } = {}) { return new CodemodeBrowserLocator(this, { kind: "role", role, ...options }); } getByText(text: string) { return new CodemodeBrowserLocator(this, { kind: "text", text }); } getByLabel(text: string) { return new CodemodeBrowserLocator(this, { kind: "label", text }); } getByPlaceholder(text: string) { return new CodemodeBrowserLocator(this, { kind: "placeholder", text }); } getByTestId(value: string) { return new CodemodeBrowserLocator(this, { kind: "test_id", value }); } locator(selector: string) { return new CodemodeBrowserLocator(this, { kind: "css", selector }); } ref(ref: string) { return new CodemodeBrowserLocator(this, { kind: "ref", ref }); } } export class CodemodeBrowserLocator { constructor( private readonly tab: CodemodeBrowserTab, readonly locator: BrowserLocator, ) {} async click( options: { button?: "left" | "right" | "middle" } = {}, callOptions: CodemodeCallOptions = {}, ) { return await this.tab.act( { type: "click", locator: this.locator, ...options }, {}, callOptions, ); } async doubleClick(callOptions: CodemodeCallOptions = {}) { return await this.tab.act({ type: "double_click", locator: this.locator }, {}, callOptions); } async hover(callOptions: CodemodeCallOptions = {}) { return await this.tab.act({ type: "hover", locator: this.locator }, {}, callOptions); } async fill(value: string, callOptions: CodemodeCallOptions = {}) { return await this.tab.act({ type: "fill", locator: this.locator, value }, {}, callOptions); } async type(text: string, callOptions: CodemodeCallOptions = {}) { return await this.tab.act({ type: "type", locator: this.locator, text }, {}, callOptions); } async press(key: string, callOptions: CodemodeCallOptions = {}) { return await this.tab.act({ type: "press", locator: this.locator, key }, {}, callOptions); } async select(values: string[], callOptions: CodemodeCallOptions = {}) { return await this.tab.act({ type: "select", locator: this.locator, values }, {}, callOptions); } async check(checked = true, callOptions: CodemodeCallOptions = {}) { return await this.tab.act({ type: "check", locator: this.locator, checked }, {}, callOptions); } } export class CodemodeAuth { constructor(private readonly client: CodemodeClientProvider) {} async listConnections( options: { includeArchived?: boolean | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "list_connections"; result: SiteAuthConnectionListResponse; }>(this.client, PATH.browserAuth, { operation: "list_connections", ...options }, callOptions); return response.result; } async getConnection( siteAuthConnectionId: string, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "get_connection"; result: SiteAuthConnection; }>( this.client, PATH.browserAuth, { operation: "get_connection", siteAuthConnectionId }, callOptions, ); return response.result; } async listRuns( options: { browserSessionId?: string | undefined; siteAuthConnectionId?: string | undefined; includeSettled?: boolean | undefined; } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "list_runs"; result: AuthRunListResponse }>( this.client, PATH.browserAuth, { operation: "list_runs", ...options }, callOptions, ); return response.result; } async getRun(authRunId: string, callOptions: CodemodeCallOptions = {}): Promise { const response = await callStructured<{ operation: "get_run"; result: AuthRun }>( this.client, PATH.browserAuth, { operation: "get_run", authRunId }, callOptions, ); return response.result; } use(browserSessionId: string, authRunId: string): CodemodeAuthRun { return new CodemodeAuthRun(this.client, browserSessionId, authRunId); } } export class CodemodeBrowserAuth { constructor( private readonly client: CodemodeClientProvider, private readonly browserSessionId: string, ) {} async listRuns( options: { siteAuthConnectionId?: string | undefined; includeSettled?: boolean | undefined; } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "list_runs"; result: AuthRunListResponse }>( this.client, PATH.browserAuth, { operation: "list_runs", browserSessionId: this.browserSessionId, ...options }, callOptions, ); return response.result; } async start( request: Omit, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "start"; result: AuthRunMutationResponse }>( this.client, PATH.browserAuth, { operation: "start", browserSessionId: this.browserSessionId, ...request }, callOptions, ); return this.use(response.result.run.id); } use(authRunId: string): CodemodeAuthRun { return new CodemodeAuthRun(this.client, this.browserSessionId, authRunId); } } export class CodemodeAuthRun { constructor( private readonly client: CodemodeClientProvider, readonly browserSessionId: string, readonly id: string, ) {} async get(callOptions: CodemodeCallOptions = {}): Promise { const response = await callStructured<{ operation: "get_run"; result: AuthRun }>( this.client, PATH.browserAuth, { operation: "get_run", authRunId: this.id }, callOptions, ); if (response.result.browserSessionId !== this.browserSessionId) { throw new Error("Auth run belongs to another BrowserSession"); } return response.result; } async report( request: Omit, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "report"; result: AuthRunMutationResponse }>( this.client, PATH.browserAuth, { operation: "report", browserSessionId: this.browserSessionId, authRunId: this.id, ...request, }, callOptions, ); return response.result; } async advanceExternal( request: Omit, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "advance_external"; result: ExternalAuthRunResponse; }>( this.client, PATH.browserAuth, { operation: "advance_external", browserSessionId: this.browserSessionId, authRunId: this.id, ...request, }, callOptions, ); return response.result; } async verify( request: Omit, callOptions: CodemodeCallOptions = {}, ): Promise { const response = await callStructured<{ operation: "verify"; result: AuthRunMutationResponse }>( this.client, PATH.browserAuth, { operation: "verify", browserSessionId: this.browserSessionId, authRunId: this.id, ...request, }, callOptions, ); return response.result; } } export class CodemodeComputerCollection { constructor(private readonly client: CodemodeClientProvider) {} async list( options: { includeTerminal?: boolean | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return ( await callStructured(this.client, PATH.discover, options, callOptions) ).computers; } async open( options: CodemodeComputerOpenOptions = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const opened = await callStructured<{ session: ComputerSession; targets: ComputerTarget[] }>( this.client, PATH.computerOpen, options, callOptions, ); return new CodemodeComputer(this.client, opened.session.id); } use(computerSessionId: string): CodemodeComputer { return new CodemodeComputer(this.client, computerSessionId); } } export class CodemodeComputer { readonly targets: CodemodeComputerTargetCollection; readonly apps: CodemodeComputerTargetCollection; readonly clipboard: CodemodeComputerClipboard; constructor( private readonly client: CodemodeClientProvider, readonly id: string, ) { this.targets = new CodemodeComputerTargetCollection(client, id); this.apps = this.targets; this.clipboard = new CodemodeComputerClipboard(client, id, this.targets); } async refresh(callOptions: CodemodeCallOptions = {}): Promise { const result = await callStructured<{ session: ComputerSession; targets: ComputerTarget[] }>( this.client, PATH.computerOpen, { computerSessionId: this.id }, callOptions, ); return result.session; } async observe( targetId?: string, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.targets .use(await this.targets.resolveId(targetId, callOptions)) .observe(callOptions); } async act( action: ComputerAction, options: ComputerActionFences & { targetId?: string | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.targets .use(await this.targets.resolveId(options.targetId, callOptions)) .act(action, options, callOptions); } async launch(applicationId: string, callOptions: CodemodeCallOptions = {}) { const targetId = await this.targets.resolveId(undefined, callOptions); await this.targets.use(targetId).act({ type: "launch", applicationId }, {}, callOptions); return await this.targets.list(callOptions); } async end(callOptions: CodemodeCallOptions = {}): Promise { return await callStructured( this.client, PATH.computerLifecycle, { computerSessionId: this.id, action: "end" }, callOptions, ); } } export class CodemodeComputerClipboard { constructor( private readonly client: CodemodeClientProvider, readonly computerSessionId: string, private readonly targets: CodemodeComputerTargetCollection, ) {} async read(callOptions: CodemodeCallOptions = {}): Promise { return await callStructured( this.client, PATH.computerClipboard, { computerSessionId: this.computerSessionId }, callOptions, ); } async write( text: string, options: { targetId?: string | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.mutate({ type: "clipboard", operation: "write", text }, options, callOptions); } async clear( options: { targetId?: string | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.mutate({ type: "clipboard", operation: "clear" }, options, callOptions); } async copy( options: { targetId?: string | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.mutate({ type: "clipboard", operation: "copy" }, options, callOptions); } async paste( options: { targetId?: string | undefined } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await this.mutate({ type: "clipboard", operation: "paste" }, options, callOptions); } private async mutate( action: Extract, options: { targetId?: string | undefined }, callOptions: CodemodeCallOptions, ): Promise { const targetId = await this.targets.resolveSeatId(options.targetId, callOptions); return await this.targets.use(targetId).act(action, {}, callOptions); } } export class CodemodeComputerTargetCollection { constructor( private readonly client: CodemodeClientProvider, private readonly computerSessionId: string, ) {} async list(callOptions: CodemodeCallOptions = {}): Promise { return await callStructured( this.client, PATH.computerTargets, { computerSessionId: this.computerSessionId }, callOptions, ); } use(targetId: string): CodemodeComputerTarget { return new CodemodeComputerTarget(this.client, this.computerSessionId, targetId); } async focused(callOptions: CodemodeCallOptions = {}): Promise { return this.use(await this.resolveId(undefined, callOptions)); } async resolveId(targetId: string | undefined, callOptions: CodemodeCallOptions): Promise { const listed = await this.list(callOptions); if (targetId) { if (!listed.targets.some((target) => target.id === targetId)) { throw new Error(`Computer target is unavailable: ${targetId}`); } return targetId; } const focused = listed.targets.filter((target) => target.focused); if (focused.length === 1) return focused[0]!.id; if (listed.targets.length === 1) return listed.targets[0]!.id; if (listed.targets.length === 0) throw new Error("Computer has no app, window, or screen targets"); throw new Error("Computer target is ambiguous; select an exact app or window"); } async resolveSeatId( targetId: string | undefined, callOptions: CodemodeCallOptions, ): Promise { if (targetId) return await this.resolveId(targetId, callOptions); const listed = await this.list(callOptions); const screens = listed.targets.filter((target) => target.kind === "screen"); if (screens.length === 1) return screens[0]!.id; return await this.resolveId(undefined, callOptions); } } export class CodemodeComputerTarget { constructor( private readonly client: CodemodeClientProvider, readonly computerSessionId: string, readonly id: string, ) {} async observe(callOptions: CodemodeCallOptions = {}): Promise { return await callStructured( this.client, PATH.computerObserve, { computerSessionId: this.computerSessionId, targetId: this.id }, callOptions, ); } async act( action: ComputerAction, fences: ComputerActionFences = {}, callOptions: CodemodeCallOptions = {}, ): Promise { return await callStructured( this.client, PATH.computerAct, { computerSessionId: this.computerSessionId, targetId: this.id, action, ...fences }, callOptions, ); } async requestHuman( reason: string, options: { kind?: "manual_login" | "mfa" | "external_action" | "confirmation" | "other"; expiresInSeconds?: number | undefined; } = {}, callOptions: CodemodeCallOptions = {}, ): Promise { const observation = await this.observe(callOptions); return await callStructured( this.client, PATH.requestHuman, { operation: "request", resourceKind: "computer_session", resourceId: this.computerSessionId, targetId: this.id, expectedControllerGeneration: observation.target.controllerGeneration, expectedTargetGeneration: observation.target.targetGeneration, expectedDocumentGeneration: null, kind: options.kind ?? "other", reason, ...(options.expiresInSeconds === undefined ? {} : { expiresInSeconds: options.expiresInSeconds }), }, callOptions, ); } getByRole(role: string, options: { name?: string; exact?: boolean } = {}) { return new CodemodeComputerLocator(this, { kind: "role", role, ...options }); } getByText(text: string, exact?: boolean) { return new CodemodeComputerLocator(this, { kind: "text", text, ...(exact ? { exact } : {}) }); } getByLabel(text: string, exact?: boolean) { return new CodemodeComputerLocator(this, { kind: "label", text, ...(exact ? { exact } : {}) }); } getByIdentifier(value: string) { return new CodemodeComputerLocator(this, { kind: "identifier", value }); } ref(ref: string) { return new CodemodeComputerLocator(this, { kind: "ref", ref }); } } export class CodemodeComputerLocator { constructor( private readonly target: CodemodeComputerTarget, readonly locator: ComputerLocator, ) {} async invoke(callOptions: CodemodeCallOptions = {}) { return await this.semantic("invoke", undefined, callOptions); } async focus(callOptions: CodemodeCallOptions = {}) { return await this.semantic("focus", undefined, callOptions); } async setValue(value: string | number | boolean, callOptions: CodemodeCallOptions = {}) { return await this.semantic("set_value", value, callOptions); } async select(callOptions: CodemodeCallOptions = {}) { return await this.semantic("select", undefined, callOptions); } async expand(callOptions: CodemodeCallOptions = {}) { return await this.semantic("expand", undefined, callOptions); } async collapse(callOptions: CodemodeCallOptions = {}) { return await this.semantic("collapse", undefined, callOptions); } private async semantic( action: Extract["action"], value: string | number | boolean | undefined, callOptions: CodemodeCallOptions, ) { return await this.target.act( { type: "semantic", locator: this.locator, action, ...(value === undefined ? {} : { value }), }, {}, callOptions, ); } } /** Keep screenshot pixels out of Code Mode text output; `view_image` reads the local file. */ async function materializeBrowserScreenshot( result: AttemptToolResult, saveTo: string | undefined, ): Promise { if (result.isError) throw new CodemodeToolExecutionError(result); if (!result.structuredContent) { throw new Error("Codemode tool interaction.browser.screenshot returned no structured content"); } const image = result.content.find((content) => content.type === "image"); if ( typeof result.structuredContent.mediaType === "string" && image && result.structuredContent.mediaType !== image.mimeType ) { throw new Error("Codemode browser screenshot image type differs from frame metadata"); } const images = await materializeCodemodeImages(result, { ...(saveTo === undefined ? {} : { saveTo }), requireOne: true, }); return { ...images[0]!, metadata: result.structuredContent as CodemodeBrowserScreenshotMetadata, }; } export function createOpenGeniCodemode( client: CodemodeClient | CodemodeClientProvider = () => environmentCodemodeClient(), ) { return new OpenGeniCodemode(client); } export const openGeni = createOpenGeniCodemode();