import { parseBase64 } from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; import type { z } from 'zod'; import { createRecordCliCommand } from '../cli/record-command'; import { camelToKebab, getKeyAliases } from '../key-alias-utils'; import { type CliReportSession, generateCliReportSession, readCliReportSession, writeCliReportSession, } from './cli-report-session'; import { createNamespacedInitArgSchema, extractNamespacedArgs, sanitizeNamespacedArgs, } from './init-arg-utils'; import { type ToolDefaults, mergeToolDefaults } from './tool-defaults'; import { generateCommonTools, generateToolsFromActionSpace, } from './tool-generator'; import type { ActionSpaceItem, BaseAgent, BaseDevice, IMidsceneTools, ToolCliMetadata, ToolDefinition, ToolSchema, } from './types'; const debug = getDebug('agent-tools:base-tools'); /** * Declarative description of a platform's agent init args. * Collapses the `extractAgentInitParam` / `sanitizeToolArgs` / * `getAgentInitArgSchema` trio into a single data declaration. */ export interface InitArgSpec { /** Arg namespace, e.g. `android`, `ios`. */ namespace: string; /** Zod shape describing the init args. Field names drive the tool schema. */ shape: Record; /** * Optional CLI presentation hints. These affect `--help` output for * single-platform CLIs but do not alter YAML protocol keys. */ cli?: { /** Prefer bare `--device-id`-style options in platform CLI help output. */ preferBareKeys?: boolean; /** Override the displayed option name for specific init arg fields. */ preferredNames?: Record; }; /** * Adapt extracted namespaced args into the concrete `TInitParam` passed to * `ensureAgent`. Defaults to returning the raw extracted record. */ adapt?: ( extracted: Record | undefined, ) => TInitParam | undefined; } /** * Base class for platform-specific Midscene tools. * @typeParam TAgent - Platform-specific agent type. * @typeParam TInitParam - Platform-specific init parameter consumed by * `ensureAgent`. Defaults to `undefined` for platforms that take no args. */ export abstract class BaseMidsceneTools< TAgent extends BaseAgent = BaseAgent, TInitParam = unknown, > implements IMidsceneTools { protected agent?: TAgent; protected toolDefinitions: ToolDefinition[] = []; /** * Default options injected into every generated tool call (e.g. forced deep * locate / deep think). Set from startup/CLI behavior flags before * `initTools()` so they are baked into the generated tool handlers. * See https://github.com/web-infra-dev/midscene/issues/2446. */ protected toolDefaults: ToolDefaults = {}; /** * Declarative init-arg spec. Subclasses that accept CLI init args should * set this once and get `extractAgentInitParam` / `sanitizeToolArgs` / * `getAgentInitArgSchema` auto-implemented. * * Declared with `declare` so that TS doesn't emit an `Object.defineProperty` * for this field on the base constructor, which would otherwise overwrite * a subclass field initializer under `useDefineForClassFields`. */ protected declare readonly initArgSpec?: InitArgSpec; /** * Ensure agent is initialized and ready for use. * Must be implemented by subclasses to create platform-specific agent. * @param initParam Optional initialization parameter (platform-specific, e.g., URL, device ID) * @returns Promise resolving to initialized agent instance * @throws Error if agent initialization fails */ protected abstract ensureAgent(initParam?: TInitParam): Promise; private getInitArgKeys(): readonly string[] { return this.initArgSpec ? Object.keys(this.initArgSpec.shape) : []; } /** * Extract a platform-specific agent init parameter from CLI tool args. */ protected extractAgentInitParam( args: Record, ): TInitParam | undefined { if (!this.initArgSpec) { return undefined; } const extracted = extractNamespacedArgs( args, this.initArgSpec.namespace, this.getInitArgKeys(), ); if (this.initArgSpec.adapt) { return this.initArgSpec.adapt(extracted); } return extracted as TInitParam | undefined; } /** * Remove platform-specific init args before dispatching a tool payload to the action itself. */ protected sanitizeToolArgs( args: Record, ): Record { if (!this.initArgSpec) { return args; } return sanitizeNamespacedArgs( args, this.initArgSpec.namespace, this.getInitArgKeys(), ); } /** * Expose platform-specific init args on action/common tool schemas. */ protected getAgentInitArgSchema(): ToolSchema { if (!this.initArgSpec) { return {}; } return createNamespacedInitArgSchema( this.initArgSpec.namespace, this.initArgSpec.shape, ); } /** * Expose CLI-only metadata for platform init args so single-platform help can * show ergonomic bare flags while the underlying schema stays namespaced. * When `preferBareKeys` is enabled, single-platform CLIs only accept the * bare spellings; namespaced dotted spellings remain available through the * YAML schema instead of the platform CLI surface. */ protected getAgentInitArgCliMetadata(): ToolCliMetadata | undefined { if (!this.initArgSpec?.cli) { return undefined; } const options = Object.fromEntries( this.getInitArgKeys().map((key) => { const canonicalKey = `${this.initArgSpec!.namespace}.${key}`; const preferredName = this.initArgSpec!.cli?.preferredNames?.[key] ?? (this.initArgSpec!.cli?.preferBareKeys ? camelToKebab(key) : canonicalKey); const acceptedNames = new Set([ preferredName, ...(this.initArgSpec!.cli?.preferBareKeys ? getKeyAliases(key) : getKeyAliases(canonicalKey)), ]); acceptedNames.delete(preferredName); return [ canonicalKey, { preferredName, aliases: [...acceptedNames], }, ]; }), ); return { options }; } /** * Optional: prepare platform-specific tools (e.g., device connection) */ protected preparePlatformTools(): ToolDefinition[] { return []; } protected getCliReportSessionName(): string | undefined { return undefined; } protected createNewCliReportSession( targetIdentity?: string, ): CliReportSession | undefined { const sessionName = this.getCliReportSessionName(); if (!sessionName) { return undefined; } return generateCliReportSession(sessionName, targetIdentity); } protected commitCliReportSession(session?: CliReportSession): void { if (session) { writeCliReportSession(session); } } protected readCliReportFileName(): string | undefined { const sessionName = this.getCliReportSessionName(); if (!sessionName) { return undefined; } return readCliReportSession(sessionName)?.reportFileName; } protected readCliReportAgentOptions(): | { reportFileName: string; reportAttributes: Record; } | undefined { const reportFileName = this.readCliReportFileName(); if (!reportFileName) { return undefined; } return { reportFileName, reportAttributes: { 'data-group-id': reportFileName, }, }; } /** * Must be implemented by subclasses to create a temporary device instance * This allows getting real actionSpace without connecting to device */ protected abstract createTemporaryDevice(): BaseDevice; /** * Initialize all tools by querying actionSpace * Uses two-layer fallback strategy: * 1. Try to get actionSpace from connected agent (if available) * 2. Create temporary device instance to read actionSpace (always succeeds) */ public async initTools(): Promise { this.toolDefinitions = []; // 1. Add platform-specific tools first (device connection, etc.) // These don't require an agent and should always be available const platformTools = this.preparePlatformTools(); this.toolDefinitions.push(...platformTools); // 2. Get action space: use pre-set agent if available, otherwise temp device. // For CLI usage, agent is deferred to the first real command. let actionSpace: ActionSpaceItem[]; if (this.agent) { actionSpace = await this.agent.getActionSpace(); debug( 'Action space from agent:', actionSpace.map((a) => a.name).join(', '), ); } else { const tempDevice = this.createTemporaryDevice(); actionSpace = tempDevice.actionSpace(); await tempDevice.destroy?.(); debug( 'Action space from temporary device:', actionSpace.map((a) => a.name).join(', '), ); } // 3. Generate tools from action space (core innovation) const actionTools = generateToolsFromActionSpace( actionSpace, (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)), (args = {}) => this.sanitizeToolArgs(args), this.getAgentInitArgSchema(), this.getAgentInitArgCliMetadata(), this.toolDefaults, ); // 4. Add common tools (screenshot, act, assert) const commonTools = generateCommonTools( (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)), this.getAgentInitArgSchema(), this.getAgentInitArgCliMetadata(), this.toolDefaults, ); this.toolDefinitions.push(...actionTools, ...commonTools); debug('Total tools prepared:', this.toolDefinitions.length); } /** * Cleanup method - destroy agent and release resources */ public async destroy(): Promise { await this.agent?.destroy?.(); } /** * Get tool definitions */ public getToolDefinitions(): ToolDefinition[] { return this.toolDefinitions; } /** Commands that exist only on the foreground CLI surface. */ public getCliToolDefinitions(): ToolDefinition[] { return [ createRecordCliCommand( (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)), this.getAgentInitArgSchema(), this.getAgentInitArgCliMetadata(), ), ]; } /** * Set agent for the tools manager */ public setAgent(agent: TAgent): void { this.agent = agent; } /** * Set the default options injected into generated tool calls. Must be called * before `initTools()` because the values are captured into the generated * tool handlers. Merges with any previously set defaults. */ public setToolDefaults(toolDefaults: ToolDefaults): void { this.toolDefaults = mergeToolDefaults(this.toolDefaults, toolDefaults); } /** * Helper: Convert base64 screenshot to image content array */ protected buildScreenshotContent(screenshot: string) { const { mimeType, body } = parseBase64(screenshot); return [ { type: 'image' as const, data: body, mimeType, }, ]; } /** * Helper: Build a simple text result for tool responses */ protected buildTextResult(text: string) { return { content: [{ type: 'text' as const, text }], }; } /** * Create a disconnect handler for releasing platform resources * @param platformName Human-readable platform name for the response message * @returns Handler function that destroys the agent and returns appropriate response */ protected createDisconnectHandler(platformName: string) { return async () => { if (!this.agent) { return this.buildTextResult('No active connection to disconnect'); } try { await this.agent.destroy?.(); } catch (error) { debug('Failed to destroy agent during disconnect:', error); } this.agent = undefined; return this.buildTextResult(`Disconnected from ${platformName}`); }; } }