export type TModelAuthMode = 'api-key' | 'siwc'; export const MIDSCENE_MODEL_AUTH_MODE = 'MIDSCENE_MODEL_AUTH_MODE'; export const MIDSCENE_PLANNING_MODEL_AUTH_MODE = 'MIDSCENE_PLANNING_MODEL_AUTH_MODE'; export const MIDSCENE_LOCATE_MODEL_AUTH_MODE = 'MIDSCENE_LOCATE_MODEL_AUTH_MODE'; export const MIDSCENE_INSIGHT_MODEL_AUTH_MODE = 'MIDSCENE_INSIGHT_MODEL_AUTH_MODE'; // config keys export const MIDSCENE_MODEL_INIT_CONFIG_JSON = 'MIDSCENE_MODEL_INIT_CONFIG_JSON'; export const MIDSCENE_MODEL_EXTRA_BODY_JSON = 'MIDSCENE_MODEL_EXTRA_BODY_JSON'; export const MIDSCENE_MODEL_NAME = 'MIDSCENE_MODEL_NAME'; export const MIDSCENE_DEBUG_MODEL_PROFILE = 'MIDSCENE_DEBUG_MODEL_PROFILE'; export const MIDSCENE_DEBUG_MODEL_RESPONSE = 'MIDSCENE_DEBUG_MODEL_RESPONSE'; export const MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG = 'MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG'; export const MIDSCENE_DEBUG_MODE = 'MIDSCENE_DEBUG_MODE'; export const MIDSCENE_CHROME_PATH = 'MIDSCENE_CHROME_PATH'; /** * @deprecated Use MIDSCENE_CHROME_PATH instead. This is kept for backward compatibility. */ export const MIDSCENE_MCP_CHROME_PATH = 'MIDSCENE_MCP_CHROME_PATH'; export const DOCKER_CONTAINER = 'DOCKER_CONTAINER'; // Observability export const MIDSCENE_LANGSMITH_DEBUG = 'MIDSCENE_LANGSMITH_DEBUG'; export const MIDSCENE_LANGFUSE_DEBUG = 'MIDSCENE_LANGFUSE_DEBUG'; export const MIDSCENE_MODEL_SOCKS_PROXY = 'MIDSCENE_MODEL_SOCKS_PROXY'; export const MIDSCENE_MODEL_HTTP_PROXY = 'MIDSCENE_MODEL_HTTP_PROXY'; // New primary names for public API export const MIDSCENE_MODEL_API_KEY = 'MIDSCENE_MODEL_API_KEY'; export const MIDSCENE_MODEL_BASE_URL = 'MIDSCENE_MODEL_BASE_URL'; export const MIDSCENE_MODEL_TIMEOUT = 'MIDSCENE_MODEL_TIMEOUT'; export const MIDSCENE_MODEL_TEMPERATURE = 'MIDSCENE_MODEL_TEMPERATURE'; export const MIDSCENE_MODEL_RETRY_COUNT = 'MIDSCENE_MODEL_RETRY_COUNT'; export const MIDSCENE_MODEL_RETRY_INTERVAL = 'MIDSCENE_MODEL_RETRY_INTERVAL'; export const MIDSCENE_MODEL_REASONING_EFFORT = 'MIDSCENE_MODEL_REASONING_EFFORT'; export const MIDSCENE_MODEL_REASONING_ENABLED = 'MIDSCENE_MODEL_REASONING_ENABLED'; export const MIDSCENE_MODEL_REASONING_BUDGET = 'MIDSCENE_MODEL_REASONING_BUDGET'; export const MIDSCENE_MODEL_PROTOCOL = 'MIDSCENE_MODEL_PROTOCOL'; export const MIDSCENE_LOCATE_MODEL_STREAM_MODE = 'MIDSCENE_LOCATE_MODEL_STREAM_MODE'; export const MIDSCENE_PLANNING_MODEL_STREAM_MODE = 'MIDSCENE_PLANNING_MODEL_STREAM_MODE'; export const MIDSCENE_INSIGHT_MODEL_STREAM_MODE = 'MIDSCENE_INSIGHT_MODEL_STREAM_MODE'; export const MIDSCENE_MODEL_STREAM_MODE = 'MIDSCENE_MODEL_STREAM_MODE'; export const MIDSCENE_MODEL_RESPONSE_FORMAT = 'MIDSCENE_MODEL_RESPONSE_FORMAT'; export type TModelReasoningEnabled = boolean | 'default'; export type TModelProtocol = 'openai-chat' | 'openai-responses'; export type TModelResponseFormat = 'none' | 'auto'; /** * @deprecated Use MIDSCENE_MODEL_API_KEY instead. This is kept for backward compatibility. */ export const OPENAI_API_KEY = 'OPENAI_API_KEY'; /** * @deprecated Use MIDSCENE_MODEL_BASE_URL instead. This is kept for backward compatibility. */ export const OPENAI_BASE_URL = 'OPENAI_BASE_URL'; /** * @deprecated Use MIDSCENE_MODEL_INIT_CONFIG_JSON instead. This is kept for backward compatibility. */ export const MIDSCENE_OPENAI_INIT_CONFIG_JSON = 'MIDSCENE_OPENAI_INIT_CONFIG_JSON'; /** * @deprecated Use MIDSCENE_MODEL_HTTP_PROXY instead. This is kept for backward compatibility. */ export const MIDSCENE_OPENAI_HTTP_PROXY = 'MIDSCENE_OPENAI_HTTP_PROXY'; /** * @deprecated Use MIDSCENE_MODEL_SOCKS_PROXY instead. This is kept for backward compatibility. */ export const MIDSCENE_OPENAI_SOCKS_PROXY = 'MIDSCENE_OPENAI_SOCKS_PROXY'; export const MIDSCENE_ADB_PATH = 'MIDSCENE_ADB_PATH'; export const MIDSCENE_ADB_REMOTE_HOST = 'MIDSCENE_ADB_REMOTE_HOST'; export const MIDSCENE_ADB_REMOTE_PORT = 'MIDSCENE_ADB_REMOTE_PORT'; export const MIDSCENE_ANDROID_IME_STRATEGY = 'MIDSCENE_ANDROID_IME_STRATEGY'; export const MIDSCENE_ANDROID_SCREENSHOT_STRATEGY = 'MIDSCENE_ANDROID_SCREENSHOT_STRATEGY'; export const MIDSCENE_IOS_DEVICE_UDID = 'MIDSCENE_IOS_DEVICE_UDID'; export const MIDSCENE_IOS_SIMULATOR_UDID = 'MIDSCENE_IOS_SIMULATOR_UDID'; export const MIDSCENE_IOS_DEVICE_CLASS_OVERRIDE = 'MIDSCENE_IOS_DEVICE_CLASS_OVERRIDE'; export const MIDSCENE_CACHE = 'MIDSCENE_CACHE'; export const MIDSCENE_USE_VLM_UI_TARS = 'MIDSCENE_USE_VLM_UI_TARS'; export const MIDSCENE_USE_QWEN_VL = 'MIDSCENE_USE_QWEN_VL'; export const MIDSCENE_USE_QWEN3_VL = 'MIDSCENE_USE_QWEN3_VL'; export const MIDSCENE_USE_DOUBAO_VISION = 'MIDSCENE_USE_DOUBAO_VISION'; export const MIDSCENE_USE_GEMINI = 'MIDSCENE_USE_GEMINI'; export const MIDSCENE_USE_VL_MODEL = 'MIDSCENE_USE_VL_MODEL'; export const MATCH_BY_POSITION = 'MATCH_BY_POSITION'; export const MIDSCENE_REPORT_TAG_NAME = 'MIDSCENE_REPORT_TAG_NAME'; export const MIDSCENE_REPORT_QUIET = 'MIDSCENE_REPORT_QUIET'; export const MIDSCENE_PREFERRED_LANGUAGE = 'MIDSCENE_PREFERRED_LANGUAGE'; export const MIDSCENE_CACHE_MAX_FILENAME_LENGTH = 'MIDSCENE_CACHE_MAX_FILENAME_LENGTH'; export const MIDSCENE_REPLANNING_CYCLE_LIMIT = 'MIDSCENE_REPLANNING_CYCLE_LIMIT'; export const MIDSCENE_RUN_DIR = 'MIDSCENE_RUN_DIR'; export const MIDSCENE_RECORD_MODEL_CALL = 'MIDSCENE_RECORD_MODEL_CALL'; // INSIGHT (visual understanding and extraction) export const MIDSCENE_INSIGHT_MODEL_NAME = 'MIDSCENE_INSIGHT_MODEL_NAME'; export const MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY = 'MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY'; export const MIDSCENE_INSIGHT_MODEL_HTTP_PROXY = 'MIDSCENE_INSIGHT_MODEL_HTTP_PROXY'; export const MIDSCENE_INSIGHT_MODEL_BASE_URL = 'MIDSCENE_INSIGHT_MODEL_BASE_URL'; export const MIDSCENE_INSIGHT_MODEL_API_KEY = 'MIDSCENE_INSIGHT_MODEL_API_KEY'; export const MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON = 'MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON'; export const MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON = 'MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON'; export const MIDSCENE_INSIGHT_MODEL_TIMEOUT = 'MIDSCENE_INSIGHT_MODEL_TIMEOUT'; export const MIDSCENE_INSIGHT_MODEL_TEMPERATURE = 'MIDSCENE_INSIGHT_MODEL_TEMPERATURE'; export const MIDSCENE_INSIGHT_MODEL_RETRY_COUNT = 'MIDSCENE_INSIGHT_MODEL_RETRY_COUNT'; export const MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL = 'MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL'; export const MIDSCENE_INSIGHT_MODEL_FAMILY = 'MIDSCENE_INSIGHT_MODEL_FAMILY'; export const MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT = 'MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT'; export const MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED = 'MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED'; export const MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET = 'MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET'; export const MIDSCENE_INSIGHT_MODEL_PROTOCOL = 'MIDSCENE_INSIGHT_MODEL_PROTOCOL'; export const MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT = 'MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT'; // LOCATE export const MIDSCENE_LOCATE_MODEL_NAME = 'MIDSCENE_LOCATE_MODEL_NAME'; export const MIDSCENE_LOCATE_MODEL_SOCKS_PROXY = 'MIDSCENE_LOCATE_MODEL_SOCKS_PROXY'; export const MIDSCENE_LOCATE_MODEL_HTTP_PROXY = 'MIDSCENE_LOCATE_MODEL_HTTP_PROXY'; export const MIDSCENE_LOCATE_MODEL_BASE_URL = 'MIDSCENE_LOCATE_MODEL_BASE_URL'; export const MIDSCENE_LOCATE_MODEL_API_KEY = 'MIDSCENE_LOCATE_MODEL_API_KEY'; export const MIDSCENE_LOCATE_MODEL_INIT_CONFIG_JSON = 'MIDSCENE_LOCATE_MODEL_INIT_CONFIG_JSON'; export const MIDSCENE_LOCATE_MODEL_EXTRA_BODY_JSON = 'MIDSCENE_LOCATE_MODEL_EXTRA_BODY_JSON'; export const MIDSCENE_LOCATE_MODEL_TIMEOUT = 'MIDSCENE_LOCATE_MODEL_TIMEOUT'; export const MIDSCENE_LOCATE_MODEL_TEMPERATURE = 'MIDSCENE_LOCATE_MODEL_TEMPERATURE'; export const MIDSCENE_LOCATE_MODEL_RETRY_COUNT = 'MIDSCENE_LOCATE_MODEL_RETRY_COUNT'; export const MIDSCENE_LOCATE_MODEL_RETRY_INTERVAL = 'MIDSCENE_LOCATE_MODEL_RETRY_INTERVAL'; export const MIDSCENE_LOCATE_MODEL_FAMILY = 'MIDSCENE_LOCATE_MODEL_FAMILY'; export const MIDSCENE_LOCATE_MODEL_REASONING_EFFORT = 'MIDSCENE_LOCATE_MODEL_REASONING_EFFORT'; export const MIDSCENE_LOCATE_MODEL_REASONING_ENABLED = 'MIDSCENE_LOCATE_MODEL_REASONING_ENABLED'; export const MIDSCENE_LOCATE_MODEL_REASONING_BUDGET = 'MIDSCENE_LOCATE_MODEL_REASONING_BUDGET'; export const MIDSCENE_LOCATE_MODEL_PROTOCOL = 'MIDSCENE_LOCATE_MODEL_PROTOCOL'; export const MIDSCENE_LOCATE_MODEL_RESPONSE_FORMAT = 'MIDSCENE_LOCATE_MODEL_RESPONSE_FORMAT'; // PLANNING export const MIDSCENE_PLANNING_MODEL_NAME = 'MIDSCENE_PLANNING_MODEL_NAME'; export const MIDSCENE_PLANNING_MODEL_SOCKS_PROXY = 'MIDSCENE_PLANNING_MODEL_SOCKS_PROXY'; export const MIDSCENE_PLANNING_MODEL_HTTP_PROXY = 'MIDSCENE_PLANNING_MODEL_HTTP_PROXY'; export const MIDSCENE_PLANNING_MODEL_BASE_URL = 'MIDSCENE_PLANNING_MODEL_BASE_URL'; export const MIDSCENE_PLANNING_MODEL_API_KEY = 'MIDSCENE_PLANNING_MODEL_API_KEY'; export const MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON = 'MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON'; export const MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON = 'MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON'; export const MIDSCENE_PLANNING_MODEL_TIMEOUT = 'MIDSCENE_PLANNING_MODEL_TIMEOUT'; export const MIDSCENE_PLANNING_MODEL_TEMPERATURE = 'MIDSCENE_PLANNING_MODEL_TEMPERATURE'; export const MIDSCENE_PLANNING_MODEL_RETRY_COUNT = 'MIDSCENE_PLANNING_MODEL_RETRY_COUNT'; export const MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL = 'MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL'; export const MIDSCENE_PLANNING_MODEL_FAMILY = 'MIDSCENE_PLANNING_MODEL_FAMILY'; export const MIDSCENE_PLANNING_MODEL_REASONING_EFFORT = 'MIDSCENE_PLANNING_MODEL_REASONING_EFFORT'; export const MIDSCENE_PLANNING_MODEL_REASONING_ENABLED = 'MIDSCENE_PLANNING_MODEL_REASONING_ENABLED'; export const MIDSCENE_PLANNING_MODEL_REASONING_BUDGET = 'MIDSCENE_PLANNING_MODEL_REASONING_BUDGET'; export const MIDSCENE_PLANNING_MODEL_PROTOCOL = 'MIDSCENE_PLANNING_MODEL_PROTOCOL'; export const MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT = 'MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT'; export const MIDSCENE_MODEL_FAMILY = 'MIDSCENE_MODEL_FAMILY'; /** * env keys declared but unused */ export const UNUSED_ENV_KEYS = [MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG]; /** * env keys for debug or basic run * can not be override by overrideAIConfig */ export const BASIC_ENV_KEYS = [ MIDSCENE_DEBUG_MODE, MIDSCENE_DEBUG_MODEL_PROFILE, MIDSCENE_DEBUG_MODEL_RESPONSE, MIDSCENE_RUN_DIR, MIDSCENE_RECORD_MODEL_CALL, ] as const; export const BOOLEAN_ENV_KEYS = [ MIDSCENE_CACHE, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_LANGFUSE_DEBUG, MIDSCENE_REPORT_QUIET, ] as const; export const NUMBER_ENV_KEYS = [ MIDSCENE_CACHE_MAX_FILENAME_LENGTH, MIDSCENE_REPLANNING_CYCLE_LIMIT, ] as const; export const STRING_ENV_KEYS = [ MIDSCENE_ADB_PATH, MIDSCENE_ADB_REMOTE_HOST, MIDSCENE_ADB_REMOTE_PORT, MIDSCENE_ANDROID_IME_STRATEGY, MIDSCENE_ANDROID_SCREENSHOT_STRATEGY, MIDSCENE_IOS_DEVICE_UDID, MIDSCENE_IOS_SIMULATOR_UDID, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_PREFERRED_LANGUAGE, MATCH_BY_POSITION, MIDSCENE_CHROME_PATH, MIDSCENE_MCP_CHROME_PATH, DOCKER_CONTAINER, ] as const; /** * Non model related env keys, used for globally controlling the behavior of midscene * Can not be override by agent.modelConfig but can be override by overrideAIConfig * Can be access at any time */ export const GLOBAL_ENV_KEYS = [ ...BOOLEAN_ENV_KEYS, ...NUMBER_ENV_KEYS, ...STRING_ENV_KEYS, ] as const; /** * Model related eve keys, used for declare which model to use. * Can be override by both agent.modelConfig and overrideAIConfig * Can only be access after agent.constructor */ export const MODEL_ENV_KEYS = [ MIDSCENE_MODEL_AUTH_MODE, MIDSCENE_PLANNING_MODEL_AUTH_MODE, MIDSCENE_LOCATE_MODEL_AUTH_MODE, MIDSCENE_INSIGHT_MODEL_AUTH_MODE, MIDSCENE_INSIGHT_MODEL_STREAM_MODE, MIDSCENE_PLANNING_MODEL_STREAM_MODE, MIDSCENE_LOCATE_MODEL_STREAM_MODE, MIDSCENE_MODEL_STREAM_MODE, // model default MIDSCENE_MODEL_NAME, MIDSCENE_MODEL_INIT_CONFIG_JSON, MIDSCENE_MODEL_EXTRA_BODY_JSON, MIDSCENE_MODEL_API_KEY, MIDSCENE_MODEL_BASE_URL, MIDSCENE_MODEL_SOCKS_PROXY, MIDSCENE_MODEL_HTTP_PROXY, MIDSCENE_MODEL_TIMEOUT, MIDSCENE_MODEL_TEMPERATURE, MIDSCENE_MODEL_RETRY_COUNT, MIDSCENE_MODEL_RETRY_INTERVAL, MIDSCENE_MODEL_REASONING_EFFORT, MIDSCENE_MODEL_REASONING_ENABLED, MIDSCENE_MODEL_REASONING_BUDGET, MIDSCENE_MODEL_PROTOCOL, MIDSCENE_MODEL_RESPONSE_FORMAT, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_QWEN3_VL, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_VL_MODEL, // model default legacy OPENAI_API_KEY, OPENAI_BASE_URL, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_SOCKS_PROXY, // INSIGHT (visual understanding and extraction) MIDSCENE_INSIGHT_MODEL_NAME, MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY, MIDSCENE_INSIGHT_MODEL_HTTP_PROXY, MIDSCENE_INSIGHT_MODEL_BASE_URL, MIDSCENE_INSIGHT_MODEL_API_KEY, MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON, MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON, MIDSCENE_INSIGHT_MODEL_TIMEOUT, MIDSCENE_INSIGHT_MODEL_TEMPERATURE, MIDSCENE_INSIGHT_MODEL_RETRY_COUNT, MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL, MIDSCENE_INSIGHT_MODEL_FAMILY, MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT, MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED, MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET, MIDSCENE_INSIGHT_MODEL_PROTOCOL, MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT, // LOCATE MIDSCENE_LOCATE_MODEL_NAME, MIDSCENE_LOCATE_MODEL_SOCKS_PROXY, MIDSCENE_LOCATE_MODEL_HTTP_PROXY, MIDSCENE_LOCATE_MODEL_BASE_URL, MIDSCENE_LOCATE_MODEL_API_KEY, MIDSCENE_LOCATE_MODEL_INIT_CONFIG_JSON, MIDSCENE_LOCATE_MODEL_EXTRA_BODY_JSON, MIDSCENE_LOCATE_MODEL_TIMEOUT, MIDSCENE_LOCATE_MODEL_TEMPERATURE, MIDSCENE_LOCATE_MODEL_RETRY_COUNT, MIDSCENE_LOCATE_MODEL_RETRY_INTERVAL, MIDSCENE_LOCATE_MODEL_FAMILY, MIDSCENE_LOCATE_MODEL_REASONING_EFFORT, MIDSCENE_LOCATE_MODEL_REASONING_ENABLED, MIDSCENE_LOCATE_MODEL_REASONING_BUDGET, MIDSCENE_LOCATE_MODEL_PROTOCOL, MIDSCENE_LOCATE_MODEL_RESPONSE_FORMAT, // PLANNING MIDSCENE_PLANNING_MODEL_NAME, MIDSCENE_PLANNING_MODEL_SOCKS_PROXY, MIDSCENE_PLANNING_MODEL_HTTP_PROXY, MIDSCENE_PLANNING_MODEL_BASE_URL, MIDSCENE_PLANNING_MODEL_API_KEY, MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON, MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON, MIDSCENE_PLANNING_MODEL_TIMEOUT, MIDSCENE_PLANNING_MODEL_TEMPERATURE, MIDSCENE_PLANNING_MODEL_RETRY_COUNT, MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL, MIDSCENE_PLANNING_MODEL_FAMILY, MIDSCENE_PLANNING_MODEL_REASONING_EFFORT, MIDSCENE_PLANNING_MODEL_REASONING_ENABLED, MIDSCENE_PLANNING_MODEL_REASONING_BUDGET, MIDSCENE_PLANNING_MODEL_PROTOCOL, MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT, MIDSCENE_MODEL_FAMILY, ] as const; export const ALL_ENV_KEYS = [ ...UNUSED_ENV_KEYS, ...BASIC_ENV_KEYS, ...GLOBAL_ENV_KEYS, ...MODEL_ENV_KEYS, ] as const; export type TEnvKeys = (typeof ALL_ENV_KEYS)[number]; export type TGlobalConfig = Record; /** * valid Model family types */ export type TModelFamily = | 'jev' | 'qwen2.5-vl' | 'qwen3-vl' | 'qwen3' | 'qwen3.5' | 'qwen3.6' | 'doubao-vision' | 'doubao-seed' | 'gemini' | 'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5' | 'glm-v' | 'auto-glm' | 'auto-glm-multilingual' | 'gpt-5' | 'gpt-6' | 'deepseek' | 'kimi' | 'kimi3' | 'xiaomi-mimo'; export const MODEL_FAMILY_VALUES: TModelFamily[] = [ 'jev', 'doubao-vision', 'doubao-seed', 'gemini', 'qwen2.5-vl', 'qwen3-vl', 'qwen3', 'qwen3.5', 'qwen3.6', 'vlm-ui-tars', 'vlm-ui-tars-doubao', 'vlm-ui-tars-doubao-1.5', 'glm-v', 'auto-glm', 'auto-glm-multilingual', 'gpt-5', 'gpt-6', 'deepseek', 'kimi', 'kimi3', 'xiaomi-mimo', ]; export type TModelStreamMode = 'stream' | 'non-stream'; export interface IModelConfigForInsight { // model name [MIDSCENE_INSIGHT_MODEL_NAME]: string; // proxy [MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY]?: string; [MIDSCENE_INSIGHT_MODEL_HTTP_PROXY]?: string; // OpenAI [MIDSCENE_INSIGHT_MODEL_BASE_URL]?: string; [MIDSCENE_INSIGHT_MODEL_API_KEY]?: string; [MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON]?: string; [MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON]?: string; [MIDSCENE_INSIGHT_MODEL_PROTOCOL]?: TModelProtocol; [MIDSCENE_INSIGHT_MODEL_STREAM_MODE]?: TModelStreamMode; [MIDSCENE_INSIGHT_MODEL_AUTH_MODE]?: TModelAuthMode; // timeout [MIDSCENE_INSIGHT_MODEL_TIMEOUT]?: string; // temperature [MIDSCENE_INSIGHT_MODEL_TEMPERATURE]?: string; // model family [MIDSCENE_INSIGHT_MODEL_FAMILY]?: TModelFamily; } export interface IModelConfigForLocate { // model name [MIDSCENE_LOCATE_MODEL_NAME]: string; // proxy [MIDSCENE_LOCATE_MODEL_SOCKS_PROXY]?: string; [MIDSCENE_LOCATE_MODEL_HTTP_PROXY]?: string; // OpenAI [MIDSCENE_LOCATE_MODEL_BASE_URL]?: string; [MIDSCENE_LOCATE_MODEL_API_KEY]?: string; [MIDSCENE_LOCATE_MODEL_INIT_CONFIG_JSON]?: string; [MIDSCENE_LOCATE_MODEL_EXTRA_BODY_JSON]?: string; [MIDSCENE_LOCATE_MODEL_PROTOCOL]?: TModelProtocol; [MIDSCENE_LOCATE_MODEL_STREAM_MODE]?: TModelStreamMode; [MIDSCENE_LOCATE_MODEL_AUTH_MODE]?: TModelAuthMode; // timeout [MIDSCENE_LOCATE_MODEL_TIMEOUT]?: string; // temperature [MIDSCENE_LOCATE_MODEL_TEMPERATURE]?: string; // model family [MIDSCENE_LOCATE_MODEL_FAMILY]?: TModelFamily; } export interface IModelConfigForPlanning { // model name [MIDSCENE_PLANNING_MODEL_NAME]: string; // proxy [MIDSCENE_PLANNING_MODEL_SOCKS_PROXY]?: string; [MIDSCENE_PLANNING_MODEL_HTTP_PROXY]?: string; // OpenAI [MIDSCENE_PLANNING_MODEL_BASE_URL]?: string; [MIDSCENE_PLANNING_MODEL_API_KEY]?: string; [MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON]?: string; [MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON]?: string; [MIDSCENE_PLANNING_MODEL_PROTOCOL]?: TModelProtocol; [MIDSCENE_PLANNING_MODEL_STREAM_MODE]?: TModelStreamMode; [MIDSCENE_PLANNING_MODEL_AUTH_MODE]?: TModelAuthMode; // timeout [MIDSCENE_PLANNING_MODEL_TIMEOUT]?: string; // temperature [MIDSCENE_PLANNING_MODEL_TEMPERATURE]?: string; // model family [MIDSCENE_PLANNING_MODEL_FAMILY]?: TModelFamily; } /** * Model configuration for Planning intent. * * IMPORTANT: Planning MUST use a vision language model (VL mode). * DOM-based planning is not supported. * * Required: MIDSCENE_MODEL_FAMILY must be set to one of 「TModelFamily」 */ export interface IModelConfigForDefault { // model name [MIDSCENE_MODEL_NAME]: string; // proxy [MIDSCENE_MODEL_SOCKS_PROXY]?: string; [MIDSCENE_MODEL_HTTP_PROXY]?: string; // OpenAI [MIDSCENE_MODEL_BASE_URL]?: string; [MIDSCENE_MODEL_API_KEY]?: string; [MIDSCENE_MODEL_INIT_CONFIG_JSON]?: string; [MIDSCENE_MODEL_EXTRA_BODY_JSON]?: string; [MIDSCENE_MODEL_PROTOCOL]?: TModelProtocol; [MIDSCENE_MODEL_STREAM_MODE]?: TModelStreamMode; [MIDSCENE_MODEL_AUTH_MODE]?: TModelAuthMode; // extra [MIDSCENE_MODEL_FAMILY]?: TModelFamily; // temperature [MIDSCENE_MODEL_TEMPERATURE]?: string; // reasoning effort [MIDSCENE_MODEL_REASONING_EFFORT]?: string; // enable reasoning (boolean/default as string) [MIDSCENE_MODEL_REASONING_ENABLED]?: string; // reasoning budget (number as string) [MIDSCENE_MODEL_REASONING_BUDGET]?: string; // Response format strategy (none/auto) [MIDSCENE_MODEL_RESPONSE_FORMAT]?: TModelResponseFormat; } export interface IModelConfigForDefaultLegacy { // model name [MIDSCENE_MODEL_NAME]: string; // proxy [MIDSCENE_OPENAI_SOCKS_PROXY]?: string; [MIDSCENE_OPENAI_HTTP_PROXY]?: string; // OpenAI [OPENAI_BASE_URL]?: string; [OPENAI_API_KEY]?: string; [MIDSCENE_OPENAI_INIT_CONFIG_JSON]?: string; } /** * - locate: visual element localization * - insight: visual understanding and extraction * - planning: planning * - default: base configuration for all capabilities */ export type TIntent = 'insight' | 'planning' | 'locate' | 'default'; /** * Env-style model configuration map supplied directly to the agent. * Numbers are allowed so callers can pass numeric env values (e.g. limits) without casting. */ export type TModelConfig = Record; export enum UITarsModelVersion { V1_0 = '1.0', V1_5 = '1.5', DOUBAO_1_5_15B = 'doubao-1.5-15B', DOUBAO_1_5_20B = 'doubao-1.5-20B', } /** * Callback to create custom OpenAI client instance * @param config - Resolved model configuration including apiKey, baseURL, modelName, intent, slot, etc. * @returns OpenAI client instance (can be wrapped with langsmith, langfuse, etc.) * * Note: Wrapper functions like langsmith's wrapOpenAI() return the same OpenAI instance * with enhanced behavior, so the return type remains compatible with OpenAI. * * Note: The return type is `any` in the shared package to avoid requiring openai as a dependency. * The actual implementation should return an OpenAI instance. * * @example * ```typescript * import OpenAI from 'openai'; * import { wrapOpenAI } from 'langsmith/wrappers'; * * createOpenAIClient: async (openai, opts) => { * // Wrap with langsmith for planning tasks * if (opts.baseURL?.includes('planning')) { * return wrapOpenAI(openai, { metadata: { task: 'planning' } }); * } * * return openai; * } * ``` */ export type CreateOpenAIClientFn = ( openAIInstance: any, options: Record, ) => Promise; // OpenAI instance, but typed as `any` to avoid dependency export interface IModelConfig { /** * proxy */ socksProxy?: string; httpProxy?: string; /** * model */ modelName: string; /** * OpenAI */ openaiBaseURL?: string; openaiApiKey?: string; openaiExtraConfig?: Record; protocol?: TModelProtocol; /** * Extra body parameters merged into each model request body. * Unlike openaiExtraConfig (which configures the OpenAI client instance), * this is spread directly into the protocol request body. * Example: { "chat_template_kwargs": { "enable_thinking": true } } */ extraBody?: Record; /** * Midscene hard timeout per model request in milliseconds, including body reads. * Must be finite and non-negative. Defaults to 180000 (180 seconds). * Each retry gets a fresh timeout; retry delays are excluded. * Set to 0 to disable only the Midscene hard timeout. * SDK and network timeouts may still apply; callers can cancel via AbortSignal. */ timeout?: number; /** * Temperature for model sampling. */ temperature?: number; streamMode?: TModelStreamMode; authMode?: TModelAuthMode; /** * Number of retries when AI call fails. * Default is 1 (retry once after failure). * Retries occur on HTTP errors or when the model response cannot be * structurally parsed. */ retryCount?: number; /** * Interval between retries in milliseconds. * Default is 2000. */ retryInterval?: number; /** * Reasoning effort level for the model. * Passed through to model-family-specific parameters (e.g., reasoning_effort for doubao, reasoning.effort for gpt-5). */ reasoningEffort?: string; /** * Enable/disable reasoning for the model. * Passed through to model-family-specific parameters (e.g., enable_thinking for qwen, thinking.type for doubao/glm-v). * "default" means following the model provider's default without sending reasoning controls. */ reasoningEnabled?: TModelReasoningEnabled; /** * Reasoning token budget for the model. * Passed through to model-family-specific parameters (e.g., thinking_budget for qwen). */ reasoningBudget?: number; /** * Response format strategy. "auto" lets the model adapter enable a * provider-supported structured response format for eligible intents. */ responseFormat?: TModelResponseFormat; /** * Model family - unified model configuration * Maps directly to model families like 'qwen2.5-vl', 'qwen3-vl', 'doubao-vision', 'doubao-seed', etc. */ modelFamily?: TModelFamily; uiTarsModelVersion?: UITarsModelVersion; modelDescription: string; /** * The semantic intent this config is requested for. * For example, getModelConfig('planning') always returns intent === 'planning'. */ intent: TIntent; /** * The model-config slot this config was resolved from. * For example, getModelConfig('planning') may resolve from slot === 'default' * when MIDSCENE_PLANNING_MODEL_NAME is not configured. */ slot: TIntent; /** * Custom OpenAI client factory function * * If provided, this function will be called to create OpenAI client instances * for each AI call, allowing you to: * - Wrap clients with observability tools (langsmith, langfuse) * - Use custom OpenAI-compatible clients * - Apply different configurations based on intent */ createOpenAIClient?: CreateOpenAIClientFn; }