/** * Route handlers for dictation processing. */ import { z } from "zod"; import { DictationRequestSchema } from "../../api/requests/dictation.js"; import { type ProfileResolution, resolveProfile, } from "../../daemon/dictation-profile-store.js"; import { applyDictionary, expandSnippets, } from "../../daemon/dictation-text-processing.js"; import { detectDictationModeHeuristic } from "../../daemon/handlers/dictation.js"; import type { DictationRequest } from "../../daemon/message-types/diagnostics.js"; import type { DictationContext } from "../../daemon/message-types/shared.js"; import { createTimeout, extractToolUse, getConfiguredProvider, userMessage, } from "../../providers/provider-send-message.js"; import { getLogger } from "../../util/logger.js"; import { ACTOR_PRINCIPALS } from "../auth/route-policy.js"; import { BadRequestError } from "./errors.js"; import type { RouteDefinition, RouteHandlerArgs } from "./types.js"; const log = getLogger("diagnostics-routes"); // --------------------------------------------------------------------------- // Dictation // --------------------------------------------------------------------------- type DictationMode = "dictation" | "command" | "action"; const DICTATION_CLASSIFICATION_TIMEOUT_MS = 5000; const MAX_WINDOW_TITLE_LENGTH = 100; function sanitizeWindowTitle(title: string | undefined): string { if (!title) { return ""; } return title.replace(/[<>]/g, "").slice(0, MAX_WINDOW_TITLE_LENGTH); } interface DictationBody { transcription: string; context: DictationContext; profileId?: string; } function buildAppMetadataBlock(context: DictationContext): string { const windowTitle = sanitizeWindowTitle(context.windowTitle); return [ "", `App: ${context.appName} (${context.bundleIdentifier})`, `Window: ${windowTitle}`, "", ].join("\n"); } function buildCombinedDictationPrompt( body: DictationBody, stylePrompt?: string, ): string { const sections = [ "You are a voice input assistant. You will receive a speech transcription and must:", '1. Classify it as "dictation" (text to insert) or "action" (task for an assistant to execute)', "2. If dictation, clean up the text. If action, return the raw transcription.", "", "## Classification", 'DICTATION examples: "Hey how are you doing", "I think we should move forward with the proposal", "Dear team comma please review the attached document"', 'ACTION examples: "Message Aaron on Slack saying hey what\'s up", "Send an email to the team about the meeting", "Open Spotify and play my playlist", "Search for flights to Denver", "Create a new document in Google Docs"', "", "Key signals for ACTION: the user is addressing an assistant and asking it to DO something (send, message, open, search, create, schedule, etc.)", "Key signals for DICTATION: the user is composing text content that should be typed out as-is", `Cursor in text field: ${body.context.cursorInTextField ? "yes" : "no"} -- if yes, lean toward dictation unless the intent to command is clear.`, "", "## Cleanup Rules (for dictation mode only)", "- Fix grammar, punctuation, and capitalization", "- Remove filler words (um, uh, like, you know)", '- Rewrite vague or hedging language ("so yeah probably", "I guess maybe") into clear, confident statements', "- Maintain the speaker's intent and meaning", ]; if (stylePrompt) { sections.push( "", "## User Style (HIGHEST PRIORITY)", "The user has configured these style preferences. They OVERRIDE the default tone adaptation below.", "Follow these instructions precisely -- they reflect the user's personal writing voice and preferences.", "", stylePrompt, ); } sections.push("", "## Tone Adaptation"); if (stylePrompt) { sections.push( "Use these as fallback guidance only when the User Style above does not cover a specific aspect:", ); } else { sections.push("Adapt your output tone based on the active application:"); } sections.push( "- Email apps (Gmail, Mail): Professional but warm. Use proper greetings and sign-offs if appropriate.", "- Slack: Casual and conversational. Match typical chat style.", "- Code editors (VS Code, Xcode): Technical and concise. Code comments style.", "- Terminal: Command-like, terse.", "- Messages/iMessage: Very casual, texting style. Short sentences.", "- Notes/Docs: Neutral, clear writing.", "- Default: Match the user's natural voice.", "", "## Context Clues", "- Window title may contain recipient name (Slack DMs, email compose)", "- If you can identify a recipient, adapt formality to the apparent relationship", "- Maintain the user's natural voice -- don't over-formalize casual speech", "- The user's writing patterns and preferences may be available from memory context -- follow those when present", "", buildAppMetadataBlock(body.context), ); return sections.join("\n"); } function buildCommandPrompt(body: DictationBody, stylePrompt?: string): string { const sections = [ "You are a text transformation assistant. The user has selected text and given a voice command to transform it.", "", "## Rules", "- Apply the instruction to the selected text", "- Return ONLY the transformed text, nothing else", "- Do NOT add explanations or commentary", ]; if (stylePrompt) { sections.push( "", "## User Style (HIGHEST PRIORITY)", "The user has configured these style preferences. They OVERRIDE the default tone adaptation below.", "Follow these instructions precisely -- they reflect the user's personal writing voice and preferences.", "", stylePrompt, ); } sections.push("", "## Tone Adaptation"); if (stylePrompt) { sections.push( "Use these as fallback guidance only when the User Style above does not cover a specific aspect:", ); } else { sections.push("Match the tone to the active application context:"); } sections.push( "- Email apps (Gmail, Mail): Professional but warm.", "- Slack: Casual and conversational.", "- Code editors (VS Code, Xcode): Technical and concise.", "- Terminal: Command-like, terse.", "- Messages/iMessage: Very casual, texting style.", "- Notes/Docs: Neutral, clear writing.", "- Default: Match the user's natural voice.", "", "## Context Clues", "- Window title may contain recipient name (Slack DMs, email compose)", "- If you can identify a recipient, adapt formality to the apparent relationship", "- Maintain the user's natural voice -- don't over-formalize casual speech", "- The user's writing patterns and preferences may be available from memory context -- follow those when present", "", buildAppMetadataBlock(body.context), "", "Selected text:", body.context.selectedText ?? "", "", `Instruction: ${body.transcription}`, ); return sections.join("\n"); } function computeMaxTokens(inputLength: number): number { const estimatedInputTokens = Math.ceil(inputLength / 3); return Math.max(256, estimatedInputTokens + 128); } interface DictationResult { text: string; mode: DictationMode; actionPlan?: string; resolvedProfileId: string; profileSource: ProfileResolution["source"]; } async function handleDictation(body: DictationBody): Promise { log.info( { transcriptionLength: body.transcription.length }, "Dictation request received", ); const resolution = resolveProfile( body.context.bundleIdentifier, body.context.appName, body.profileId, ); const { profile, source: profileSource } = resolution; log.info( { profileId: profile.id, profileSource }, "Resolved dictation profile", ); const profileMeta = { resolvedProfileId: profile.id, profileSource, }; const stylePrompt = profile.stylePrompt || undefined; // Command mode: selected text present if ( body.context.selectedText && body.context.selectedText.trim().length > 0 ) { log.info({ mode: "command" }, "Command mode (selected text present)"); return handleCommandMode(body, profile, profileMeta, stylePrompt); } // Non-command: single LLM call that classifies AND cleans in one shot const transcription = expandSnippets(body.transcription, profile.snippets); try { const provider = await getConfiguredProvider("interactionClassifier"); if (!provider) { log.warn( "Dictation: no provider available, using heuristic + raw transcription", ); // Build a compatible msg for the heuristic const mode = detectDictationModeHeuristic({ type: "dictation_request", transcription: body.transcription, context: body.context, } as DictationRequest); const normalizedText = applyDictionary(transcription, profile.dictionary); if (mode === "action") { return { text: body.transcription, mode: "action", actionPlan: `User wants to: ${body.transcription}`, ...profileMeta, }; } return { text: normalizedText, mode, ...profileMeta, }; } const systemPrompt = buildCombinedDictationPrompt(body, stylePrompt); const maxTokens = computeMaxTokens(transcription.length); const { signal, cleanup } = createTimeout( DICTATION_CLASSIFICATION_TIMEOUT_MS, ); try { const response = await provider.sendMessage( [userMessage(`Transcription: "${transcription}"`)], { tools: [ { name: "process_dictation", description: "Classify the voice input and return cleaned text", input_schema: { type: "object" as const, properties: { mode: { type: "string", enum: ["dictation", "action"], description: "dictation = user wants text inserted/cleaned up for typing. action = user wants the assistant to perform a task.", }, text: { type: "string", description: "If dictation: the cleaned/formatted text ready for insertion. If action: the raw transcription unchanged.", }, reasoning: { type: "string", description: "Brief reasoning for the classification", }, }, required: ["mode", "text", "reasoning"], }, }, ], systemPrompt, config: { callSite: "interactionClassifier", max_tokens: maxTokens, tool_choice: { type: "tool" as const, name: "process_dictation", }, }, signal, }, ); cleanup(); const toolBlock = extractToolUse(response); if (toolBlock) { const input = toolBlock.input as { mode?: string; text?: string; reasoning?: string; }; const mode: DictationMode = input.mode === "action" ? "action" : "dictation"; log.info( { mode, reasoning: input.reasoning }, "LLM dictation classify+clean", ); if (mode === "action") { return { text: body.transcription, mode: "action", actionPlan: `User wants to: ${body.transcription}`, ...profileMeta, }; } const cleanedText = input.text?.trim() || transcription; const normalizedText = applyDictionary(cleanedText, profile.dictionary); return { text: normalizedText, mode: "dictation", ...profileMeta, }; } log.warn("No tool_use block in combined dictation call, using heuristic"); } finally { cleanup(); } } catch (err) { const message = err instanceof Error ? err.message : String(err); log.warn( { err: message }, "Combined dictation LLM call failed, using heuristic", ); } // Heuristic fallback const fallbackMode = detectDictationModeHeuristic({ type: "dictation_request", transcription: body.transcription, context: body.context, } as DictationRequest); log.info({ mode: fallbackMode }, "Using heuristic fallback"); if (fallbackMode === "action") { return { text: body.transcription, mode: "action", actionPlan: `User wants to: ${body.transcription}`, ...profileMeta, }; } const normalizedText = applyDictionary(transcription, profile.dictionary); return { text: normalizedText, mode: fallbackMode, ...profileMeta, }; } async function handleCommandMode( body: DictationBody, profile: ReturnType["profile"], profileMeta: { resolvedProfileId: string; profileSource: ProfileResolution["source"]; }, stylePrompt: string | undefined, ): Promise { const systemPrompt = buildCommandPrompt(body, stylePrompt); const inputLength = (body.context.selectedText ?? "").length + body.transcription.length; const maxTokens = Math.max(1024, computeMaxTokens(inputLength)); try { const provider = await getConfiguredProvider("interactionClassifier"); if (!provider) { log.warn("Command mode: no provider available, returning selected text"); const normalizedText = applyDictionary( body.context.selectedText ?? body.transcription, profile.dictionary, ); return { text: normalizedText, mode: "command", ...profileMeta, }; } const response = await provider.sendMessage( [userMessage(body.transcription)], { tools: [], systemPrompt, config: { callSite: "interactionClassifier", max_tokens: maxTokens }, }, ); const textBlock = response.content.find((b) => b.type === "text"); const cleanedText = textBlock && "text" in textBlock ? textBlock.text.trim() : (body.context.selectedText ?? body.transcription); const normalizedText = applyDictionary(cleanedText, profile.dictionary); return { text: normalizedText, mode: "command", ...profileMeta, }; } catch (err) { log.error({ err }, "Command mode LLM call failed, returning selected text"); const normalizedText = applyDictionary( body.context.selectedText ?? body.transcription, profile.dictionary, ); return { text: normalizedText, mode: "command", ...profileMeta, }; } } // --------------------------------------------------------------------------- // Route definitions // --------------------------------------------------------------------------- export const ROUTES: RouteDefinition[] = [ { operationId: "dictation_post", endpoint: "dictation", method: "POST", policy: { requiredScopes: ["chat.write"], allowedPrincipalTypes: ACTOR_PRINCIPALS, }, summary: "Process dictation", description: "Classify voice input as dictation or action, clean up text, and apply user style preferences.", tags: ["diagnostics"], requestBody: DictationRequestSchema, responseBody: z.object({ text: z.string().describe("Processed text output"), mode: z.string().describe("Detected mode: dictation, command, or action"), actionPlan: z .string() .optional() .describe("Action plan (only when mode is action)"), resolvedProfileId: z.string().describe("Resolved dictation profile ID"), profileSource: z.string().describe("How the profile was resolved"), }), handler: async ({ body = {} }: RouteHandlerArgs) => { const { transcription, context, profileId } = body as unknown as DictationBody; if (!transcription) { throw new BadRequestError("transcription is required"); } if (!context) { throw new BadRequestError("context is required"); } return handleDictation({ transcription, context, profileId }); }, }, ];