import { v4 as uuid, v7 as uuidv7 } from "uuid"; import { z } from "zod"; import { SurfaceActionSchema } from "../api/events/ui-surface-show.js"; import type { AssistantEvent } from "../api/index.js"; import { CardSurfaceDataSchema, ChoiceSurfaceDataSchema, coerceSurfaceDataRecord, DynamicPagePreviewSchema, DynamicPageSurfaceDataSchema, isDaemonInternalSurfaceType, isKnownSurfaceType, MODEL_INVOKABLE_SURFACE_TYPES, normalizeCopyBlockShowData, normalizeVisualShowData, OAuthConnectSurfaceDataSchema, safeParseSurfaceData, SurfaceTypeSchema, } from "../api/surfaces.js"; import { getApp, getAppDirPath, getAppPreview, linkAppToConversationLineage, resolveAppDir, resolveEffectiveAppHtml, updateApp, } from "../apps/app-store.js"; import { recordActivationEvent } from "../onboarding/onboarding-events-store.js"; import { getMessages, updateMessageContent, } from "../persistence/conversation-crud.js"; import { isActivationSession } from "../plugins/defaults/memory/activation-session-store.js"; import { assistantEventHub, broadcastMessage, } from "../runtime/assistant-event-hub.js"; import { ambiguousSameUserError, enforceSameActorOrErrorResult, pickSameUserAutoResolve, } from "../runtime/auth/same-actor.js"; import { resolveCapabilities } from "../runtime/capabilities.js"; import type { InteractiveUiRequest, InteractiveUiResult, } from "../runtime/interactive-ui-types.js"; import { activationMomentEmitsAtShow, type ActivationMomentParam, activationStepNameForMomentParam, isActivationMomentParam, } from "../telemetry/activation-funnel.js"; import { resolveAppId } from "../tools/apps/resolve-app-id.js"; import type { ToolExecutionResult } from "../tools/types.js"; import { getLogger } from "../util/logger.js"; import { isPlainObject } from "../util/object.js"; import type { Conversation } from "./conversation.js"; import { buildConversationErrorMessage } from "./conversation-error.js"; import { launchConversation } from "./conversation-launch.js"; import { buildSurfaceShowPair, type CurrentTurnSurface, parseShowPairOrThrow, type SurfaceShowPair, type SurfaceStateEntry, } from "./conversation-surface-state.js"; import type { HostCuProxy } from "./host-cu-proxy.js"; import type { AnySurfaceData, CardSurfaceData, ChoiceSurfaceData, ConfirmationSurfaceData, DynamicPagePreview, DynamicPageSurfaceData, FormSurfaceData, SurfaceAction, SurfaceData, SurfaceType, TableColumn, TableRow, UiSurfaceShow, } from "./message-protocol.js"; import { INTERACTIVE_SURFACE_TYPES } from "./message-protocol.js"; import { isRowVisibleToUntrustedActor } from "./message-provenance.js"; import type { TrustContext } from "./trust-context-types.js"; import { restingTrust } from "./trust-context-types.js"; export { buildSurfaceShowPair, type CurrentTurnSurface, parseSurfaceShowPair, restoreSurfaceStateEntry, type StoredSurfaceAction, type SurfaceShowPair, type SurfaceStateEntry, } from "./conversation-surface-state.js"; import type { HostAppControlInput } from "./message-types/host-app-control.js"; import type { UserMessageAttachment } from "./message-types/shared.js"; /** * Prefix of the synthetic user-message text this module writes when a surface * action has no custom prompt of its own (`[User action on app: ...]`, * `[User action on surface: ...]`). * * Machine-authored, so turns carrying it must be stamped `scripted` and kept * out of activation counts. Deliberately the SAME anchor the analytics * classifier matches on (`stg_telemetry__scripted_turn.sql`, "Synthetic * UI-surface action events"): the two signals are compared against each other * by the `assert_scripted_signals_agree` dbt test, so if this string ever * changes, that model's anchor has to change with it or the test fires. */ const SYNTHETIC_SURFACE_ACTION_PREFIX = "[User action on "; /** * True when `content` is the synthetic fallback text above rather than a * custom prompt supplied by the surface. * * Only the synthetic form is scripted. A surface that supplies its own prompt * is treated as a real turn, because that is what the existing analytics * classifier does. Widening this to every surface action would silently move * activation beyond the bug being fixed. */ function isSyntheticSurfaceActionContent(content: string): boolean { return content.startsWith(SYNTHETIC_SURFACE_ACTION_PREFIX); } const log = getLogger("conversation-surfaces"); // Tolerant variant of SurfaceActionSchema for parsing raw model output. // The canonical schema rejects unknown style values; this one coerces them // to "secondary" so a single mistyped style doesn't drop all actions. const ModelActionSchema = SurfaceActionSchema.extend({ style: z .enum(["primary", "secondary", "destructive"]) .catch("secondary") .optional(), }); const MAX_UNDO_DEPTH = 10; /** * Whether a surface-action turn (a user clicked a button or submitted a form on * a UI surface) runs with a human declared present. The handler does not yet * learn the actor's interface, so it cannot declare presence the way the send * route does; the turn runs non-interactive, which is what the state-derived * fallback resolved to between turns. Deriving it from the actor's interface * is a policy change tracked separately. */ const SURFACE_ACTION_TURN_IS_INTERACTIVE = false; /** * Pending surface types that do not hold the one-interactive-surface-at-a-time * lock. Each renders content the user reads (or settles on its own) rather * than a question they must answer, so a live one must not block the next * surface. */ const NON_BLOCKING_PENDING_SURFACE_TYPES = new Set([ "dynamic_page", "visual", "voice_picker", ]); /** * Surface types that carry no terminal action: the card settles when the user * interacts with it, so no click could ever satisfy an attached `actions` * entry or an explicit `await_action`. Both are generic ui_show params though, * so nothing stops the model attaching them here, and doing so wedges the * turn: the client latches `awaiting_user_input` on the presence of actions * alone and no action ever arrives to clear it, leaving the composer disabled * and Stop hidden while the daemon is still streaming. Stripping them makes * the "never blocks a turn" contract structural rather than advisory. */ const ACTIONLESS_SURFACE_TYPES = new Set(["voice_picker"]); /** * Debounce window for persisting `ui_surface_update` data back to the * message row. Surfaces typically receive bursts of updates (e.g. a * Workspace Health Check ticking off items rapidly) — collapsing them * to a single DB write avoids hammering SQLite while still bounding the * "lost work on crash" window to ~half a second. */ const SURFACE_PERSIST_DEBOUNCE_MS = 500; /** * In-flight debounced persist timers keyed by `surfaceId`. Surface IDs * are UUIDs and globally unique, so a module-level map is safe across * conversations. Each entry holds the latest data snapshot — newer * updates clobber older ones since the persisted row carries the full * merged state, not a delta. */ const pendingSurfacePersists = new Map< string, { timer: ReturnType; conversationId: string; data: AnySurfaceData; } >(); /** A located `ui_surface` block plus everything needed to write it back. */ type PersistedSurfaceHit = { rowId: string; blocks: unknown[]; block: Record; blockIndex: number; }; /** * Locate the persisted `ui_surface` content block for `surfaceId`. * * The single owner of how a persisted surface block is resolved *in this * module*: newest message row first, first matching block within that row, * stopping at the first hit. Every persisted read and write here goes through * it so a change to storage resolution, ordering, or matching can never make * them target different blocks. `findPersistedSurfaceState` * (`runtime/routes/surface-conversation-resolver.ts`) is the other persisted * `ui_surface` reader and deliberately does not: it runs its own SQL scan, * bounded by the compaction boundary, for the read-only content route. * * `filterByProvenance` applies {@link isRowVisibleToUntrustedActor} per row, * the identical predicate `loadFromDb` applies when building `messages`. It * covers the persisted type/data read ({@link findPersistedSurfaceInfo}) * only. Writes scan unfiltered, so a completion write can mutate a row the * requester cannot see and push requester-controlled summary text into it. * The live-state path (`ctx.surfaceState` / `ctx.pendingSurfaceActions`) is * unfiltered too: it is scoped to the trust the conversation was loaded * under, not the requester's. * * The scan is deliberately unbounded by compaction (see * {@link findPersistedSurfaceInfo}), and it throws rather than swallowing DB * errors so each caller keeps its own logging. */ function findPersistedSurfaceBlock( conversationId: string, surfaceId: string, opts: { filterByProvenance: boolean }, ): PersistedSurfaceHit | undefined { const { filterByProvenance } = opts; const rows = getMessages(conversationId); for (let r = rows.length - 1; r >= 0; r--) { if (filterByProvenance && !isRowVisibleToUntrustedActor(rows[r].metadata)) { continue; } const blocks: unknown[] = rows[r].content; const blockIndex = blocks.findIndex((pb) => { const rb = pb as Record; return rb.type === "ui_surface" && rb.surfaceId === surfaceId; }); if (blockIndex !== -1) { return { rowId: rows[r].id, blocks, block: blocks[blockIndex] as Record, blockIndex, }; } } return undefined; } /** * Persist the latest `data` for a `ui_surface` content block by locating it * with {@link findPersistedSurfaceBlock} and patching its `data` field. * * Safe to call before the assistant message has been persisted (mid-stream): * the scan simply finds nothing and bails. The next update after * `handleMessageComplete` runs will pick up the now-persisted row. */ function persistSurfaceData( conversationId: string, surfaceId: string, data: SurfaceData, ): void { try { const hit = findPersistedSurfaceBlock(conversationId, surfaceId, { filterByProvenance: false, }); if (!hit) { return; } hit.block.data = data; updateMessageContent(hit.rowId, JSON.stringify(hit.blocks)); } catch (err) { log.debug( { err, surfaceId, conversationId }, "Failed to persist surface data update", ); } } /** * Schedule a debounced write of the merged surface data back to the * persisted message row. Repeated calls within the debounce window * collapse to a single write carrying the latest data. */ export function scheduleSurfaceDataPersist( conversationId: string, surfaceId: string, data: SurfaceData, ): void { const existing = pendingSurfacePersists.get(surfaceId); if (existing) { clearTimeout(existing.timer); } const timer = setTimeout(() => { pendingSurfacePersists.delete(surfaceId); persistSurfaceData(conversationId, surfaceId, data); }, SURFACE_PERSIST_DEBOUNCE_MS); pendingSurfacePersists.set(surfaceId, { timer, conversationId, data }); } /** * Force-flush any pending debounced persist for `surfaceId`. Called on * surface completion so the final state is durable before the surface * record transitions to `completed`. */ export function flushSurfaceDataPersist(surfaceId: string): void { const pending = pendingSurfacePersists.get(surfaceId); if (!pending) { return; } clearTimeout(pending.timer); pendingSurfacePersists.delete(surfaceId); persistSurfaceData(pending.conversationId, surfaceId, pending.data); } /** * Discard (without writing) any pending debounced persist for `surfaceId`. * Called on dismissal so an in-flight `ui_update` snapshot cannot land after * the surface block has been removed. */ export function cancelSurfaceDataPersist(surfaceId: string): void { const pending = pendingSurfacePersists.get(surfaceId); if (!pending) { return; } clearTimeout(pending.timer); pendingSurfacePersists.delete(surfaceId); } /** * Cancel all pending debounced persists. Called on conversation * teardown to avoid timers firing against torn-down state. * * Use `flushPendingSurfaceDataPersists` instead on a clean shutdown * path where the latest in-flight surface state should still be * written before teardown. */ export function cancelPendingSurfaceDataPersists( conversationId?: string, ): void { for (const [surfaceId, pending] of pendingSurfacePersists) { if (conversationId && pending.conversationId !== conversationId) { continue; } clearTimeout(pending.timer); pendingSurfacePersists.delete(surfaceId); } } /** * Synchronously flush all pending debounced persists, optionally scoped * to a single conversation. Called on clean conversation teardown so an * update that arrived inside the 500ms debounce window still lands in * the DB before the conversation goes away. Each entry is removed from * the pending map after its write fires. */ export function flushPendingSurfaceDataPersists(conversationId?: string): void { for (const [surfaceId, pending] of pendingSurfacePersists) { if (conversationId && pending.conversationId !== conversationId) { continue; } clearTimeout(pending.timer); pendingSurfacePersists.delete(surfaceId); persistSurfaceData(pending.conversationId, surfaceId, pending.data); } } /** * Mark a `ui_surface` content block as completed in the database so that * history reconstruction preserves the completion state. Also updates * in-memory messages when available. * * Never throws: a persistence hiccup must not fail the surface action that * triggered it. Returns whether the completion is safe to announce to * clients, so a caller about to broadcast `ui_surface_complete` can withhold * it. A write that threw is not safe: it leaves a persisted block that reverts * to pending on the next history reseed. Finding no block to write IS safe: a * standalone surface owns none, so nothing can revert. * * A surface still in `ctx.currentTurnSurfaces` owns no persisted block either, * but one is coming: the turn-finalization appenders build it from that * snapshot. The completion is stamped onto the snapshot so the appended block * carries it, instead of landing as a fresh pending card the next reseed * reactivates. */ export function markSurfaceCompleted( ctx: { conversationId: string; messages?: Array<{ content: unknown }>; currentTurnSurfaces?: CurrentTurnSurface[]; }, surfaceId: string, summary: string, ): boolean { // Force-flush any pending debounced data persist so the completion // patch lands on top of the latest data instead of racing with it. flushSurfaceDataPersist(surfaceId); try { const hit = findPersistedSurfaceBlock(ctx.conversationId, surfaceId, { filterByProvenance: false, }); if (hit) { hit.block.completed = true; hit.block.completionSummary = summary; updateMessageContent(hit.rowId, JSON.stringify(hit.blocks)); } else { // Mutated in place so a later `ui_update` that respreads the entry // carries the completion forward. const pendingSnapshot = ctx.currentTurnSurfaces?.find( (s) => s.surfaceId === surfaceId, ); if (pendingSnapshot) { pendingSnapshot.completed = true; pendingSnapshot.completionSummary = summary; } } } catch (err) { // Error, not warn: a silent failure here presents as the user's answer being discarded. log.error( { err, conversationId: ctx.conversationId, surfaceId }, "Failed to persist surface completion to DB", ); // In-memory messages stay untouched so this process's history cannot // claim an answered card the database still holds as pending. return false; } // Update in-memory messages when available so subsequent reads within // this session see the change without waiting for DB. Newest match only, // matching the persisted patch above: marking every copy of a duplicated // surfaceId would make memory and history disagree, and a reseed would // revert the extras. if (ctx.messages) { outer: for (let i = ctx.messages.length - 1; i >= 0; i--) { const msg = ctx.messages[i]; if (!Array.isArray(msg.content)) { continue; } for (const block of msg.content) { const b = block as Record; if (b.type === "ui_surface" && b.surfaceId === surfaceId) { b.completed = true; b.completionSummary = summary; break outer; } } } } return true; } /** What a persisted `ui_surface` block can still tell us once it is cold. */ type PersistedSurfaceInfo = { /** The block's `surfaceType`, absent when it carries no string type. */ surfaceType: string | undefined; /** The block's `data`, absent when it carries no plain-object data. */ data: Record | undefined; }; /** * Read a `ui_surface` block's `surfaceType` and `data` out of persisted * history. Both travel together because every caller that has lost the live * entry needs both: the type to decide completion, the data for the labels a * completion summary quotes. * * `requesterCanAccessMemory` is the REQUESTER's memory capability, not the * trust class whatever conversation happens to be loaded under. The live path * this falls back from hides guardian-provenance rows from an untrusted actor * (`loadFromDb` → `restoreSurfaceStateFromHistory`), so the live entry is * missing precisely when the row was filtered out. The scan therefore applies * the same per-row predicate rather than handing back what the filter dropped. * * This must NOT be routed through `findPersistedSurfaceState` * (`runtime/routes/surface-conversation-resolver.ts`): that helper is bounded * by `liveHistoryStartRow` / `contextCompactedMessageCount` and by design will * not see a surface behind the compaction boundary, which is exactly the case * this lookup exists to serve. {@link findPersistedSurfaceBlock}, which also * backs the write paths, is unbounded, so reads and writes stay consistent. * * Hits are deliberately not memoized into `conversation.surfaceState`; see * `runtime/routes/surface-content-routes.ts` for why that shared map must not * absorb scan results. * * Exported for its unit test only, and not a supported read API. In-repo callers * go through `handleSurfaceAction`. */ export function findPersistedSurfaceInfo( conversationId: string, surfaceId: string, opts: { requesterCanAccessMemory: boolean }, ): PersistedSurfaceInfo | undefined { try { const hit = findPersistedSurfaceBlock(conversationId, surfaceId, { filterByProvenance: !opts.requesterCanAccessMemory, }); if (!hit) { return undefined; } const data = hit.block.data; return { surfaceType: typeof hit.block.surfaceType === "string" ? hit.block.surfaceType : undefined, data: isPlainObject(data) ? data : undefined, }; } catch (err) { log.warn( { err, conversationId, surfaceId }, "Failed to read persisted surface info from DB", ); } return undefined; } /** * Complete a `ui_surface` card and notify live clients, addressed only by * conversation + surface id. * * Unlike {@link maybeCompleteSurfaceAfterAction}, this needs no live `Conversation` * instance, so it can run from flows that don't own one — projecting a * terminal guardian-request status onto its in-app approval card when the * request was resolved on another surface (or by the expiry sweep). Persists * the completion (reload-safe) first, then broadcasts `ui_surface_complete` so * every connected client of this guardian converges. * * The broadcast is deliberately NOT gated on the persist, unlike the * user-action completion paths: the underlying request is already resolved, so * withholding the announcement on a transient write failure strands a live, * clickable approval card for a decision that has already been made. */ export function completeSurfaceAndNotify( conversationId: string, surfaceId: string, summary: string, ): void { markSurfaceCompleted({ conversationId }, surfaceId, summary); broadcastMessage({ type: "ui_surface_complete", conversationId, surfaceId, summary, }); } /** * Remove a `ui_surface` content block from history so a passively dismissed * surface does not survive a reload. The live client drops a dismissed surface * entirely; this converges persisted state with that behaviour. Cancels any * pending debounced data persist first so a late `ui_update` snapshot cannot * re-add the block, then strips the block from in-memory messages and the DB. */ export function removeSurfaceBlock( ctx: { conversationId: string; messages?: Array<{ content: unknown }> }, surfaceId: string, ): void { cancelSurfaceDataPersist(surfaceId); if (ctx.messages) { for (let i = ctx.messages.length - 1; i >= 0; i--) { const msg = ctx.messages[i]; if (!Array.isArray(msg.content)) { continue; } const idx = msg.content.findIndex((block) => { const b = block as Record; return b.type === "ui_surface" && b.surfaceId === surfaceId; }); if (idx !== -1) { msg.content.splice(idx, 1); break; } } } try { const hit = findPersistedSurfaceBlock(ctx.conversationId, surfaceId, { filterByProvenance: false, }); if (hit) { hit.blocks.splice(hit.blockIndex, 1); updateMessageContent(hit.rowId, JSON.stringify(hit.blocks)); } } catch (err) { log.warn({ err, surfaceId }, "Failed to remove dismissed surface from DB"); } } const TASK_PROGRESS_TEMPLATE_FIELDS = ["title", "status", "steps"] as const; const TASK_PROGRESS_CARD_STATUSES = new Set([ "in_progress", "completed", "failed", ]); const TASK_PROGRESS_STEP_STATUSES = new Set([ "pending", "in_progress", "completed", "failed", ]); /** * Coerce a model-supplied `steps` value into a renderable array. Drops * non-object and label-less entries, accepts `title` as a `label` alias, and * defaults a missing/invalid per-step status to "pending". Returns `[]` for a * missing or non-array input so an indeterminate card still renders. */ function normalizeTaskProgressSteps( value: unknown, ): Array> { if (!Array.isArray(value)) { return []; } return value .filter((step): step is Record => isPlainObject(step)) .map((step) => { const label = typeof step.label === "string" ? step.label : typeof step.title === "string" ? step.title : ""; const status = typeof step.status === "string" && TASK_PROGRESS_STEP_STATUSES.has(step.status) ? step.status : "pending"; return { ...step, label, status }; }) .filter((step) => (step.label as string).trim().length > 0); } /** * Guarantee a task_progress card reaches the client with a well-formed * `templateData` object so a coarse or indeterminate attempt (missing steps, * missing status) renders instead of being silently dropped. Fills only * missing fields — a fully-specified card is left intact. */ function ensureTaskProgressTemplateData( normalized: Record, ): void { const templateData: Record = isPlainObject( normalized.templateData, ) ? { ...normalized.templateData } : {}; if ( typeof templateData.title !== "string" && typeof normalized.title === "string" ) { templateData.title = normalized.title; } if ( typeof templateData.status !== "string" || !TASK_PROGRESS_CARD_STATUSES.has(templateData.status) ) { templateData.status = "in_progress"; } templateData.steps = normalizeTaskProgressSteps(templateData.steps); normalized.templateData = templateData; } /** * Migrate dynamic_page fields from the top-level tool input into `data`. * * The LLM sometimes sends `html`, `width`, `height`, or `preview` at the * top level instead of nested inside `data`. Without this normalization the * surface opens blank because `rawData` is `{}`. */ function normalizeDynamicPageShowData( input: Record, rawData: Record, ): DynamicPageSurfaceData { const normalized: Record = { ...rawData }; if (typeof normalized.html !== "string" && typeof input.html === "string") { normalized.html = input.html; } if (normalized.width == null && input.width != null) { normalized.width = input.width; } if (normalized.height == null && input.height != null) { normalized.height = input.height; } if (!isPlainObject(normalized.preview) && isPlainObject(input.preview)) { normalized.preview = input.preview; } return DynamicPageSurfaceDataSchema.parse(normalized); } /** First entry that is a non-empty (trimmed) string, else undefined. */ function firstNonEmptyString(values: unknown[]): string | undefined { for (const value of values) { if (typeof value === "string" && value.trim().length > 0) { return value; } } return undefined; } /** All non-empty (trimmed) strings from the values list. */ function allNonEmptyStrings(values: unknown[]): string[] { const result: string[] = []; for (const value of values) { if (typeof value === "string" && value.trim().length > 0) { result.push(value); } } return result; } function normalizeCardShowData( input: Record, rawData: Record, ): CardSurfaceData { const normalized: Record = { ...rawData }; // Older prompt examples sent template/templateData at the top level. if ( typeof normalized.template !== "string" && typeof input.template === "string" ) { normalized.template = input.template; } if ( !isPlainObject(normalized.templateData) && isPlainObject(input.templateData) ) { normalized.templateData = input.templateData; } // The LLM sometimes sends `title` or `body` at the top-level tool input // instead of nesting them inside `data`. The Swift client requires `title` // inside the card data dict — without it `parseCardData` returns nil and // the surface is silently dropped. Copy them from input when missing. if ( typeof normalized.title !== "string" && typeof input.title === "string" && input.title.trim().length > 0 ) { normalized.title = input.title; } if (typeof normalized.body !== "string" && typeof input.body === "string") { normalized.body = input.body; } // The model sees every surface type's schema in the ui_show tool description, // so it frequently borrows keys from sibling surfaces when emitting a card. // Recover those into the canonical card fields, checking both data-level and // top-level (input) placement. Multiple matches are concatenated (body) or // first-wins (title/subtitle); all alias keys are deleted afterward so they // don't appear as droppedKeys noise. // // body aliases: copy_block's `text`, confirmation's `message`, generic // `content`, and cross-surface `description` (choice/form/oauth/work_result/ // dynamic_page — 5 types use it), work_result's `summary`, confirmation's // `detail`. const bodyAliasKeys = [ "text", "message", "content", "description", "summary", "detail", ] as const; if (typeof normalized.body !== "string" || normalized.body.trim() === "") { const candidates = allNonEmptyStrings( bodyAliasKeys.map((k) => { const dataVal = normalized[k]; if (typeof dataVal === "string" && dataVal.trim().length > 0) { return dataVal; } return input[k]; }), ); if (candidates.length > 0) { // Temporary: concatenate all matching aliases so no content is lost. // A future pass should define per-alias semantic roles (e.g. summary // as a subtitle, detail as supplementary) once production telemetry // reveals which combinations actually occur. normalized.body = candidates.join("\n\n"); } } for (const key of bodyAliasKeys) { delete normalized[key]; } // title aliases: natural synonyms the model reaches for when it doesn't // use `title` verbatim. const titleAliasKeys = ["heading", "header", "name"] as const; if (typeof normalized.title !== "string" || normalized.title.trim() === "") { const aliased = firstNonEmptyString([ ...titleAliasKeys.map((k) => normalized[k]), ...titleAliasKeys.map((k) => input[k]), ]); if (aliased !== undefined) { normalized.title = aliased; } } for (const key of titleAliasKeys) { delete normalized[key]; } // subtitle aliases: table's `caption`, natural synonym `subheading`. if ( typeof normalized.subtitle !== "string" && typeof input.subtitle === "string" ) { normalized.subtitle = input.subtitle; } const subtitleAliasKeys = ["subheading", "caption"] as const; if ( typeof normalized.subtitle !== "string" || normalized.subtitle.trim() === "" ) { const aliased = firstNonEmptyString([ ...subtitleAliasKeys.map((k) => normalized[k]), ...subtitleAliasKeys.map((k) => input[k]), ]); if (aliased !== undefined) { normalized.subtitle = aliased; } } for (const key of subtitleAliasKeys) { delete normalized[key]; } if (!Array.isArray(normalized.metadata) && Array.isArray(input.metadata)) { normalized.metadata = input.metadata; } // task_progress cards: additional fallbacks for title from templateData. if ( normalized.template === "task_progress" && typeof normalized.title !== "string" ) { if ( isPlainObject(normalized.templateData) && typeof normalized.templateData.title === "string" ) { normalized.title = normalized.templateData.title; } else { normalized.title = "Task Progress"; } } if ( normalized.template === "task_progress" && typeof normalized.body !== "string" ) { normalized.body = ""; } if (normalized.template === "task_progress") { ensureTaskProgressTemplateData(normalized); } // Parse, don't assert. The old `as unknown as CardSurfaceData` accepted any // shape, so anything the model nested under an unmodelled key was carried // through unread. Parsing draws the boundary; the dropped-key log surfaces // the shapes we still don't recover, so the recovery list above can grow from // real traffic rather than guesswork. const droppedKeys = Object.keys(normalized).filter( (key) => !(key in CardSurfaceDataSchema.shape), ); if (droppedKeys.length > 0) { log.warn( { droppedKeys }, "ui_show card data carried keys the card contract does not model; their content will not render", ); } const parsed = CardSurfaceDataSchema.safeParse(normalized); if (parsed.success) { return parsed.data; } log.warn( { issues: parsed.error.issues }, "ui_show card data failed CardSurfaceDataSchema; rendering only the fields that validated", ); return CardSurfaceDataSchema.parse({ title: typeof normalized.title === "string" ? normalized.title : undefined, subtitle: typeof normalized.subtitle === "string" ? normalized.subtitle : undefined, body: typeof normalized.body === "string" ? normalized.body : undefined, }); } function normalizeTaskProgressCardPatch( existingCard: CardSurfaceData, patch: Record, ): Record { if (existingCard.template !== "task_progress") { return patch; } const normalizedPatch: Record = { ...patch }; const mergedTemplateData: Record = isPlainObject( existingCard.templateData, ) ? { ...existingCard.templateData } : {}; let updatedTemplateData = false; if (isPlainObject(normalizedPatch.templateData)) { Object.assign(mergedTemplateData, normalizedPatch.templateData); updatedTemplateData = true; } // Accept top-level task_progress fields from older prompt examples and // move them into templateData where the Swift client expects them. for (const key of TASK_PROGRESS_TEMPLATE_FIELDS) { if (key in normalizedPatch) { mergedTemplateData[key] = normalizedPatch[key]; delete normalizedPatch[key]; updatedTemplateData = true; } } if (updatedTemplateData) { normalizedPatch.templateData = mergedTemplateData; } return normalizedPatch; } function buildChoiceActions(data: ChoiceSurfaceData): SurfaceAction[] { return data.options.map((option) => ({ id: option.id, label: option.title, style: option.recommended ? "primary" : "secondary", data: { choiceId: option.id, choiceTitle: option.title, selectedIds: [option.id], selectedTitles: [option.title], ...(option.description ? { choiceDescription: option.description } : {}), ...(option.recommended ? { recommended: true } : {}), ...(option.data ?? {}), }, })); } function isTaskProgressCardData(data: SurfaceData | Record) { return (data as Record).template === "task_progress"; } function isSlackTaskProgressUiException( ctx: Conversation, toolName: string, input: Record, ): boolean { if (ctx.channelCapabilities?.channel !== "slack") { return false; } if (toolName === "ui_show") { if (input.surface_type !== "card") { return false; } const data = normalizeCardShowData( input, coerceSurfaceDataRecord(input.data), ); return isTaskProgressCardData(data); } if (toolName === "ui_update") { const surfaceId = input.surface_id; if (typeof surfaceId !== "string") { return false; } const stored = ctx.surfaceState.get(surfaceId); if (!stored || stored.surfaceType !== "card") { return false; } if (!isTaskProgressCardData(stored.data)) { return false; } const rawPatch = coerceSurfaceDataRecord(input.data); const patch = normalizeTaskProgressCardPatch(stored.data, rawPatch); return isTaskProgressCardData({ ...stored.data, ...patch }); } return false; } export type SurfaceMutex = { (surfaceId: string, fn: () => T | Promise): Promise; /** Number of surfaces with an active chain — exposed for tests. */ readonly size: number; }; /** * Per-surface async mutex using Promise chaining. * Operations on the same surfaceId are serialized; different surfaces run concurrently. */ export function createSurfaceMutex(): SurfaceMutex { const chains = new Map>(); const mutex = ( surfaceId: string, fn: () => T | Promise, ): Promise => { const prev = chains.get(surfaceId) ?? Promise.resolve(); const next = prev.then(fn, fn); // Keep the chain alive but swallow errors so one failure doesn't block subsequent ops const tail = next.then( () => {}, () => {}, ); chains.set(surfaceId, tail); // Clean up the map entry once the queue settles to prevent unbounded growth tail.then(() => { if (chains.get(surfaceId) === tail) { chains.delete(surfaceId); } }); return next; }; Object.defineProperty(mutex, "size", { get: () => chains.size }); return mutex as SurfaceMutex; } // ── Standalone surface lifecycle ──────────────────────────────────── // // Daemon-driven UI surfaces that block the caller (skill, IPC handler) // until the user responds or the timeout elapses. Unlike LLM-invoked // surfaces (ui_show tool), these never trigger an LLM follow-up turn — // the result is returned directly to the requesting code. /** Default timeout for standalone surfaces when the caller does not specify one. */ const DEFAULT_STANDALONE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes /** * How long a tombstone entry persists after a standalone surface is completed. * Late client actions arriving within this window are silently dropped. */ const STANDALONE_TOMBSTONE_TTL_MS = 30_000; // 30 seconds /** * Check whether the conversation can show interactive UI surfaces. * Fails closed when no client is connected or the channel doesn't * support dynamic UI. */ export function canShowInteractiveUi( ctx: Pick, ): boolean { if (ctx.hasNoClient) { return false; } if (ctx.channelCapabilities && !ctx.channelCapabilities.supportsDynamicUi) { return false; } return true; } /** * Show a standalone UI surface and return a Promise that resolves when * the user submits, cancels, or the timeout elapses. * * This is the core entry point for daemon-driven (non-LLM) UI requests. * It performs the fail-closed capability check, emits `ui_surface_show`, * stores surface state, arms the timeout, and registers a pending entry * so that `handleSurfaceAction` can intercept the callback. */ export function showStandaloneSurface( ctx: Conversation, request: InteractiveUiRequest, surfaceId: string, ): Promise { // ── Fail-closed: no interactive UI capability ── if (!canShowInteractiveUi(ctx)) { log.warn( { conversationId: ctx.conversationId, surfaceType: request.surfaceType, hasNoClient: ctx.hasNoClient, channel: ctx.channelCapabilities?.channel, }, "standalone surface: no interactive UI capability; failing closed", ); return Promise.resolve({ status: "cancelled" as const, surfaceId, cancellationReason: "no_interactive_surface", }); } // The pendingStandaloneSurfaces map must exist on the context. // The Conversation class always initializes it; if absent, fail closed. if (!ctx.pendingStandaloneSurfaces) { log.warn( { conversationId: ctx.conversationId, surfaceType: request.surfaceType }, "standalone surface: pendingStandaloneSurfaces map missing; failing closed", ); return Promise.resolve({ status: "cancelled" as const, surfaceId, cancellationReason: "no_interactive_surface", }); } const pendingMap = ctx.pendingStandaloneSurfaces; const timeoutMs = request.timeoutMs ?? DEFAULT_STANDALONE_TIMEOUT_MS; // Build the correlated surface pair from the request payload. const surfaceType: SurfaceType = request.surfaceType; const pair = buildStandaloneSurfaceData(request); const actions = request.actions?.map((a) => ({ id: a.id, label: a.label, style: (a.variant === "danger" ? "destructive" : (a.variant ?? "secondary")) as "primary" | "secondary" | "destructive", })); return new Promise((resolve) => { // ── Arm timeout ── const timer = setTimeout(() => { // Notify the client BEFORE cleanup so the surface is dismissed on // the client side, preventing stale user interactions from reaching // handleSurfaceAction and being misrouted to the LLM. try { broadcastMessage({ type: "ui_surface_complete", conversationId: ctx.conversationId, surfaceId, summary: "Timed out", }); } catch (err) { log.warn( { err, conversationId: ctx.conversationId, surfaceId }, "Failed to emit ui_surface_complete on timeout", ); } cleanupStandaloneSurface(ctx, surfaceId); log.info( { conversationId: ctx.conversationId, surfaceId, timeoutMs }, "standalone surface timed out", ); resolve({ status: "timed_out", surfaceId }); }, timeoutMs); // ── Register pending entry ── pendingMap.set(surfaceId, { resolve, timer, surfaceType, }); // ── Store surface state ── ctx.surfaceState.set(surfaceId, { title: request.title, actions, ...pair, }); broadcastMessage({ type: "ui_surface_show", conversationId: ctx.conversationId, surfaceId, title: request.title, actions, display: "inline", ...pair, }); log.info( { conversationId: ctx.conversationId, surfaceId, surfaceType, timeoutMs, }, "standalone surface shown", ); }); } /** * Build a correlated surface pair from an InteractiveUiRequest. * Maps the generic `data` payload to the typed shape expected by the * surface type. */ function buildStandaloneSurfaceData( request: InteractiveUiRequest, ): SurfaceShowPair { if (request.surfaceType === "confirmation") { const data: ConfirmationSurfaceData = { message: typeof request.data.message === "string" ? request.data.message : (request.title ?? "Please confirm"), detail: typeof request.data.detail === "string" ? request.data.detail : undefined, confirmLabel: typeof request.data.confirmLabel === "string" ? request.data.confirmLabel : undefined, cancelLabel: typeof request.data.cancelLabel === "string" ? request.data.cancelLabel : undefined, destructive: typeof request.data.destructive === "boolean" ? request.data.destructive : undefined, }; return { surfaceType: "confirmation", data }; } // Preserve the full form payload (pages, pageLabels, and any future // additive keys) via spreading. Apply defensive normalization so that // `fields` is always a valid array — callers that use `pages` instead // of top-level `fields` may omit the latter entirely. const raw = request.data; const hasFields = Array.isArray(raw.fields) && raw.fields.length > 0; const fields: FormSurfaceData["fields"] = hasFields ? (raw.fields as FormSurfaceData["fields"]) : []; return { surfaceType: "form", data: { ...raw, fields } as FormSurfaceData }; } /** * Cleanup a standalone surface entry: clear the timeout timer, remove * the pending entry, remove surface state, and record a short-lived * tombstone so late client actions are silently dropped instead of * falling through to the LLM path. Idempotent — safe to call multiple * times for the same surfaceId. */ export function cleanupStandaloneSurface( ctx: Pick< Conversation, | "pendingStandaloneSurfaces" | "recentlyCompletedStandaloneSurfaces" | "surfaceState" | "pendingSurfaceActions" | "lastSurfaceAction" | "accumulatedSurfaceState" | "surfaceUndoStacks" >, surfaceId: string, ): void { const entry = ctx.pendingStandaloneSurfaces?.get(surfaceId); if (entry) { clearTimeout(entry.timer); ctx.pendingStandaloneSurfaces?.delete(surfaceId); } ctx.surfaceState.delete(surfaceId); ctx.pendingSurfaceActions.delete(surfaceId); ctx.lastSurfaceAction.delete(surfaceId); ctx.accumulatedSurfaceState.delete(surfaceId); ctx.surfaceUndoStacks.delete(surfaceId); // Record a tombstone so late client actions are silently dropped. if (ctx.recentlyCompletedStandaloneSurfaces) { // Clear any existing tombstone timer for this surfaceId (idempotency). const existingTimer = ctx.recentlyCompletedStandaloneSurfaces.get(surfaceId); if (existingTimer) { clearTimeout(existingTimer); } const tombstoneTimer = setTimeout(() => { ctx.recentlyCompletedStandaloneSurfaces?.delete(surfaceId); }, STANDALONE_TOMBSTONE_TTL_MS); ctx.recentlyCompletedStandaloneSurfaces.set(surfaceId, tombstoneTimer); } } /** * How long to wait for a client to acknowledge an `open_panel` command * before reporting failure to the model. The ack round-trip is one SSE * delivery plus one HTTP POST, so this is generous — it only elapses when * no connected client rendered the panel (event dropped, no client * listening, or a client build that predates panel acknowledgment). */ const OPEN_PANEL_ACK_TIMEOUT_MS = 10_000; /** * Open the channel-setup drawer on a connected client and wait for the * client's acknowledgment. * * `open_panel` is a side-effect-only command: it is never persisted to the * transcript, so a dropped event is unrecoverable by reload. The ack is what * makes the emitting tool result truthful — "displayed" must mean a client * actually rendered the drawer, not merely that the event was emitted. * * Reuses the standalone-surface pending machinery: the client responds via * the existing surface-action route (`actionId: "ack"` on success, `"nack"` * with `data.reason` when it received the event but could not open the * panel), which `handleSurfaceAction` intercepts and resolves without an * LLM turn. The surface-state entry exists only so the surface-action route * can resolve the owning conversation by `surfaceId`; `cleanupStandaloneSurface` * removes it on every outcome. */ export function openChannelSetupPanel( ctx: Conversation, surfaceId: string, data: Record, options?: { signal?: AbortSignal; timeoutMs?: number }, ): Promise { if (!canShowInteractiveUi(ctx) || !ctx.pendingStandaloneSurfaces) { log.warn( { conversationId: ctx.conversationId, hasNoClient: ctx.hasNoClient, channel: ctx.channelCapabilities?.channel, }, "channel_setup panel: no interactive UI capability; failing closed", ); return Promise.resolve({ status: "cancelled" as const, surfaceId, cancellationReason: "no_interactive_surface" as const, }); } const pendingMap = ctx.pendingStandaloneSurfaces; const signal = options?.signal; const timeoutMs = options?.timeoutMs ?? OPEN_PANEL_ACK_TIMEOUT_MS; if (signal?.aborted) { return Promise.resolve({ status: "cancelled" as const, surfaceId, cancellationReason: "resolver_unavailable" as const, }); } return new Promise((resolve) => { const settle = (result: InteractiveUiResult) => { signal?.removeEventListener("abort", onAbort); resolve(result); }; const timer = setTimeout(() => { cleanupStandaloneSurface(ctx, surfaceId); log.warn( { conversationId: ctx.conversationId, surfaceId, timeoutMs }, "channel_setup panel: no client acknowledged open_panel", ); settle({ status: "timed_out", surfaceId }); }, timeoutMs); const onAbort = () => { cleanupStandaloneSurface(ctx, surfaceId); settle({ status: "cancelled", surfaceId, cancellationReason: "resolver_unavailable", }); }; signal?.addEventListener("abort", onAbort, { once: true }); pendingMap.set(surfaceId, { resolve: settle, timer, surfaceType: "channel_setup", }); // Registered so the surface-action route's by-surfaceId conversation // lookup finds this conversation when the ack arrives without a // conversationId. Cleared by cleanupStandaloneSurface on all outcomes. ctx.surfaceState.set(surfaceId, { surfaceType: "channel_setup", data: safeParseSurfaceData("channel_setup", data) ?? {}, }); ctx.emit({ type: "open_panel", panelType: "channel_setup", data, conversationId: ctx.conversationId, surfaceId, }); }); } /** * Handle content_changed action from document editor. * Auto-saves the document content to the app store. */ function handleDocumentContentChanged( ctx: Conversation, surfaceId: string, data?: Record, ): void { if (!data) { log.warn({ surfaceId }, "content_changed action missing data"); return; } const { title, content, wordCount } = data as { title?: string; content?: string; wordCount?: number; }; if (!title && !content) { log.warn({ surfaceId }, "content_changed action missing title or content"); return; } // Find the app ID from the surface state const surfaceState = ctx.surfaceState.get(surfaceId); if (!surfaceState || surfaceState.surfaceType !== "dynamic_page") { log.warn({ surfaceId }, "Surface not found or not a dynamic page"); return; } const appId = surfaceState.data.appId; if (!appId || !appId.startsWith("doc-")) { // Not a document app, ignore log.debug({ surfaceId, appId }, "Not a document app, skipping auto-save"); return; } try { const app = getApp(appId); if (!app) { log.warn({ appId }, "Document app not found"); return; } // Regenerate the editor HTML with updated content // We need to import the editor template dynamically import("../tools/document/editor-template.js") .then(({ generateEditorHTML }) => { const updatedHtml = generateEditorHTML( title || app.name, content || "", ); updateApp(appId, { name: title || app.name, description: `Document with ${wordCount ?? 0} words`, preview: content?.slice(0, 200), htmlDefinition: updatedHtml, }); log.info({ appId, wordCount }, "Document auto-saved"); }) .catch((err) => { log.error( { err, appId }, "Failed to import editor template for auto-save", ); }); } catch (err) { log.error({ err, appId }, "Failed to auto-save document"); } } /** * Handle state_update action from a dynamic page. * Accumulates state via shallow merge without triggering an LLM turn. */ function handleStateUpdate( ctx: Conversation, surfaceId: string, data?: Record, ): void { if (!data) { log.debug({ surfaceId }, "state_update action called with no data"); return; } const surfaceState = ctx.surfaceState.get(surfaceId); if (!surfaceState || surfaceState.surfaceType !== "dynamic_page") { log.warn( { surfaceId, surfaceType: surfaceState?.surfaceType }, "state_update action received for non-dynamic_page surface", ); return; } const existing = ctx.accumulatedSurfaceState.get(surfaceId) ?? {}; const merged = { ...existing, ...data }; ctx.accumulatedSurfaceState.set(surfaceId, merged); log.debug( { surfaceId, accumulatedState: merged }, "Accumulated surface state updated", ); } function pushUndoState( surfaceUndoStacks: Map, surfaceId: string, html: string, ): void { let stack = surfaceUndoStacks.get(surfaceId); if (!stack) { stack = []; surfaceUndoStacks.set(surfaceId, stack); } stack.push(html); if (stack.length > MAX_UNDO_DEPTH) { stack.shift(); } } export function handleSurfaceUndo(ctx: Conversation, surfaceId: string): void { const stack = ctx.surfaceUndoStacks.get(surfaceId); if (!stack || stack.length === 0) { ctx.emit({ type: "ui_surface_undo_result", conversationId: ctx.conversationId, surfaceId, success: false, remainingUndos: 0, }); return; } const previousHtml = stack.pop()!; const stored = ctx.surfaceState.get(surfaceId); if (!stored || stored.surfaceType !== "dynamic_page") { ctx.emit({ type: "ui_surface_undo_result", conversationId: ctx.conversationId, surfaceId, success: false, remainingUndos: stack.length, }); return; } const data = stored.data; // If app-backed, also revert the persisted app and refresh all surfaces for this app if (data.appId) { try { updateApp(data.appId, { htmlDefinition: previousHtml }); } catch (err) { log.error({ appId: data.appId, err }, "Failed to revert app during undo"); } // Update ALL surfaces that share this appId (not just the requesting one) for (const [sid, s] of ctx.surfaceState.entries()) { if (s.surfaceType !== "dynamic_page") { continue; } const sData = s.data; if (sData.appId !== data.appId) { continue; } const revertedData: DynamicPageSurfaceData = { ...sData, html: previousHtml, }; s.data = revertedData; ctx.emit({ type: "ui_surface_update", conversationId: ctx.conversationId, surfaceId: sid, data: revertedData, }); } // Sync sibling undo stacks: pop the top entry if it matches the HTML we // just reverted to, preventing phantom no-op undo steps on siblings. for (const [sid, s] of ctx.surfaceState.entries()) { if (sid === surfaceId) { continue; } if (s.surfaceType !== "dynamic_page") { continue; } const sData = s.data; if (sData.appId !== data.appId) { continue; } const siblingStack = ctx.surfaceUndoStacks.get(sid); if (siblingStack && siblingStack.length > 0) { const top = siblingStack[siblingStack.length - 1]; if (top === previousHtml) { siblingStack.pop(); } } } } else { // Ephemeral surface — update only the requesting surface const revertedData: DynamicPageSurfaceData = { ...data, html: previousHtml, }; stored.data = revertedData; ctx.emit({ type: "ui_surface_update", conversationId: ctx.conversationId, surfaceId, data: revertedData, }); } ctx.emit({ type: "ui_surface_undo_result", conversationId: ctx.conversationId, surfaceId, success: true, remainingUndos: stack.length, }); log.info( { conversationId: ctx.conversationId, surfaceId, remaining: stack.length }, "Surface undo applied", ); } /** Extract a human-readable label from a table row using the first column value. */ export function describeTableRow( row: TableRow, columns: TableColumn[], ): string { if (columns.length === 0) { return row.id; } const firstColId = columns[0].id; const cell = row.cells[firstColId]; if (cell == null) { return row.id; } if (typeof cell === "string") { return cell; } return cell.text; } const MAX_DESELECTION_ITEMS = 20; /** Format a list of deselected item labels as a bullet list, capped at MAX_DESELECTION_ITEMS. */ export function formatDeselectionList(labels: string[]): string { if (labels.length === 0) { return ""; } const shown = labels.slice(0, MAX_DESELECTION_ITEMS); const lines = shown.map((l) => `- ${l}`); if (labels.length > MAX_DESELECTION_ITEMS) { lines.push(`(and ${labels.length - MAX_DESELECTION_ITEMS} more)`); } return lines.join("\n"); } /** * Compute a deselection description by diffing selectedIds against the stored * surface state rows/items. Returns empty string when nothing was deselected. */ export function buildDeselectionDescription( surfaceType: SurfaceType, surfaceState: SurfaceShowPair | undefined, selectedIds: string[], ): string { if (!surfaceState) { return ""; } const selectedSet = new Set(selectedIds); if (surfaceType === "table" && surfaceState.surfaceType === "table") { const tableData = surfaceState.data; const deselectedLabels: string[] = []; for (const row of tableData.rows) { if (row.selectable === false) { continue; } if (!selectedSet.has(row.id)) { deselectedLabels.push(describeTableRow(row, tableData.columns)); } } if (deselectedLabels.length === 0) { return ""; } return `\n\nDeselected items (user chose NOT to include):\n${formatDeselectionList( deselectedLabels, )}`; } if (surfaceType === "list" && surfaceState.surfaceType === "list") { const listData = surfaceState.data; const deselectedLabels: string[] = []; for (const item of listData.items) { if (!selectedSet.has(item.id)) { deselectedLabels.push(item.title); } } if (deselectedLabels.length === 0) { return ""; } return `\n\nDeselected items (user chose NOT to include):\n${formatDeselectionList( deselectedLabels, )}`; } return ""; } export type SurfaceActionResult = | { accepted: true; conversationId: string } | { accepted: false; error: string } | void; const SURFACE_COMPLETE_FLAG = "_completeSurface"; const SURFACE_COMPLETION_SUMMARY_FIELD = "_completionSummary"; function getRequestedSurfaceCompletionSummary( data?: Record, ): string | null { if (data?.[SURFACE_COMPLETE_FLAG] !== true) { return null; } const summary = typeof data[SURFACE_COMPLETION_SUMMARY_FIELD] === "string" ? data[SURFACE_COMPLETION_SUMMARY_FIELD].trim() : ""; return summary || "Completed"; } /** * Best-effort recorder for a single activation-funnel moment. Gated on the * conversation being a marked activation-rail session. Fire-and-forget: never * throws, never blocks or alters the surface-action flow (a failure is logged * and swallowed). Shared by the show-time path (`first_wow_executed`, recorded * when the surface renders) and the commit-time path (the other moments, * recorded when the user commits the surface). */ function recordActivationMoment( ctx: Conversation, moment: ActivationMomentParam, ): void { try { if (!isActivationSession(ctx.conversationId)) { return; } recordActivationEvent({ stepName: activationStepNameForMomentParam(moment), sessionId: ctx.conversationId, }); } catch (err) { log.warn( { err, conversationId: ctx.conversationId, moment }, "Failed to record activation moment", ); } } /** * Best-effort activation-funnel emission on a user surface commit. * * When the committed surface carries a commit-timing `activationMoment` tag, * record the corresponding funnel milestone. Show-timing moments * (`first_wow_executed`) are recorded at render time in `surfaceProxyResolver` * and are NOT stored on `surfaceState`, so they never reach this path. The tag * is cleared after the first record so re-entrant or repeated commits on the * same surface do not double-emit (and the deterministic `daemon_event_id` * collapses any cross-surface duplicate downstream anyway). * * Must be called only from terminal-commit paths (user clicked an action / * submitted / selected-and-committed), NOT from intermediate non-terminal * events (`selection_changed` / `content_changed` / `state_update`). */ function maybeEmitActivationMoment(ctx: Conversation, surfaceId: string): void { const stored = ctx.surfaceState.get(surfaceId); const moment = stored?.activationMoment; if (!moment) { return; } // Clear the tag first so this can fire at most once per surface even if the // commit path is re-entered. stored.activationMoment = undefined; recordActivationMoment(ctx, moment); } /** * Record a surface action the message queue refused. * * The route answers `{ ok: true }` for a rejected enqueue and the client * optimistically completes the card on that, so the user sees an answered card * that the next history reseed reverts. Nothing else on this path logs, so * without this the only user-visible cause of a reverted card leaves no trace. */ function logSurfaceActionRejected( ctx: Conversation, surfaceId: string, actionId: string, surfaceType: string | undefined, ): void { log.warn( { conversationId: ctx.conversationId, surfaceId, actionId, surfaceType, queueDepth: ctx.getQueueDepth(), }, "Surface action rejected by the message queue", ); } // One-shot interactive surfaces auto-complete once their action message is // accepted (they never accept further actions). const ONE_SHOT_SURFACE_TYPES = [ "choice", "oauth_connect", "form", "confirmation", "file_upload", "task_preferences", ]; /** * The completion rule for an accepted surface action: complete when the client * asked for it explicitly (`_completeSurface`), or when the surface is a * one-shot type. No-op otherwise. Owns the whole rule, including reading the * `_completeSurface` request out of the raw action data. * * Called from both `handleSurfaceAction` branches, one holding a pending entry * and one with none, always after `enqueueMessage` accepted the turn so a * rejected enqueue leaves the surface answerable. Persists first and broadcasts * `ui_surface_complete` only once that write is known not to have failed: a * client must not render a card the next history reseed reverts. * * "No pending entry" is broader than "restored from history", and with * `pendingSurfaceActions` empty the two cannot be told apart: a surface shown * with `await_action: false` never registered a pending entry, and a daemon * restart drops the entry of one that did. So a one-shot surface that never * awaited an action does complete on its first action. * * `liveSurfaceType` / `liveSurfaceData` come from in-memory state. When the * type is missing the persisted block supplies both, read at most once per * action and only once the decision actually needs it. */ function maybeCompleteSurfaceAfterAction( ctx: Conversation, surfaceId: string, actionId: string, actionData: Record | undefined, opts: { liveSurfaceType: string | undefined; liveSurfaceData: Record | undefined; requesterCanAccessMemory: boolean; submittedData?: Record; }, ): void { const requestedSummary = getRequestedSurfaceCompletionSummary(actionData); let surfaceType = opts.liveSurfaceType; let surfaceData = opts.liveSurfaceData; // Only the type-driven one-shot rule needs history: an explicit request // carries its own summary and decides on its own. Type and labels come from // the same read so the decision and the summary can never disagree. if (surfaceType === undefined && !requestedSummary) { const persisted = findPersistedSurfaceInfo(ctx.conversationId, surfaceId, { requesterCanAccessMemory: opts.requesterCanAccessMemory, }); surfaceType = persisted?.surfaceType; surfaceData = persisted?.data ?? surfaceData; } const isOneShot = surfaceType !== undefined && ONE_SHOT_SURFACE_TYPES.includes(surfaceType); if (!requestedSummary && !isOneShot) { return; } const summary = requestedSummary ?? buildCompletionSummary(surfaceType, actionId, actionData, surfaceData); if (!markSurfaceCompleted(ctx, surfaceId, summary)) { return; } broadcastMessage({ type: "ui_surface_complete", conversationId: ctx.conversationId, surfaceId, summary, ...(opts.submittedData ? { submittedData: opts.submittedData } : {}), }); } export async function handleSurfaceAction( ctx: Conversation, surfaceId: string, actionId: string, data?: Record, // JWT-verified committer principal; threaded so enqueued turns can // reconstruct the same-user binding for host proxies (CU / app-control), // mirroring the normal message path. sourceActorPrincipalId?: string, // Trust of the actor committing this action, resolved by the route from the // verified requester. Passed rather than read back off the conversation so a // click that lands mid-turn is attributed to its clicker instead of to // whoever occupies the resting slot. requesterTrustContext?: TrustContext, ): Promise { const actionTrustContext = requesterTrustContext ?? restingTrust(ctx); // ── Standalone surface interception ────────────────────────────── // Daemon-driven surfaces (from `requestInteractiveUi`) register a // pending entry in `pendingStandaloneSurfaces`. When the user clicks // an action, resolve the caller's Promise directly and return WITHOUT // enqueuing a model message — consumed standalone callbacks never // trigger an LLM follow-up turn. // // This block runs BEFORE launch_conversation dispatch so that a // standalone form whose submittedData happens to contain // `_action: "launch_conversation"` is resolved as a standalone // interaction rather than triggering a conversation launch. const standalone = ctx.pendingStandaloneSurfaces?.get(surfaceId); if (standalone) { const stored = ctx.surfaceState.get(surfaceId); const summary = buildCompletionSummary( standalone.surfaceType, actionId, data, stored?.data as Record | undefined, ); // Determine result status from the action. const isCancellation = actionId === "cancel" || actionId === "dismiss"; const status: InteractiveUiResult["status"] = isCancellation ? "cancelled" : "submitted"; const result: InteractiveUiResult = { status, surfaceId, actionId, ...(data ? { submittedData: data } : {}), ...(isCancellation ? { cancellationReason: "user_dismissed" as const } : {}), summary, }; // channel_setup pendings are protocol-level acknowledgments for the // `open_panel` command, not user-facing surfaces: nothing was rendered // in the transcript, so there is no completion to broadcast, persist, // or count as an activation commit. Resolve and clean up only. if (standalone.surfaceType === "channel_setup") { cleanupStandaloneSurface(ctx, surfaceId); standalone.resolve(result); log.info( { conversationId: ctx.conversationId, surfaceId, actionId, status }, "open_panel acknowledgment resolved", ); return { accepted: true, conversationId: ctx.conversationId }; } // Broadcast unconditionally: `showStandaloneSurface` renders straight to // the client without appending a `ui_surface` block, so there is no // persisted state a failed completion write could contradict. broadcastMessage({ type: "ui_surface_complete", conversationId: ctx.conversationId, surfaceId, summary, submittedData: data, }); markSurfaceCompleted(ctx, surfaceId, summary); // Terminal user commit on a standalone surface (submit, not cancel/dismiss) // — record an activation milestone if tagged. Must run before // `cleanupStandaloneSurface` clears the surface state. if (!isCancellation) { maybeEmitActivationMoment(ctx, surfaceId); } // Cleanup and resolve — order matters: cleanup clears the timer // before resolve() unblocks the caller. cleanupStandaloneSurface(ctx, surfaceId); standalone.resolve(result); log.info( { conversationId: ctx.conversationId, surfaceId, actionId, status, }, "standalone surface resolved by user action", ); // Return without enqueuing a model message. return { accepted: true, conversationId: ctx.conversationId }; } // ── Tombstone guard for recently-completed standalone surfaces ──── // After a standalone surface times out or is resolved, cleanup removes // all state. Without this guard a late client action would fall through // to the history-restored path below and enqueue a message to the LLM. if (ctx.recentlyCompletedStandaloneSurfaces?.has(surfaceId)) { log.debug( { conversationId: ctx.conversationId, surfaceId, actionId }, "Dropping late action for recently-completed standalone surface", ); return { accepted: true, conversationId: ctx.conversationId }; } // `launch_conversation` actions spawn a fresh conversation inline instead // of round-tripping through the LLM with a `[User action on card surface: // ...]` chat message. This dispatch must run BEFORE the pending-vs-not // branching below: `ui_show` unconditionally calls // `pendingSurfaceActions.set(...)` for any interactive card (regardless of // the `persistent` flag), so on the very first click of a freshly-rendered // launcher card `pending` is already set. Without this hoist the launch // branch would fall through into the pending path and the LLM round-trip // would happen on every click. if ( data && typeof data === "object" && (data as Record)._action === "launch_conversation" ) { const payload = data as Record; const title = typeof payload.title === "string" ? payload.title : ""; const seedPrompt = typeof payload.seedPrompt === "string" ? payload.seedPrompt : ""; const anchorMessageId = typeof payload.anchorMessageId === "string" ? payload.anchorMessageId : undefined; if (!title || !seedPrompt) { return { accepted: false, error: "missing_title_or_seedPrompt" }; } // Launch actions don't consume the surface — persistent launcher cards // keep accepting clicks afterward. Drop the pending entry (if any) so // sibling button presses on the same card aren't blocked behind a stale // expectation that this surface still owes an answer to the LLM. ctx.pendingSurfaceActions.delete(surfaceId); // `ctx` is the origin Conversation — inherit its trust context so the // spawned conversation keeps guardian / trust-class state. // // `launchConversation` is the sole emitter of `open_conversation` for // this path. We pass `focus: false` so the client registers a sidebar // entry for the spawned conversation without switching focus away from // the origin — critical for fan-out UX where one click launches // multiple conversations. // // The helper also kicks off the seed turn fire-and-forget, so this // `await` resolves as soon as the conversation is created + titled + // published to the event hub. The HTTP POST /v1/surface-actions // response returns promptly — the seed turn runs in the background. const originTrustContext = actionTrustContext; const { conversationId } = await launchConversation({ title, seedPrompt, focus: false, ...(anchorMessageId ? { anchorMessageId } : {}), ...(originTrustContext ? { originTrustContext } : {}), }); log.info( { originConversationId: ctx.conversationId, conversationId, surfaceId }, "launch_conversation dispatched inline from surface action", ); // Launching a child conversation is a terminal user commit — record an // activation milestone if tagged. The helper clears the tag after firing, // so the other commit-path call sites below can't double-emit. maybeEmitActivationMoment(ctx, surfaceId); return { accepted: true, conversationId }; } // Trust of the actor committing THIS action, so the completion path's // persisted read stays scoped to exactly what this actor's live view would // have held. Unresolvable trust fails closed to the untrusted filter. const requesterCanAccessMemory = resolveCapabilities( actionTrustContext?.trustClass, ).canAccessMemory; const pending = ctx.pendingSurfaceActions.get(surfaceId); // When surfaces are restored from history (e.g. onboarding cards), there is // no in-memory pendingSurfaceActions entry. Handle non-terminal actions // directly, and forward custom/relay actions to the LLM. if (!pending) { // Non-terminal actions don't need stored state — handle directly. if (actionId === "selection_changed") { log.debug( { surfaceId, data }, "Selection changed (history-restored, not forwarding)", ); return; } if (actionId === "content_changed") { log.debug( { surfaceId }, "Content changed (history-restored, no surface state — skipping)", ); return; } if (actionId === "state_update") { if (data) { const existing = ctx.accumulatedSurfaceState.get(surfaceId) ?? {}; ctx.accumulatedSurfaceState.set(surfaceId, { ...existing, ...data }); } log.debug( { surfaceId, data }, "Silent state accumulated (history-restored)", ); return; } // Determine message content from the action. const stored = ctx.surfaceState.get(surfaceId); const actionDef = stored?.actions?.find((a) => a.id === actionId); const mergedData: Record | undefined = actionDef?.data || data ? { ...actionDef?.data, ...data } : undefined; const isRelay = actionId === "relay_prompt" || actionId === "agent_prompt"; const prompt = isRelay && typeof mergedData?.prompt === "string" ? mergedData.prompt.trim() : ""; // Read accumulated state once — used by both relay and custom action paths. const accState = ctx.accumulatedSurfaceState.get(surfaceId); const hasAccState = accState && Object.keys(accState).length > 0; // Extract file attachments from action data so they are sent as proper // image/file content blocks instead of dumping base64 into the text. let attachments: UserMessageAttachment[] = []; let actionDataForText = mergedData; if (mergedData && Array.isArray(mergedData.files)) { const files = mergedData.files as Array>; attachments = files .filter( (f) => typeof f.filename === "string" && typeof f.mimeType === "string" && typeof f.data === "string", ) .map((f) => ({ filename: f.filename as string, mimeType: f.mimeType as string, data: f.data as string, ...(typeof f.extractedText === "string" ? { extractedText: f.extractedText } : {}), })); // Only remove files from the text payload when we successfully parsed // attachments — otherwise preserve the original data so the model still // sees the files field (e.g. IDs/paths from dynamic app actions). if (attachments.length > 0) { const { files: _files, ...rest } = mergedData; actionDataForText = Object.keys(rest).length > 0 ? rest : undefined; } } let content: string; let displayContent: string | undefined; if (prompt) { content = prompt; // Re-append accumulated state so the LLM sees it, matching the pending path. if (hasAccState) { content += `\n\nAccumulated surface state: ${JSON.stringify(accState)}`; } } else { // Custom action from an app (e.g. sendAction('answer_selected', {...})) const summary = actionId .replace(/_/g, " ") .replace(/\b\w/g, (c) => c.toUpperCase()); content = `[User action on app: ${summary}]`; if (attachments.length > 0) { const names = attachments.map((a) => a.filename).join(", "); content += `\n\nUploaded files: ${names}`; } if (actionDataForText && Object.keys(actionDataForText).length > 0) { content += `\n\nAction data: ${JSON.stringify(actionDataForText)}`; } if (hasAccState) { content += `\n\nAccumulated surface state: ${JSON.stringify(accState)}`; } displayContent = summary; } log.info( { conversationId: ctx.conversationId, surfaceId, actionId, // False means the surface type and the labels its completion summary // quotes can only come from persisted history. hasLiveState: stored !== undefined, contentLength: content.length, contentPreview: content.slice(0, 200), attachmentCount: attachments.length, attachments: attachments.map((a) => ({ filename: a.filename, mimeType: a.mimeType, dataLength: a.data?.length ?? 0, hasExtractedText: !!a.extractedText, })), }, "Surface action: preparing to send message to model", ); const requestId = uuidv7(); ctx.surfaceActionRequestIds.add(requestId); // Pass conversationId so events without an inline conversationId (e.g. // text_delta) are published with the correct conversation scope and // reach the SSE subscriber filtered to this conversation. const onEvent = (msg: AssistantEvent) => broadcastMessage(msg, ctx.conversationId); const result = ctx.enqueueMessage({ content, attachments, onEvent, requestId, activeSurfaceId: surfaceId, displayContent, sourceActorPrincipalId, trustContext: actionTrustContext, isInteractive: SURFACE_ACTION_TURN_IS_INTERACTIVE, // Rides the metadata bag rather than a typed option: the queue // round-trips `metadata` but not `PersistMessageOptions`. metadata: { scripted: isSyntheticSurfaceActionContent(content) }, }); if (result.rejected) { ctx.surfaceActionRequestIds.delete(requestId); logSurfaceActionRejected(ctx, surfaceId, actionId, stored?.surfaceType); return; } // Terminal user commit accepted — record the activation milestone if this // surface was tagged (best-effort, no-op otherwise). Deferred until after // the rejection check so a queue-full click doesn't over-report a moment // (and the one-shot tag stays intact for the user's retry). maybeEmitActivationMoment(ctx, surfaceId); maybeCompleteSurfaceAfterAction(ctx, surfaceId, actionId, mergedData, { liveSurfaceType: stored?.surfaceType, liveSurfaceData: stored?.data as Record | undefined, requesterCanAccessMemory, submittedData: actionDataForText, }); // One-shot: clear accumulated state now that the message has been accepted. // Deferred until after rejection check so state is preserved for retry on rejection. if (hasAccState) { ctx.accumulatedSurfaceState.delete(surfaceId); } // Echo the prompt to the client so it appears in the chat UI. // Deferred until after rejection check to avoid ghost messages. if (prompt) { broadcastMessage({ type: "user_message_echo", text: prompt, conversationId: ctx.conversationId, }); } if (result.queued) { log.info( { surfaceId, actionId, requestId }, "Surface action queued (conversation busy, history-restored)", ); return; } // Conversation is idle — process the message immediately. log.info( { surfaceId, actionId, requestId, attachmentCount: attachments.length }, "Processing surface action immediately (history-restored) with attachments", ); ctx .processMessage({ content, attachments, onEvent, requestId, activeSurfaceId: surfaceId, displayContent, sourceActorPrincipalId, // Reached only when `enqueueMessage` declined to queue: the click is // starting this turn, so it is the actor the run belongs to. trustContext: actionTrustContext, isInteractive: SURFACE_ACTION_TURN_IS_INTERACTIVE, scripted: isSyntheticSurfaceActionContent(content), }) .catch((err) => { const message = err instanceof Error ? err.message : String(err); log.error( { err, surfaceId, actionId }, "Failed to process history-restored surface action", ); onEvent( buildConversationErrorMessage(ctx.conversationId, { code: "CONVERSATION_PROCESSING_FAILED", userMessage: `Something went wrong: ${message}`, retryable: false, debugDetails: `History-restored surface action processing failed: ${message}`, errorCategory: "processing_failed", }), ); }); return; } const retainPending = pending.surfaceType === "dynamic_page"; // selection_changed is a non-terminal state update — don't consume the // pending entry or send a message. if (actionId === "selection_changed") { log.debug( { surfaceId, data }, "Selection changed (non-terminal, not forwarding)", ); return; } // content_changed is a non-terminal state update for document auto-save // Save the document content and don't forward to the conversation if (actionId === "content_changed") { handleDocumentContentChanged(ctx, surfaceId, data); return; } // state_update is a silent accumulation action — merge data into accumulated // state without triggering an LLM turn. if (actionId === "state_update") { handleStateUpdate(ctx, surfaceId, data); return; } // Merge stored action-level data (from ui_show definition) with client-sent // data. This is critical for relay_prompt buttons: the client only sends the // actionId, but the prompt payload lives in the action definition's data. const stored = ctx.surfaceState.get(surfaceId); const actionDef = stored?.actions?.find((a) => a.id === actionId); const mergedData: Record | undefined = actionDef?.data || data ? { ...actionDef?.data, ...data } : undefined; ctx.lastSurfaceAction.set(surfaceId, { actionId, data: mergedData }); const shouldRelayPrompt = actionId === "relay_prompt" || actionId === "agent_prompt"; const prompt = shouldRelayPrompt && typeof mergedData?.prompt === "string" ? mergedData.prompt.trim() : ""; // Build a human-readable summary so the LLM clearly understands the // user's decision instead of parsing raw JSON. const surfaceData = stored?.data as Record | undefined; const summary = buildCompletionSummary( pending.surfaceType, actionId, mergedData, surfaceData, ); // Extract file attachments from action data so they are sent as proper // image/file content blocks instead of dumping base64 into the text. let pendingAttachments: UserMessageAttachment[] = []; let mergedDataForText = mergedData; if (mergedData && Array.isArray(mergedData.files)) { const files = mergedData.files as Array>; pendingAttachments = files .filter( (f) => typeof f.filename === "string" && typeof f.mimeType === "string" && typeof f.data === "string", ) .map((f) => ({ filename: f.filename as string, mimeType: f.mimeType as string, data: f.data as string, ...(typeof f.extractedText === "string" ? { extractedText: f.extractedText } : {}), })); // Only remove files from the text payload when we successfully parsed // attachments — otherwise preserve the original data so the model still // sees the files field. if (pendingAttachments.length > 0) { const { files: _files, ...rest } = mergedData; mergedDataForText = Object.keys(rest).length > 0 ? rest : undefined; } } let fallbackContent = `[User action on ${pending.surfaceType} surface: ${summary}]`; if (pendingAttachments.length > 0) { const names = pendingAttachments.map((a) => a.filename).join(", "); fallbackContent += `\n\nUploaded files: ${names}`; } // Append structured data so the LLM has access to IDs/values it needs // to act on (e.g. selectedIds for archiving). if (mergedDataForText && Object.keys(mergedDataForText).length > 0) { fallbackContent += `\n\nAction data: ${JSON.stringify(mergedDataForText)}`; } // Append deselection context for table/list surfaces so the LLM knows what the user chose to keep. const selectedIds = mergedData?.selectedIds as string[] | undefined; if ( selectedIds && (pending.surfaceType === "table" || pending.surfaceType === "list") ) { fallbackContent += buildDeselectionDescription( pending.surfaceType, stored, selectedIds, ); } const accumulatedState = ctx.accumulatedSurfaceState.get(surfaceId); if (accumulatedState && Object.keys(accumulatedState).length > 0) { fallbackContent += `\n\nAccumulated surface state: ${JSON.stringify(accumulatedState)}`; } // When a relay_prompt button also carries selection data (e.g. list/table // surface with a canned prompt + user-selected rows), append the selection // context so the LLM sees both the prompt and the user's selections. let content = prompt || fallbackContent; if (prompt && selectedIds && mergedData) { if (pending.surfaceType === "table" || pending.surfaceType === "list") { content += buildDeselectionDescription( pending.surfaceType, stored, selectedIds, ); } } // When prompt is truthy, fallbackContent (which includes accumulated state) // is discarded. Re-append accumulated state so the LLM sees it. if (prompt && accumulatedState && Object.keys(accumulatedState).length > 0) { content += `\n\nAccumulated surface state: ${JSON.stringify(accumulatedState)}`; } // Show the user plain-text instead of raw JSON action data. const displayContent = prompt ? undefined : buildUserFacingLabel( pending.surfaceType, actionId, mergedData, surfaceData, ); const requestId = uuidv7(); ctx.surfaceActionRequestIds.add(requestId); // Pass conversationId so events without an inline conversationId (e.g. // text_delta) are published with the correct conversation scope and // reach the SSE subscriber filtered to this conversation. const onEvent = (msg: AssistantEvent) => broadcastMessage(msg, ctx.conversationId); log.info( { surfaceId, actionId, attachmentCount: pendingAttachments.length, attachments: pendingAttachments.map((a) => ({ filename: a.filename, mimeType: a.mimeType, dataLength: a.data?.length ?? 0, })), contentPreview: content.slice(0, 200), }, "Surface action follow-up: preparing to send message to model", ); const result = ctx.enqueueMessage({ content, attachments: pendingAttachments, onEvent, requestId, activeSurfaceId: surfaceId, displayContent, sourceActorPrincipalId, trustContext: actionTrustContext, isInteractive: SURFACE_ACTION_TURN_IS_INTERACTIVE, // Rides the metadata bag rather than a typed option: the queue // round-trips `metadata` but not `PersistMessageOptions`. metadata: { scripted: isSyntheticSurfaceActionContent(content) }, }); if (result.rejected) { ctx.surfaceActionRequestIds.delete(requestId); logSurfaceActionRejected(ctx, surfaceId, actionId, pending.surfaceType); return; } // Terminal user commit accepted — record the activation milestone if this // surface was tagged (best-effort, no-op otherwise). Deferred until after the // rejection check so a queue-full click doesn't over-report a moment (and the // one-shot tag stays intact for the user's retry). maybeEmitActivationMoment(ctx, surfaceId); maybeCompleteSurfaceAfterAction(ctx, surfaceId, actionId, mergedData, { liveSurfaceType: pending.surfaceType, liveSurfaceData: surfaceData, requesterCanAccessMemory, submittedData: mergedDataForText, }); // One-shot: clear accumulated state now that the message has been accepted. // Deferred until after rejection check so state is preserved for retry on rejection. if (accumulatedState && Object.keys(accumulatedState).length > 0) { ctx.accumulatedSurfaceState.delete(surfaceId); } // Echo the user's prompt to the client so it appears in the chat UI. // Deferred until after rejection check to avoid ghost messages. if (shouldRelayPrompt && prompt) { broadcastMessage({ type: "user_message_echo", text: prompt, conversationId: ctx.conversationId, }); } if (result.queued) { if (!retainPending) { ctx.pendingSurfaceActions.delete(surfaceId); } // `enqueueMessage` already acked the queued row on `onEvent` with its // `message_queued` event; nothing more to broadcast here. log.info( { surfaceId, actionId, requestId }, "Surface action queued (conversation busy)", ); return; } if (!retainPending) { ctx.pendingSurfaceActions.delete(surfaceId); } log.info( { surfaceId, actionId, requestId, attachmentCount: pendingAttachments.length, }, "Processing surface action as follow-up with attachments", ); ctx .processMessage({ content, attachments: pendingAttachments, onEvent, requestId, activeSurfaceId: surfaceId, displayContent, sourceActorPrincipalId, // Same as the history-restored branch: the enqueue declined, so this // click is the turn about to run. trustContext: actionTrustContext, isInteractive: SURFACE_ACTION_TURN_IS_INTERACTIVE, scripted: isSyntheticSurfaceActionContent(content), }) .catch((err) => { const message = err instanceof Error ? err.message : String(err); log.error( { err, surfaceId, actionId }, "Error processing surface action", ); onEvent({ type: "error", conversationId: ctx.conversationId, message: `Failed to process surface action: ${message}`, }); }); } /** * After an app_refresh, refresh any active surface that displays the updated app. */ export function refreshSurfacesForApp( ctx: Conversation, appId: string, opts?: { fileChange?: boolean; status?: string }, ): boolean { const app = getApp(appId); if (!app) { return false; } let refreshed = false; for (const [surfaceId, stored] of ctx.surfaceState.entries()) { if (stored.surfaceType !== "dynamic_page") { continue; } const data = stored.data; if (data.appId !== appId) { continue; } // Push current HTML onto the undo stack before overwriting pushUndoState(ctx.surfaceUndoStacks, surfaceId, data.html); // Update in-memory surface state so the next refinement gets fresh HTML. // For multifile apps, resolve the compiled dist/index.html with inlined // assets rather than the empty root index.html (app.htmlDefinition). const updatedData: DynamicPageSurfaceData = { ...data, html: resolveEffectiveAppHtml(app), ...(opts?.fileChange ? { reloadGeneration: (data.reloadGeneration ?? 0) + 1 } : {}), ...(opts?.status !== undefined ? { status: opts.status } : {}), }; stored.data = updatedData; // Keep the persisted snapshot in sync so updates survive conversation restart. const idx = ctx.currentTurnSurfaces.findIndex( (s) => s.surfaceId === surfaceId, ); if (idx !== -1) { ctx.currentTurnSurfaces[idx].data = updatedData; } // Push the update to the client ctx.emit({ type: "ui_surface_update", conversationId: ctx.conversationId, surfaceId, data: updatedData, }); refreshed = true; log.info( { conversationId: ctx.conversationId, surfaceId, appId }, "Auto-refreshed surface after app_refresh", ); } return refreshed; } /** * Strip a leading "Connect "/"Connected " verb from an OAuth provider label so * a supplied displayName like "Connect Gmail" doesn't double the verb when * prefixed (e.g. avoids "Connected Connect Gmail"). */ function stripConnectVerb(label: string): string { return label.replace(/^connect(?:ed)?\s+/i, ""); } export function buildCompletionSummary( surfaceType: string | undefined, actionId: string, data?: Record, surfaceData?: Record, ): string { const selectedTitles = Array.isArray(data?.selectedTitles) ? data.selectedTitles.filter( (title): title is string => typeof title === "string", ) : []; if (surfaceType === "confirmation") { if (actionId === "cancel") { const cancelLabel = typeof surfaceData?.cancelLabel === "string" ? surfaceData.cancelLabel : undefined; return cancelLabel ? `User chose: "${cancelLabel}"` : "Cancelled"; } if (actionId === "confirm") { const confirmLabel = typeof surfaceData?.confirmLabel === "string" ? surfaceData.confirmLabel : undefined; return confirmLabel ? `User chose: "${confirmLabel}"` : "Confirmed"; } if (actionId === "deny") { // The deny button's custom label is passed as cancelLabel in the // confirmation surface data (the deny action reuses the cancel label // since both represent the "reject" path). const denyLabel = typeof surfaceData?.cancelLabel === "string" ? surfaceData.cancelLabel : undefined; return denyLabel ? `User chose: "${denyLabel}"` : "Denied"; } // Preserve the actual action ID so the LLM knows the user's exact choice // rather than misreporting it as confirmed. return `User selected: ${actionId}`; } if (surfaceType === "form") { return "Submitted"; } if (surfaceType === "choice" && data) { const choiceTitle = typeof data.choiceTitle === "string" ? data.choiceTitle : undefined; if (choiceTitle) { return `User chose: "${choiceTitle}"`; } if (selectedTitles.length === 1) { return `User chose: "${selectedTitles[0]}"`; } if (selectedTitles.length > 1) { return `User chose ${selectedTitles.length} options: ${selectedTitles .map((title) => `"${title}"`) .join(", ")}`; } return `User chose: ${actionId}`; } if (surfaceType === "oauth_connect") { const providerLabel = typeof data?.providerLabel === "string" ? data.providerLabel : typeof data?.displayName === "string" ? data.displayName : typeof data?.providerKey === "string" ? data.providerKey : "OAuth"; // Strip the verb once so every branch (connected/cancelled/failed/ // fallback) is normalized — a displayName like "Connect Gmail" must not // produce "Cancelled Connect Gmail connection". const label = stripConnectVerb(providerLabel); const accountLabel = typeof data?.accountLabel === "string" ? data.accountLabel : undefined; if (actionId === "connect" || data?.status === "connected") { return accountLabel ? `Connected ${label}: ${accountLabel}` : `Connected ${label}`; } if (actionId === "cancel" || data?.status === "cancelled") { return `Cancelled ${label} connection`; } if (data?.status === "error") { return `${label} connection failed`; } return `${label} connection ${actionId}`; } if (surfaceType === "list" && data) { const selectedIds = data.selectedIds as string[] | undefined; const actionSuffix = actionId ? ` (action: ${actionId})` : ""; if (selectedIds?.length === 1) { return `Selected: ${selectedIds[0]}${actionSuffix}`; } if (selectedIds?.length) { return `Selected ${selectedIds.length} items${actionSuffix}`; } } if (surfaceType === "table" && data) { const selectedIds = data.selectedIds as string[] | undefined; const actionSuffix = actionId ? ` (action: ${actionId})` : ""; if (selectedIds?.length === 1) { return `Selected 1 row${actionSuffix}`; } if (selectedIds?.length) { return `Selected ${selectedIds.length} rows${actionSuffix}`; } } return actionId.charAt(0).toUpperCase() + actionId.slice(1); } /** * Build a plain-text label shown to the user in the chat bubble for a * surface action. Unlike `buildCompletionSummary` (which is for the LLM), * this produces natural language the user can glance at. */ function buildUserFacingLabel( surfaceType: string | undefined, actionId: string, data?: Record, surfaceData?: Record, ): string { const count = (data?.selectedIds as string[] | undefined)?.length; const selectedTitles = Array.isArray(data?.selectedTitles) ? data.selectedTitles.filter( (title): title is string => typeof title === "string", ) : []; if (surfaceType === "confirmation") { if (actionId === "cancel") { const cancelLabel = typeof surfaceData?.cancelLabel === "string" ? surfaceData.cancelLabel : undefined; return cancelLabel ?? "Cancelled"; } if (actionId === "confirm") { const confirmLabel = typeof surfaceData?.confirmLabel === "string" ? surfaceData.confirmLabel : undefined; return confirmLabel ?? "Confirmed"; } if (actionId === "deny") { const denyLabel = typeof surfaceData?.cancelLabel === "string" ? surfaceData.cancelLabel : undefined; return denyLabel ?? "Denied"; } return `Selected: ${actionId}`; } if (surfaceType === "form") { return "Submitted"; } if (surfaceType === "choice") { const choiceTitle = typeof data?.choiceTitle === "string" ? data.choiceTitle : undefined; if (choiceTitle) { return choiceTitle; } if (selectedTitles.length === 1) { return selectedTitles[0]; } if (selectedTitles.length > 1) { return `Selected ${selectedTitles.length} options`; } return "Selected"; } if (surfaceType === "oauth_connect") { const providerLabel = typeof data?.providerLabel === "string" ? data.providerLabel : typeof data?.displayName === "string" ? data.displayName : typeof data?.providerKey === "string" ? data.providerKey : "OAuth"; // Strip the verb once so every branch is normalized (e.g. a displayName // like "Connect Gmail" must not produce "Connect Gmail connection failed"). const label = stripConnectVerb(providerLabel); const accountLabel = typeof data?.accountLabel === "string" ? data.accountLabel : undefined; if (actionId === "connect" || data?.status === "connected") { return accountLabel ? `Connected ${label}: ${accountLabel}` : `Connected ${label}`; } if (actionId === "cancel" || data?.status === "cancelled") { return "Cancelled"; } if (data?.status === "error") { return `${label} connection failed`; } return `Selected: ${actionId}`; } // Table / list selection actions if (count) { const noun = count === 1 ? "item" : "items"; const action = actionId .replace(/_/g, " ") .replace(/\b\w/g, (c) => c.toUpperCase()); return `${action} ${count} ${noun}`; } // Generic fallback — humanize the action ID return actionId.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()); } /** * Layer a caller-supplied (already schema-parsed) app preview over the * app-metadata default for `app_open`. * * `DynamicPagePreviewSchema.title` is `z.string().catch("")`, so a parsed * preview that omitted its title carries `title: ""`, which would clobber the * app-name default in the spread — reassert the default whenever the parsed * preview has no non-empty title. A generated `previewImage` always wins. * Preserves the pre-parse behavior for a caller that omits a field: `parsed` * drops missing optional fields, so the default's `subtitle` survives; a * missing title falls back to the app name exactly as the old raw-object * spread did (which simply had no `title` key to override the default). */ export function buildAppOpenPreview( defaultPreview: { title: string; subtitle?: string }, preview: DynamicPagePreview | undefined, storedPreviewImage: string | null | undefined, ): DynamicPagePreview { return { ...defaultPreview, ...preview, ...(preview && !preview.title ? { title: defaultPreview.title } : {}), ...(storedPreviewImage ? { previewImage: storedPreviewImage } : {}), }; } /** * Explain why a `computer_use_*` call cannot be dispatched, after * {@link ensureHostCuProxy} has already failed to attach one. * * "No desktop client connected" is only one of the reasons, and stating it * unconditionally contradicts `assistant clients list` whenever a desktop is * plainly connected. Only counts are reported — never client ids or actor * principals — since this string reaches the model and the transcript. */ function describeComputerUseUnavailable(ctx: Conversation): string { const capable = assistantEventHub.listClientsByCapability("host_cu"); if (capable.length === 0) { return "Computer use is not available — no desktop client connected. Open the Vellum desktop app on the machine you want to control, then retry."; } return `Computer use is not available for this conversation — ${capable.length} desktop client(s) advertise host_cu, but none of them can be driven from this conversation's interface (${ctx.transportInterface ?? "unknown"}) as its current user.`; } /** * Return the conversation's CU proxy, attaching one first when the * conversation has none. * * Host-proxy attachment is decided at turn boundaries — message create and * queue drain — and a `computer_use_*` call can arrive well after that * decision was made. A conversation whose gate failed at turn start (the * desktop had not connected yet, or the actor principal was not yet * resolvable) therefore stayed permanently without a CU proxy for the rest * of the turn, and every computer-use call reported "no desktop client * connected" while `assistant clients list` showed the desktop connected and * usable. * * Re-running the same gate here — `Conversation.ensureHostProxiesForTurn`, * the very function both turn-boundary paths call — makes the decision track * the live state instead of a stale snapshot. It grants nothing on its own: * the gate is unchanged, and every dispatch below still binds to the calling * actor through the same-actor checks. */ function ensureHostCuProxy(ctx: Conversation): HostCuProxy | undefined { if (ctx.hostCuProxy) { return ctx.hostCuProxy; } ctx.ensureHostProxiesForTurn?.(ctx.transportInterface); return ctx.hostCuProxy; } /** * Resolve a proxy tool call that targets a UI surface. * Handles ui_show, ui_update, ui_dismiss, computer_use_* proxy tools, and app_open. */ export async function surfaceProxyResolver( ctx: Conversation, toolName: string, input: Record, signal?: AbortSignal, toolUseId?: string, ): Promise { // Route CU proxy tools (all computer_use_* action tools) if (toolName.startsWith("computer_use_")) { const hostCuProxy = ensureHostCuProxy(ctx); if (!hostCuProxy || !hostCuProxy.isAvailable()) { return { content: describeComputerUseUnavailable(ctx), isError: true, }; } // Terminal tools resolve immediately without a client round-trip if ( toolName === "computer_use_done" || toolName === "computer_use_respond" ) { const summary = typeof input.summary === "string" ? input.summary : typeof input.answer === "string" ? input.answer : "Task complete"; hostCuProxy.reset(); return { content: summary, isError: false }; } // Record the action and proxy to the connected desktop client const reasoning = typeof input.reasoning === "string" ? input.reasoning : undefined; let targetClientId: string | undefined = typeof input.target_client_id === "string" && input.target_client_id !== "" ? input.target_client_id : undefined; // Validate targetClientId existence, capability, and same-user binding // before recordAction so an invalid or cross-user ID does not burn a // step or pollute action history. HostBashProxy / HostFileProxy // validate at the tool-resolution layer for the same reason. The proxy // re-checks same-user (single authoritative gate); using the shared // helper keeps log payload and error wording identical at both layers. const sourceActorPrincipalId = ctx.currentTurnSourceActorPrincipalId ?? ctx.currentTurnAuthContext?.actorPrincipalId ?? ctx.authContext?.actorPrincipalId; if (targetClientId != null) { const client = assistantEventHub.getClientById(targetClientId); if (!client) { return { content: `No connected client with id '${targetClientId}'. Run \`assistant clients list --capability host_cu\` to see available clients.`, isError: true, }; } if (!client.capabilities.includes("host_cu")) { return { content: `Client '${targetClientId}' does not support host_cu. Run \`assistant clients list --capability host_cu\` to see available clients.`, isError: true, }; } const rejection = enforceSameActorOrErrorResult({ hub: assistantEventHub, sourceActorPrincipalId, targetClientId, op: "host_cu", }); if (rejection) { return rejection; } } // Untargeted CU must resolve to exactly one same-user capable client // before dispatch. Otherwise the proxy would broadcast without a target // actor binding, which is unsafe in shared runtimes. if (targetClientId == null) { const resolved = pickSameUserAutoResolve({ hub: assistantEventHub, capability: "host_cu", sourceActorPrincipalId, }); if (resolved.kind === "ambiguous") { return ambiguousSameUserError("host_cu"); } if (resolved.kind === "match") { targetClientId = resolved.clientId; } else if ( assistantEventHub.listClientsByCapability("host_cu").length > 0 ) { return { content: "Computer use is not available for the current actor. Connect a host_cu-capable client as the same user.", isError: true, }; } } hostCuProxy.recordAction(toolName, input, reasoning); return hostCuProxy.request( toolName, input, ctx.conversationId, hostCuProxy.stepCount, reasoning, signal, targetClientId, sourceActorPrincipalId, ); } // Route app-control proxy tools (all app_control_* tool variants) if (toolName.startsWith("app_control_")) { // `app_control_stop` resolves immediately: tear down the proxy without // a client round-trip. Mirrors CU's terminal-tool short-circuit // (`computer_use_done` / `computer_use_respond`). Clear the // conversation's reference (setter disposes the existing proxy) so a // later `app_control_observe`/etc. cleanly fails with "unavailable" // instead of dispatching against a torn-down proxy, and so a sibling // conversation can acquire the released singleton lock without the // disposed proxy still being addressable. // // Run this BEFORE the isAvailable() gate so a disconnected client // doesn't strand the singleton lock — stop is local-only. if (toolName === "app_control_stop") { if (ctx.hostAppControlProxy) { if (ctx.setHostAppControlProxy) { ctx.setHostAppControlProxy(undefined); } else { ctx.hostAppControlProxy.dispose(); } } return { content: "App control stopped.", isError: false }; } if (!ctx.hostAppControlProxy || !ctx.hostAppControlProxy.isAvailable()) { return { content: "App control is not available — enable the `app-control` feature flag and connect a macOS client.", isError: true, }; } // Resolve target client. Mirrors the host_cu block above: validate // explicit target_client_id (existence, capability, same-actor), then // multi-client guard when no target is supplied. App-control is // single-client-only at the host (one active session per macOS // machine), so a broadcast across multiple capable clients would fire // the same input on every machine. let targetClientId: string | undefined = typeof input.target_client_id === "string" && input.target_client_id !== "" ? input.target_client_id : undefined; const sourceActorPrincipalId = ctx.currentTurnSourceActorPrincipalId ?? ctx.currentTurnAuthContext?.actorPrincipalId ?? ctx.authContext?.actorPrincipalId; if (targetClientId != null) { const client = assistantEventHub.getClientById(targetClientId); if (!client) { return { content: `No connected client with id '${targetClientId}'. Run \`assistant clients list --capability host_app_control\` to see available clients.`, isError: true, }; } if (!client.capabilities.includes("host_app_control")) { return { content: `Client '${targetClientId}' does not support host_app_control. Run \`assistant clients list --capability host_app_control\` to see available clients.`, isError: true, }; } const rejection = enforceSameActorOrErrorResult({ hub: assistantEventHub, sourceActorPrincipalId, targetClientId, op: "host_app_control", }); if (rejection) { return rejection; } } if (targetClientId == null) { const resolved = pickSameUserAutoResolve({ hub: assistantEventHub, capability: "host_app_control", sourceActorPrincipalId, }); if (resolved.kind === "ambiguous") { return ambiguousSameUserError("host_app_control"); } if (resolved.kind === "match") { targetClientId = resolved.clientId; } else if ( assistantEventHub.listClientsByCapability("host_app_control").length > 0 ) { return { content: "App control is not available for the current actor. Connect a host_app_control-capable client as the same user.", isError: true, }; } } // The TS `HostAppControlInput` (and the Swift mirror) is a discriminated // union on `tool` ("start" | "observe" | "press" | …). The agent's raw // tool input only carries the action-specific payload (app, x/y, text, // …) — the discriminator is implied by `toolName` (`app_control_`). // Inject it here so the proxy's session-lock guard (`input.tool === // "start"`) and the Swift client's discriminated-union decoder both see // the field they require. const tool = toolName.slice("app_control_".length); const inputWithTool = { ...input, tool, } as unknown as HostAppControlInput; return ctx.hostAppControlProxy.request( toolName, inputWithTool, ctx.conversationId, signal ?? new AbortController().signal, sourceActorPrincipalId, targetClientId, ); } if (toolName === "ui_show" || toolName === "ui_update") { const caps = ctx.channelCapabilities; if ( caps && !caps.supportsDynamicUi && !isSlackTaskProgressUiException(ctx, toolName, input) ) { log.info( { toolName, channel: caps.channel, conversationId: ctx.conversationId }, "Blocked UI surface tool on channel without dynamic UI support", ); return { content: `${toolName} is unavailable on channel "${caps.channel}" because this channel cannot render dynamic UI surfaces. Use text responses or a messaging/notification tool instead.`, isError: true, }; } } if (toolName === "ui_show") { const surfaceId = uuid(); // Parse, don't cast: an unrecognized surface_type used to be asserted // into the union and fall through to an opaque passthrough the client // silently dropped. Reject it with the valid values instead. Daemon- // internal types (skill_card, call_summary) are members of SurfaceType // for persistence but are not model-invokable — reject them here too so // the model can never report an unrenderable surface as shown. Both // messages enumerate only the model-facing set, never the internal types. const parsedSurfaceType = SurfaceTypeSchema.safeParse(input.surface_type); if ( !parsedSurfaceType.success || isDaemonInternalSurfaceType(parsedSurfaceType.data) ) { const got = typeof input.surface_type === "string" && input.surface_type.trim().length > 0 ? `"${input.surface_type}" is not a valid surface_type` : "`surface_type` is missing"; return { content: `Error: ui_show was not displayed — ${got}. Valid surface_type values: ${MODEL_INVOKABLE_SURFACE_TYPES.join(", ")}. Resend ui_show with one of these values.`, isError: true, }; } const surfaceType = parsedSurfaceType.data; const title = typeof input.title === "string" ? input.title : undefined; const rawData = coerceSurfaceDataRecord(input.data); // channel_setup is a side-effect-only command: it opens the channel setup // drawer on the client. Emitted as `open_panel` (not `ui_surface_show`) // so the rolling-snapshot reducer never folds it into the transcript. // Because the event is never persisted, success is gated on a client // acknowledgment — "displayed" must mean a client actually rendered the // drawer, otherwise the model announces a panel the user cannot see. if (surfaceType === "channel_setup") { const ack = await openChannelSetupPanel( ctx, surfaceId, rawData as Record, { signal }, ); if (ack.status === "submitted" && ack.actionId === "ack") { return { content: JSON.stringify({ surfaceId, status: "displayed" }), isError: false, }; } if (ack.status === "submitted") { // Client received the event but could not open the panel (nack). const reason = typeof ack.submittedData?.reason === "string" ? ack.submittedData.reason : "unknown"; return { content: `The channel setup panel could not be opened by the connected client (reason: ${reason}). Do NOT tell the user the panel is open. Troubleshoot with the user (e.g. ask them to reopen or refresh the Vellum app), then retry ui_show.`, isError: true, }; } if (ack.status === "timed_out") { return { content: "No connected client confirmed opening the channel setup panel. Do NOT tell the user the panel is open. The user's app may be closed, viewing a different conversation, or running a version that cannot show this panel. Ask the user to open this conversation in the Vellum app (web or desktop), then retry ui_show.", isError: true, }; } // cancelled — no interactive client, or the turn was aborted. return { content: "The channel setup panel could not be opened — no connected client can render interactive UI. Do NOT tell the user the panel is open. Ask the user to open the Vellum app (web or desktop), then retry ui_show.", isError: true, }; } // Every surface type parses through its canonical schema. Card, // copy_block, dynamic_page, and visual first run bespoke normalizers // that recover fields the model placed at the top level of the tool // input; the rest go straight through `safeParseSurfaceData`. Choice and // oauth_connect parse into named bindings so their content guards below // read typed data instead of re-narrowing the union. const cardData = surfaceType === "card" ? normalizeCardShowData(input, rawData) : undefined; const choiceData = surfaceType === "choice" ? ChoiceSurfaceDataSchema.parse(rawData) : undefined; const oauthData = surfaceType === "oauth_connect" ? OAuthConnectSurfaceDataSchema.parse(rawData) : undefined; const showPair: SurfaceShowPair = cardData !== undefined ? { surfaceType: "card", data: cardData } : choiceData !== undefined ? { surfaceType: "choice", data: choiceData } : oauthData !== undefined ? { surfaceType: "oauth_connect", data: oauthData } : surfaceType === "copy_block" ? { surfaceType, data: normalizeCopyBlockShowData(input, rawData), } : surfaceType === "dynamic_page" ? { surfaceType, data: normalizeDynamicPageShowData(input, rawData), } : surfaceType === "visual" ? { surfaceType, data: normalizeVisualShowData(input, rawData), } : parseShowPairOrThrow(surfaceType, rawData); // Parse actions through the schema instead of typecasting raw model output. // The model may place actions inside `data` instead of the top-level // `actions` param — recover them so they aren't silently dropped. const rawActions = Array.isArray(input.actions) ? input.actions : Array.isArray(rawData.actions) ? rawData.actions : undefined; let inputActions: z.infer[] | undefined; if (rawActions) { const valid: z.infer[] = []; for (const raw of rawActions) { const result = ModelActionSchema.safeParse(raw); if (result.success) { valid.push(result.data); } } inputActions = valid.length > 0 ? valid : undefined; } const isActionless = ACTIONLESS_SURFACE_TYPES.has(surfaceType); const actions = isActionless ? undefined : choiceData !== undefined ? buildChoiceActions(choiceData) : inputActions; const hasActions = Array.isArray(actions) && actions.length > 0; if (surfaceType === "choice" && !hasActions) { return { content: "choice surfaces require at least one option with both id and title.", isError: true, }; } if (cardData !== undefined) { const hasTitle = (typeof title === "string" && title.trim().length > 0) || (typeof cardData.title === "string" && cardData.title.trim().length > 0); const hasBody = typeof cardData.body === "string" && cardData.body.trim().length > 0; const hasSubtitle = typeof cardData.subtitle === "string" && cardData.subtitle.trim().length > 0; const hasMetadata = Array.isArray(cardData.metadata) && cardData.metadata.length > 0; const hasTemplate = typeof cardData.template === "string"; if ( !hasTitle && !hasBody && !hasSubtitle && !hasMetadata && !hasTemplate && !hasActions ) { return { content: "Error: ui_show card requires content — provide `data.body`, a `template` (e.g. task_progress with steps), `data.metadata`, `data.subtitle`, a `title`, or `actions`. The surface was not displayed because it carried no renderable content. Resend ui_show with populated card content.", isError: true, }; } } if (oauthData !== undefined && oauthData.providerKey.length === 0) { return { content: "oauth_connect surfaces require data.providerKey.", isError: true, }; } const isInteractive = surfaceType === "card" ? hasActions : surfaceType === "list" ? hasActions : surfaceType === "table" ? hasActions : INTERACTIVE_SURFACE_TYPES.includes(surfaceType); // An explicit `await_action: true` is honored for every other type; an // actionless surface has nothing to await, so it is forced false. const awaitAction = isActionless ? false : ((input.await_action as boolean) ?? isInteractive); // Only one non-persistent interactive surface at a time. If another // surface is already awaiting user input, reject this one so the LLM // presents surfaces sequentially. dynamic_page and visual are rendered // content rather than a question posed to the user, so a pending one // never blocks the next surface. if (awaitAction) { const hasExistingPending = [...ctx.pendingSurfaceActions.values()].some( (entry) => !NON_BLOCKING_PENDING_SURFACE_TYPES.has(entry.surfaceType), ); if (hasExistingPending) { return { content: "Another interactive surface is already awaiting user input. Present one at a time — wait for the user to respond to the current surface before showing the next.", isError: true, }; } } const display = (input.display as string) === "panel" ? "panel" : "inline"; // `persistent: true` keeps the card visible through action clicks (only // marks the clicked action as spent). Forward the flag so // `SurfaceManager.showSurface` on the client sees it — without this the // field is dropped and every card dismisses on first click. const persistent = input.persistent === true ? true : undefined; const mappedActions = actions?.map((a) => ({ id: a.id, label: a.label, style: a.style ?? "secondary", ...(a.data ? { data: a.data } : {}), })); // Optional activation-rail telemetry tag. Daemon-only metadata: validated // and ignored if invalid; never forwarded to the client. const activationMoment = typeof input.activation_moment === "string" && isActivationMomentParam(input.activation_moment) ? input.activation_moment : undefined; // Show-timing moments (`first_wow_executed`) record the instant the surface // renders — a display-only result/`work_result` card may never be committed, // so a commit-time emit would never fire. We record now and do NOT store the // tag, so the commit path won't double-emit. Commit-timing moments are // stored and recorded when the user commits the surface (see // `handleSurfaceAction` → `maybeEmitActivationMoment`). const storeTagForCommit = activationMoment !== undefined && !activationMomentEmitsAtShow(activationMoment); // Track surface state for ui_update merging (includes actions so we can // look up per-action data payloads when the client sends an action back). ctx.surfaceState.set(surfaceId, { title, actions: mappedActions, ...(storeTagForCommit ? { activationMoment } : {}), ...showPair, }); if (activationMoment !== undefined && !storeTagForCommit) { recordActivationMoment(ctx, activationMoment); } log.info( { surfaceId, surfaceType, title, dataKeys: Object.keys(showPair.data), actionCount: mappedActions?.length ?? 0, display, persistent: persistent ?? false, conversationId: ctx.conversationId, }, "Sending ui_surface_show to client", ); ctx.emit({ type: "ui_surface_show", conversationId: ctx.conversationId, surfaceId, title, actions: mappedActions, display, ...(persistent ? { persistent: true } : {}), ...(toolUseId ? { toolCallId: toolUseId } : {}), ...showPair, }); // Track surface for persistence with the message. The commit-timing // activation tag rides along (daemon-only) so it survives history restore; // show-timing moments aren't stored here (already recorded at render). ctx.currentTurnSurfaces.push({ surfaceId, title, actions: mappedActions, display, ...(persistent ? { persistent: true } : {}), ...(toolUseId ? { toolCallId: toolUseId } : {}), ...(storeTagForCommit ? { activationMoment } : {}), ...showPair, }); if (awaitAction) { ctx.pendingSurfaceActions.set(surfaceId, { surfaceType }); return { content: JSON.stringify({ surfaceId, status: "awaiting_user_action", message: "Surface displayed and the user can see it. Their response will arrive as a follow-up message. Do not output any waiting message — just stop here.", }), isError: false, yieldToUser: true, }; } if (surfaceType === "visual") { return { content: JSON.stringify({ surfaceId, note: "The visual is now visible inline in the chat. Continue your response in prose and do not describe or restate what the visual shows. To replace it, call ui_dismiss with this surfaceId and show a new one.", }), isError: false, }; } return { content: JSON.stringify({ surfaceId }), isError: false }; } if (toolName === "ui_update") { const surfaceId = typeof input.surface_id === "string" ? input.surface_id : ""; let patch = coerceSurfaceDataRecord(input.data); // Merge the partial patch into the stored full surface data const stored = ctx.surfaceState.get(surfaceId); let mergedData: AnySurfaceData; let mergedPair: SurfaceShowPair | undefined; if (stored) { if (stored.surfaceType === "card") { patch = normalizeTaskProgressCardPatch(stored.data, patch); } // Push current HTML to undo stack for dynamic pages if (stored.surfaceType === "dynamic_page") { pushUndoState(ctx.surfaceUndoStacks, surfaceId, stored.data.html); } const rawMerged = { ...stored.data, ...patch }; if (!isKnownSurfaceType(stored.surfaceType)) { // Restore preserves an unknown-but-non-empty persisted `surfaceType` // verbatim (a newer/custom client-rendered surface). There is no // canonical schema to validate or normalize against — and indexing // `SURFACE_DATA_SCHEMAS` with it would read `undefined` and throw — so // forward the merge opaquely rather than dropping the update, mirroring // restore's verbatim handling of the same types. mergedData = rawMerged as AnySurfaceData; ctx.surfaceState.set(surfaceId, { ...stored, data: rawMerged, } as SurfaceStateEntry); } else { // Validate the merged data through the surface type's canonical schema // so malformed patches (e.g. metadata as a string) are caught here // instead of crashing the client's safeParse. Keep unmodeled // client-owned keys from the raw merge, but take the schema-NORMALIZED // value for every modeled field: a tolerant field (e.g. dynamic_page // `html` is `z.string().catch("")`, file_upload `acceptedTypes` coerces // to `string[]`) can accept a malformed input and coerce it, and later // daemon code (active-workspace injection's `truncateHtml`, undo-stack // pushes) assumes the canonical shape — so storing the raw value would // poison `surfaceState`. `mergedPair.data` holds only modeled keys, so // spreading it over `rawMerged` normalizes those while preserving the // client-owned ones. Only a failed parse reverts to the stored data. mergedPair = buildSurfaceShowPair(stored.surfaceType, rawMerged); if (mergedPair !== undefined) { const normalized = { ...rawMerged, ...mergedPair.data }; mergedData = normalized as AnySurfaceData; ctx.surfaceState.set(surfaceId, { ...stored, data: normalized, } as SurfaceStateEntry); } else { log.warn( { surfaceId, surfaceType: stored.surfaceType }, "ui_update patch produced invalid merged data; reverting to stored data", ); mergedData = safeParseSurfaceData(stored.surfaceType, stored.data) ?? stored.data; } } } else { // No stored state for this surfaceId, so its surface type — and // therefore its canonical schema — is unknown; forward the patch // opaquely rather than dropping the update. mergedData = patch; } ctx.emit({ type: "ui_surface_update", conversationId: ctx.conversationId, surfaceId, data: mergedData, }); // Keep the persisted snapshot in sync so updates survive conversation // restart. This must track EVERY branch that changed `mergedData` — not // just the schema-parsed one — because the turn-end persist loop writes // `currentTurnSurfaces[idx].data` to the same `ui_surface` block that the // debounced `scheduleSurfaceDataPersist` below writes. If an unknown-type // (opaquely forwarded) update updated `surfaceState` but not this array, // the two writers would persist divergent data and race on which lands // last. Gating on `idx !== -1` alone keeps all three in lockstep. const idx = ctx.currentTurnSurfaces.findIndex( (s) => s.surfaceId === surfaceId, ); if (idx !== -1) { ctx.currentTurnSurfaces[idx] = { ...ctx.currentTurnSurfaces[idx], data: mergedData, } as CurrentTurnSurface; } // Persist the merged data back to the assistant message's // `ui_surface` content block so a refresh / restart shows the // current state instead of the original creation-time snapshot. // Debounced to coalesce bursts of rapid updates. scheduleSurfaceDataPersist(ctx.conversationId, surfaceId, mergedData); return { content: "Surface updated", isError: false }; } if (toolName === "ui_dismiss") { const surfaceId = typeof input.surface_id === "string" ? input.surface_id : ""; const lastAction = ctx.lastSurfaceAction.get(surfaceId); const stored = ctx.surfaceState.get(surfaceId); if (lastAction) { const summary = buildCompletionSummary( stored?.surfaceType, lastAction.actionId, lastAction.data, stored?.data, ); // A `ui_show` surface owns a persisted block, so the completion has to // land before the client is told the card is answered. if (!markSurfaceCompleted(ctx, surfaceId, summary)) { // Tearing the live entries down here would strand the card that the // next reseed restores, so leave them for a retry and tell the model // the dismissal did not land. return { content: "Could not dismiss the surface: persisting its completion failed. The card is still showing, so try ui_dismiss again.", isError: true, }; } ctx.emit({ type: "ui_surface_complete", conversationId: ctx.conversationId, surfaceId, summary, submittedData: lastAction.data, }); } else { ctx.emit({ type: "ui_surface_dismiss", conversationId: ctx.conversationId, surfaceId, }); // The live client drops a dismissed surface entirely. Mirror that in // persisted state: pull it from the pending turn snapshot (appended to // the message at turn completion) and strip any already-persisted block, // so a reload does not resurrect a half-finished progress card. const turnIdx = ctx.currentTurnSurfaces.findIndex( (s) => s.surfaceId === surfaceId, ); if (turnIdx !== -1) { ctx.currentTurnSurfaces.splice(turnIdx, 1); } removeSurfaceBlock(ctx, surfaceId); } ctx.pendingSurfaceActions.delete(surfaceId); ctx.surfaceState.delete(surfaceId); ctx.surfaceUndoStacks.delete(surfaceId); ctx.lastSurfaceAction.delete(surfaceId); ctx.accumulatedSurfaceState.delete(surfaceId); return { content: lastAction ? "Surface completed" : "Surface dismissed", isError: false, }; } if (toolName === "app_open") { // An app surface only renders on a connected client that supports dynamic // UI. On clientless, background, or non-dynamic-UI channels (e.g. Slack) // the ui_surface_show below reaches no renderer, so fail closed with an // actionable error rather than reporting a surface the user cannot see. if (!canShowInteractiveUi(ctx)) { return { content: "app_open needs a connected client that can display app surfaces (the Vellum macOS or web app), and none is available for this conversation. The app is saved — the user can open it from a connected client.", isError: true, }; } const appId = resolveAppId(input, ctx.conversationId) ?? ""; const preview = isPlainObject(input.preview) ? DynamicPagePreviewSchema.parse(input.preview) : undefined; const openMode = typeof input.open_mode === "string" ? input.open_mode : undefined; const app = appId ? getApp(appId) : null; if (!app) { return { content: appId ? `App not found: ${appId}` : "app_id is required and no active app exists in this conversation. Call app_create first, or pass app_id explicitly.", isError: true, }; } // Track conversation association (best-effort — failures must not break open flow). try { linkAppToConversationLineage(appId, ctx.conversationId); } catch (err) { log.warn({ err, appId }, "Failed to track conversation ID on app_open"); } // Generate a minimal fallback preview from app metadata so that the // surface is always rendered as a clickable preview card (not an // un-clickable fallback chip) after conversation restart. const defaultPreview = { title: app.name, subtitle: app.description }; const storedPreview = getAppPreview(app.id); const { dirName } = resolveAppDir(app.id); // Auto-compile if dist is missing, then resolve HTML from compiled // dist/index.html with inlined assets. const { existsSync } = await import("node:fs"); const { join } = await import("node:path"); const appDir = getAppDirPath(app.id); const distIndex = join(appDir, "dist", "index.html"); if (!existsSync(distIndex)) { const { compileApp } = await import("../bundler/app-compiler.js"); const result = await compileApp(appDir); if (!result.ok) { log.warn( { appId, errors: result.errors }, "Auto-compile failed on app_open", ); } } const html = resolveEffectiveAppHtml(app); const surfaceData: DynamicPageSurfaceData = { html, appId: app.id, dirName, preview: buildAppOpenPreview(defaultPreview, preview, storedPreview), }; const surfaceId = uuid(); if (openMode === "preview") { // Inline-only preview card emitted during app_create — do not open a // workspace panel and do not register surface state. The client renders // this as a tappable inline card that opens the app on demand. ctx.emit({ type: "ui_surface_show", conversationId: ctx.conversationId, surfaceId, surfaceType: "dynamic_page", title: app.name, data: surfaceData, display: "inline", ...(toolUseId ? { toolCallId: toolUseId } : {}), } as UiSurfaceShow); // Track for message persistence so the inline card survives history reload. ctx.currentTurnSurfaces.push({ surfaceId, surfaceType: "dynamic_page", title: app.name, data: surfaceData, display: "inline", ...(toolUseId ? { toolCallId: toolUseId } : {}), }); return { content: JSON.stringify({ surfaceId, appId }), isError: false }; } ctx.surfaceState.set(surfaceId, { surfaceType: "dynamic_page", data: surfaceData, title: app.name, }); ctx.emit({ type: "ui_surface_show", conversationId: ctx.conversationId, surfaceId, surfaceType: "dynamic_page", title: app.name, data: surfaceData, ...(toolUseId ? { toolCallId: toolUseId } : {}), } as UiSurfaceShow); // Track surface for persistence ctx.currentTurnSurfaces.push({ surfaceId, surfaceType: "dynamic_page", title: app.name, data: surfaceData, ...(toolUseId ? { toolCallId: toolUseId } : {}), }); ctx.pendingSurfaceActions.set(surfaceId, { surfaceType: "dynamic_page" }); return { content: JSON.stringify({ surfaceId, appId }), isError: false }; } return { content: `Unknown proxy tool: ${toolName}`, isError: true }; }