import { wrapUntrusted } from '../tools/untrusted-envelope.js' import type { TaskHandle, TaskScheduler } from '../types/agent/scheduler.js' import { isTerminalAgentTaskState } from '../types/agent/task.js' import type { TaskId } from '../types/ids/index.js' import { SCOPE_ATTRIBUTE } from '../utils/log/types.js' import { type Logger, resolveLogger } from '../utils/logger.js' /** * How many unclaimed announcements may wait for an owner at once. * * Derived from what it has to survive rather than picked. An entry lives here * only between a gateway announcing a task and this turn saying whether the * task is its own — one microtask, for a launch made through `create_task`. * The number that has to fit is therefore the largest batch of launches that * can be in flight together before any of them is claimed: one assistant turn * of `create_task` blocks, which this codebase's own tool description * illustrates as "fan out 8 specialists" and which a provider bounds at a few * dozen tool_use blocks per response. 32 clears that with room, and a batch * bigger than it is announced rather than silently truncated. * * The ceiling is what stops this being the retention half of the leak it * exists beside: on a gateway shared with other turns, every foreign completion * lands here and is never claimed, and each one holds a whole worker result — * kilobytes at least. Bounded, the cost is 32 handles; unbounded, it is every * result every other turn on that gateway ever produced. */ const UNOWNED_BUFFER_LIMIT = 32 /** * How many owned tasks {@link CompletionInbox.describeOwnedWork} can name at * once. * * Unrelated to {@link UNOWNED_BUFFER_LIMIT} — that one bounds a memory leak; * this one bounds a request payload. `describeOwnedWork`'s output is admitted * into the request through `appendWorkContext`'s 8,000-character / ~2,000-token * gate (`runtime/query/iteration/index.ts`), and 16 tasks' worth of scheduler * state fits that with room to spare, including the overflow note below. * * Running tasks fill these slots first — see {@link CompletionInbox.runningOwned} * — so a task still going is never bumped out by launch count alone the way a * single FIFO over every owned task used to bump it. Only once every running * task has a slot do the most recently SETTLED tasks take what is left. */ const OWNED_WORK_DISPLAY_LIMIT = 16 /** * Completions that finished with nobody left to hear them. * * A worker's result reaches the supervisor as the `tool_result` of the * `create_task` that launched it. That works whenever the launching call is * still the live path — but it is not the only way a task ends: * * - the launching tool hit its deadline and the executor returned * *"timed out… it may still be running"* to the model. The worker then * finished normally, holding a result nothing would ever read. * - the task was launched in the background on purpose, so there is no * call waiting on it by design. * * In both cases the completion exists, the gateway remembers it, and the * model is never told. That is the gap this closes: the turn subscribes once, * every settled task lands here, and anything a tool did NOT hand over * inline is drained into the transcript as a notification the next turn can * read. * * The disambiguation is the whole design. An earlier version of the envelope * path was removed (`dc16d58`) because it fired for completions the blocking * tool had ALREADY delivered, so the supervisor saw every result twice — * once correctly as a `tool_result`, once as an orphan envelope. Removing it * fixed the duplicate and left the abandoned case with no channel at all. * Claiming is what tells the two apart: a tool that delivers a completion * says so, and only unclaimed completions become envelopes. * * It attaches through `onTaskCompleted`, which every `TaskScheduler` already * has, so a host gateway needs no change to take part — a host that was * firing completions into a listener set with no listeners now has one. */ export class CompletionInbox { private readonly unheard = new Map() private readonly claimed = new Set() /** Launched with nothing waiting on it, and not settled yet. */ private readonly outstanding = new Set() /** * Tasks THIS turn launched. * * `onTaskCompleted` is a broadcast and `TaskHandle` carries no turn id, so * a gateway shared between two supervisors hands every completion to both * of their inboxes. Measured: with two inboxes on one gateway, the turn * that launched nothing drained the other turn's task and would have been * told "a task you launched has finished" — a claim that was false, over * another turn's worker output, in a transcript whose model then has to * account for it. * * A shared gateway is not an abuse of the API: `SupervisorAgentConfig` * takes one, and a host that owns a gateway naturally reuses it. */ private readonly ours = new Set() /** * Owned tasks not yet observed to have settled, in launch order. * * A `Set`'s iteration order is insertion order, so the most recently * launched entry is always last — {@link describeOwnedWork} reads that * order back to front. An id leaves this set the moment its completion is * observed, by whichever of the three paths sees it first: the gateway's * broadcast, an announcement recovered from {@link unowned}, or a launch * that finds the task already terminal. It is never evicted for any other * reason, so a long-running task stays in here — and therefore visible — * for exactly as long as it is actually running, regardless of how many * other tasks this turn launches meanwhile. */ private readonly runningOwned = new Set() /** * Owned tasks observed to have settled, oldest first, bounded to * {@link OWNED_WORK_DISPLAY_LIMIT}. * * This bound is display-driven, not a memory concern the way * {@link UNOWNED_BUFFER_LIMIT} is: a settled task's own result already * reached the model once (inline or as a notification), so dropping it * from here loses a convenience, not the only record of it. */ private readonly settledOwned: TaskId[] = [] /** * Announcements that arrived before anyone said whose task it was. * * `gateway.createTask` resolves one microtask before its caller can name * the task, and a worker that finishes inside that window is announced * first — `LocalTaskScheduler` attaches its completion continuation before * it returns the handle, so the ordering is guaranteed to be reachable * rather than merely possible. Dropping an unowned announcement outright * would therefore turn the leak fix into a LOST RESULT for exactly the * fast completions the inbox exists to catch. * * So they wait here, and ownership may be claimed retroactively. What * makes that safe rather than a second leak is the bound: on a gateway * shared with other turns this fills with completions that will never be * claimed, each holding a whole worker result. */ private readonly unowned = new Map() private readonly arrivals = new Set<() => void>() private detach?: () => void /** Kept for {@link launched}: the source of truth about a task's state. */ private gateway?: TaskScheduler /** * `log` is optional and unresolved until it is actually needed (the * eviction-warning path in `hold()`), same reason as `LocalTaskScheduler`. */ constructor(private readonly log?: Logger) {} /** * Start listening. * * Returns a detach function; calling `attach` twice is a no-op rather * than a second subscription, because a doubly-attached inbox would * queue every completion twice and reproduce the exact duplicate this * class exists to prevent. */ attach(gateway: TaskScheduler): () => void { if (this.detach) return this.detach this.gateway = gateway this.detach = gateway.onTaskCompleted((handle) => { // Not known to be ours — either another turn's worker on a shared // gateway, or ours announced before the launch could be recorded. // The two are indistinguishable here, so it waits rather than // being delivered or dropped. See {@link unowned}. if (!this.ours.has(handle.taskId)) { this.hold(handle) return } // A completion claimed before it was announced — a tool that // finished its wait faster than the listener ran — is already // delivered. Nothing to queue. this.outstanding.delete(handle.taskId) this.markSettled(handle.taskId) if (this.claimed.has(handle.taskId)) return this.unheard.set(handle.taskId, handle) for (const wake of this.arrivals) wake() }) return this.detach } /** * Park an announcement nobody has claimed yet, evicting the oldest if the * buffer is full. * * An eviction is logged at WARN, and that is not decoration. If the entry * turned out to be ours, its completion has just been dropped — the * original defect, wearing the cap as a disguise — and the only evidence * would otherwise be an absence, which is precisely the shape of failure * this whole session has been closing. A reader who sees this line knows * where to look. */ private hold(handle: TaskHandle): void { if (this.unowned.size >= UNOWNED_BUFFER_LIMIT && !this.unowned.has(handle.taskId)) { const oldest = this.unowned.keys().next() if (!oldest.done) { this.unowned.delete(oldest.value) resolveLogger(this.log) .child({ [SCOPE_ATTRIBUTE]: 'scheduler/completion-inbox' }) .warn( "Unclaimed completion buffer is full — dropped the oldest. If that task was this turn's, its result is now unreachable; raise UNOWNED_BUFFER_LIMIT or launch fewer tasks per turn.", { 'namzu.scheduler.dropped': oldest.value, 'namzu.scheduler.limit': UNOWNED_BUFFER_LIMIT, }, ) } } this.unowned.set(handle.taskId, handle) } /** * Say that this turn launched the task. * * Required before anything about the task can reach this inbox — see * {@link ours}. Every launch says it, whether or not something is waiting * on the result, because the case the inbox exists for is precisely the * one where the waiter gave up. * * **The late-announcement branches are not defensive.** * `gateway.createTask` resolves one microtask before its caller can say * who owns the task, and a worker that finishes inside that window is * announced first — the same ordering that used to leave a permanent * pending flag. Without recovery the ownership check would turn that race * from a stale flag into a lost result. * * There are two, and the second is the safety net for the first. The * buffer ({@link unowned}) needs nothing from the gateway beyond the * announcement it already made. Asking `getTask` covers the case the * buffer cannot — an announcement evicted under load — but it rests on a * gateway still knowing about a task it has just settled, which is a * property of the implementations here rather than of the `TaskScheduler` * contract, and `getTask`'s own docs now say so. */ launched(taskId: TaskId): void { if (this.ours.has(taskId)) return this.ours.add(taskId) if (this.claimed.has(taskId) || this.unheard.has(taskId)) { this.markSettled(taskId) return } const parked = this.unowned.get(taskId) if (parked) { this.unowned.delete(taskId) this.unheard.set(taskId, parked) this.markSettled(taskId) for (const wake of [...this.arrivals]) wake() return } const settled = this.gateway?.getTask(taskId) if (!settled || !isTerminalAgentTaskState(settled.state)) { // Not observed terminal by any of the checks above: still running, // as far as this inbox knows. this.runningOwned.add(taskId) return } this.unheard.set(taskId, settled) this.markSettled(taskId) for (const wake of [...this.arrivals]) wake() } /** * Move an owned task from {@link runningOwned} to {@link settledOwned}. * * Idempotent: `claim` and the completion listener can both observe the * same settlement (the race either can win), and re-marking a task that * is already in {@link settledOwned} moves it to the most-recently-settled * end rather than duplicating it there. */ private markSettled(taskId: TaskId): void { this.runningOwned.delete(taskId) const at = this.settledOwned.indexOf(taskId) if (at !== -1) this.settledOwned.splice(at, 1) this.settledOwned.push(taskId) if (this.settledOwned.length > OWNED_WORK_DISPLAY_LIMIT) this.settledOwned.shift() } /** * Say that a task was launched with nothing waiting on it. * * {@link launched} plus the statement that no call will deliver the * result. Without the second half the inbox can only see completions that * have already happened, and a turn whose supervisor launched a background * worker and then answered would settle while the worker was still going — * throwing away the very result the launch existed to produce. Knowing a * task is outstanding is what lets the loop hold the turn open for it. */ expect(taskId: TaskId): void { this.launched(taskId) if (this.claimed.has(taskId)) return this.outstanding.add(taskId) } /** Whether anything is either waiting to be told or still running. */ get hasPendingWork(): boolean { return this.unheard.size > 0 || this.outstanding.size > 0 } /** * Wait for the next completion, deadline or abort, whichever comes first. * Aborting releases only this waiter; work and undelivered results remain owned. * * Bounded on purpose. A worker that never finishes must not hold a turn * open forever, and the caller decides how long "long enough" is — the * turn's own budget is the only thing that knows. */ waitForArrival(timeoutMs: number, signal?: AbortSignal): Promise { if (signal?.aborted) return Promise.resolve() if (this.unheard.size > 0) return Promise.resolve() if (this.outstanding.size === 0) return Promise.resolve() return new Promise((resolve) => { const finish = (): void => { clearTimeout(timer) this.arrivals.delete(finish) signal?.removeEventListener('abort', finish) resolve() } const timer = setTimeout(finish, timeoutMs) // `unref` where the runtime has it, so a pending wait never keeps // a process alive past the work it was waiting for. ;(timer as { unref?: () => void }).unref?.() this.arrivals.add(finish) signal?.addEventListener('abort', finish, { once: true }) if (signal?.aborted) finish() }) } /** * Say that this completion reached the model as a `tool_result`. * * Idempotent, and safe to call before the completion is announced: the * claim is remembered so a late announcement does not re-queue it. */ claim(taskId: TaskId): void { this.claimed.add(taskId) this.unheard.delete(taskId) this.outstanding.delete(taskId) // A tool only claims a result it is already holding, so a claim is // itself proof of settlement — independent of whether the gateway's // own broadcast has reached this inbox yet. Without this, a task // claimed ahead of its announcement (see the race above) stayed in // {@link runningOwned} until that announcement arrived, understating // its own "still running" count in the meantime. this.markSettled(taskId) } /** Whether anything is waiting to be told. */ get hasUnheard(): boolean { return this.unheard.size > 0 } /** * Bounded, non-consuming scheduler observations. Delivery never * establishes user-facing completion. * * Running tasks — most recently launched first, matching the order the * rest of this projection has always used — fill the {@link * OWNED_WORK_DISPLAY_LIMIT} slots first, so a task still going is named * here for as long as it keeps running, no matter how many other tasks * this turn has since launched. The most recently SETTLED tasks fill * whatever slots running tasks leave over. A settled task bumped out * entirely is not reported missing — its own result already reached the * model once — but a RUNNING task that does not fit is: the preamble * says exactly how many, so the model never mistakes the cap for the * task having ended. */ describeOwnedWork(): string | undefined { if (this.ours.size === 0) return const running = [...this.runningOwned].reverse() const shownRunning = running.slice(0, OWNED_WORK_DISPLAY_LIMIT) const stillRunningNotShown = running.length - shownRunning.length const settled = [...this.settledOwned].reverse() const shownSettled = settled.slice(0, OWNED_WORK_DISPLAY_LIMIT - shownRunning.length) const ids = [...shownRunning, ...shownSettled] const tasks = ids.map((taskId) => { let handle = this.unheard.get(taskId) try { handle = this.gateway?.getTask(taskId) ?? handle } catch { // A broken observation must not block execution or invent a task state. } return { taskId, state: handle?.state ?? 'unknown', ...(handle?.result?.status ? { turnStatus: handle.result.status } : {}), ...(handle?.result?.stopReason ? { stopReason: handle.result.stopReason } : {}), resultDelivery: this.claimed.has(taskId) ? 'delivered-to-history' : 'not-delivered', } }) const overflow = stillRunningNotShown > 0 ? ` ${shownRunning.length} running tasks are shown below, and ${stillRunningNotShown} more still running.` : '' return `Owned delegated work (current scheduler observations).${overflow} Result delivery is not proof that you answered the original request. Answer the latest operator message and complete the requested synthesis of available results unless the operator cancelled or changed that request. Reuse delivered results when sufficient; do not relaunch completed work. If a delivered result is no longer visible, retrieve it by task ID. A terminal state with a non-end_turn stop reason is not successful task completion. Do not repeat a synthesis already given.\n${JSON.stringify({ tasks, omitted: this.ours.size - ids.length })}` } /** * Tasks this turn launched that are still running. * * Read when a turn ends, so it can say which work it walked away from. * Nothing here is cancelled by being read — the ids are a statement, and * what to do about them is the host's call. */ get outstandingTaskIds(): readonly TaskId[] { return [...this.outstanding] } /** * Take every unheard completion, leaving the inbox empty. * * Draining rather than peeking: a notification that stays queued after * being delivered is the duplicate-delivery bug in a different costume. */ drain(): TaskHandle[] { if (this.unheard.size === 0) return [] const handles = [...this.unheard.values()] this.unheard.clear() for (const handle of handles) { this.claimed.add(handle.taskId) // A delivered result is not pending WORK, and `outstanding` can // still be holding this id — the listener clears it, but only if // the announcement came AFTER the launching tool said `expect`. // The other order is reachable: `expect` runs one microtask after // `gateway.createTask` resolves, and the gateway's own completion // callback can win that race for a task that finished fast. Then // `expect` re-adds an id the listener had nothing to remove, and // nothing else ever takes it off — so `hasPendingWork` stayed true // for the rest of the turn and every attempt to settle paid the // full grace period waiting for a result already in the transcript. // // Symmetric with `claim`, which clears it for the same reason. this.outstanding.delete(handle.taskId) } return handles } /** * Stop expecting a task that is never going to arrive. * * Cancelling is the case this exists for. `expect` puts a task on the * outstanding list and only a COMPLETION takes it off, so a cancelled * worker left `hasPendingWork` true for the rest of the turn — and every * attempt to settle then paid the full grace period waiting for a result * that had been called off. */ forget(taskId: TaskId): void { this.outstanding.delete(taskId) // `unheard` is deliberately NOT touched. // // The two sets mean different things. `outstanding` is pending WORK, // and cancelling is exactly the statement that it should stop being // waited for. `unheard` is a RESULT that already exists — the worker // finished, the completion arrived, and it is queued for the next // drain. Clearing it here destroyed that. // // The window is small and entirely reachable: nothing has told the // model the worker finished, and `cancel_task` says it cancels a // running task, so cancelling one that has just completed is the // obvious move rather than a mistake. The turn then reports "cancelled" // over work that was done and output that no longer exists anywhere. // // Note the asymmetry with `claim`, which does clear `unheard` — and is // right to, because there a tool has just handed the model the same // result. This one hands over nothing. for (const wake of [...this.arrivals]) wake() } /** * Stop listening. Safe to call more than once. * * A turn that ends without this leaves its listener on the gateway * forever. On a gateway the host reuses that is measurable — three * sequential turns left three live subscriptions, each still holding its * turn's handles — and the listener set only grows. Ownership stops a * retained listener from DELIVERING another turn's work; closing is what * stops it existing. */ close(): void { this.detach?.() this.detach = undefined this.gateway = undefined this.unheard.clear() this.outstanding.clear() this.ours.clear() this.runningOwned.clear() this.settledOwned.length = 0 this.claimed.clear() this.unowned.clear() // Release anyone still waiting. A closed inbox would otherwise hold // them to their own deadline for a completion that can no longer come. for (const wake of [...this.arrivals]) wake() this.arrivals.clear() } } /** How much of a worker's output rides in the notification itself. */ const NOTIFICATION_OUTPUT_LIMIT = 4_000 const NOTIFICATION_DELIMITER = /task-notification/gi /** * Defang this file's own delimiter inside a worker's text. * * Without it a worker whose output contains `` closes the * block early, and everything it wrote after that sits OUTSIDE the boundary — * reading as ordinary transcript rather than as a delegate's material. * Measured before the fix: two closing tags in one notification, with * attacker-controlled text between them. * * The replacement swaps the hyphen for an underscore rather than appending a * suffix, for the reason `neutralizeEnvelopeDelimiter` records: a replacement * that still CONTAINS the token is found again by a second pass or by any * looser matcher downstream. `task_notification` shares no substring with the * real delimiter while staying legible. * * The nested `` block defangs its own delimiter; these two * patterns are disjoint, so the order they run in does not matter. */ function neutralizeNotificationDelimiter(content: string): string { return content.replace(NOTIFICATION_DELIMITER, 'task_notification') } /** * The message a supervisor reads when a worker it stopped waiting for * finishes. * * It carries the task id, because without one the model cannot say which of * five workers this was, and it carries the output, because a notification * that only says "done" forces exactly the follow-up call this mechanism * exists to remove. Long output is truncated with the task id repeated in * the truncation notice, so the full text stays one `wait_for_task` away and * the model knows which id to ask for — that tool takes a `task_id` and * returns immediately for a task that has already finished, where the * listing takes only a state filter and could not have been followed. */ export function formatCompletionNotification(handles: readonly TaskHandle[]): string { const blocks = handles.map((handle) => { const durationMs = handle.completedAt ? handle.completedAt - handle.createdAt : undefined const turn = handle.result let output = turn?.result || turn?.lastError || '' // A hard guard can stop immediately after a tool round, before the result // assembler has a final answer. Preserve visible partial prose, never reasoning. if (!output && turn?.stopReason && turn.stopReason !== 'end_turn') { const partial = [...(turn.messages ?? [])] .reverse() .find( (message) => message.role === 'assistant' && typeof message.content === 'string' && message.content.length > 0, ) if (partial) output = `Partial output before ${turn.stopReason}:\n${partial.content}` } const overLimit = output.length > NOTIFICATION_OUTPUT_LIMIT const shown = overLimit ? output.slice(0, NOTIFICATION_OUTPUT_LIMIT) : output // Framed for the same reason the blocking `create_task` frames its // return value, and this path is the one that had nothing. A delegated // worker is the component most likely to have consumed material nobody // in this turn authored — it was told to read and report, and it ran // `read`, `grep`, `fetch` over whatever it found — and its text lands // in a parent that typically holds the broader tool grant. The same // bytes were being wrapped on one path and pasted bare on this one. // // The metadata above stays OUTSIDE the envelope: the task id, the agent // and the state are this kernel's own statements, and framing them as // untrusted material would tell the model to discount the only part of // the message it can rely on. const body = shown.length > 0 ? wrapUntrusted( { kind: 'agent-result', attributes: { agent: handle.agentId, task: handle.taskId }, provenance: `This is the output of the delegated agent "${handle.agentId}", not this agent's own work.`, }, neutralizeNotificationDelimiter(shown), ) : '(the task produced no output)' const lines = [ `task_id: ${handle.taskId}`, `agent: ${handle.agentId}`, `state: ${handle.state}`, ...(handle.state === 'completed' && turn?.stopReason && turn.stopReason !== 'end_turn' ? ['outcome: incomplete — execution stopped; this does not establish task completion.'] : []), ...(handle.result?.stopReason ? [`stop_reason: ${handle.result.stopReason}`] : []), ...(durationMs !== undefined ? [`duration_ms: ${durationMs}`] : []), '', body, // Outside the envelope, deliberately: this sentence is an // instruction from the kernel about how to get the rest, and inside // the envelope the model has just been told not to treat the // contents as instructions. // // `wait_for_task`, not `agent_task_list` — the listing takes only a // state filter, so an instruction to call it "with task_id" named a // parameter that does not exist and could not be followed. On an // already-finished task the wait returns immediately. ...(overLimit ? [`… truncated. Call wait_for_task with task_id "${handle.taskId}" for the full output.`] : []), ] return `\n${lines.join('\n')}\n` }) const preamble = handles.length === 1 ? 'A task you launched has finished. This is its result — you were not waiting on it, so it arrives here rather than as a tool result.' : `${handles.length} tasks you launched have finished. These are their results — you were not waiting on them, so they arrive here rather than as tool results.` return `${preamble}\n\n${blocks.join('\n\n')}` }