/* eslint-disable no-console */ // src/graphs/Graph.ts import { nanoid } from 'nanoid'; import { concat } from '@langchain/core/utils/stream'; import { ToolNode } from '@langchain/langgraph/prebuilt'; import { ChatVertexAI } from '@langchain/google-vertexai'; import { START, END, Command, StateGraph, Annotation, messagesStateReducer, } from '@langchain/langgraph'; import { Runnable, RunnableConfig, RunnableLambda, } from '@langchain/core/runnables'; import { ToolMessage, SystemMessage, AIMessageChunk, HumanMessage, } from '@langchain/core/messages'; import type { BaseMessageFields, MessageContent, UsageMetadata, BaseMessage, } from '@langchain/core/messages'; import type { ToolCall } from '@langchain/core/messages/tool'; import type * as t from '@/types'; import { formatAnthropicArtifactContent, ensureThinkingBlockInMessages, deduplicateSystemMessages, getContextUtilization, convertMessagesToContent, addBedrockCacheControl, modifyDeltaProperties, formatArtifactPayload, formatContentStrings, createPruneMessages, addCacheControl, getMessageId, } from '@/messages'; import { GraphNodeKeys, ContentTypes, GraphEvents, Providers, StepTypes, MessageTypes, Constants, TOOL_TURN_THINKING_BUDGET, SUMMARIZATION_CONTEXT_THRESHOLD, PROACTIVE_SUMMARY_THRESHOLD, COMPACTION_RECENT_ROUNDS, } from '@/common'; import { ToolDiscoveryCache, resetIfNotEmpty, isOpenAILike, isGoogleLike, joinKeys, sleep, createPruneCalibration, updatePruneCalibration, applyCalibration, } from '@/utils'; import type { PruneCalibrationState } from '@/types/graph'; import { buildFileManifestBlock } from '@/utils/fileManifest'; import { buildContextAnalytics, type ContextAnalytics, } from '@/utils/contextAnalytics'; import { getChatModelClass, manualToolStreamProviders } from '@/llm/providers'; import { ToolNode as CustomToolNode, toolsCondition } from '@/tools/ToolNode'; import { tool as makeStructuredTool } from '@langchain/core/tools'; import { SubagentExecutor, resolveSubagentConfigs } from '@/tools/subagent'; import { buildSubagentToolParams } from '@/tools/SubagentTool'; import { annotateMessagesForLLM, ToolOutputReferenceRegistry as ToolOutputReferenceRegistryClass, } from '@/tools/toolOutputReferences'; import { executeHooks } from '@/hooks'; import { ChatOpenAI, AzureChatOpenAI } from '@/llm/openai'; import { safeDispatchCustomEvent } from '@/utils/events'; import { mlog, mwarn } from '@/utils/logging'; import { normalizeMessageToolCalls } from '@/utils/toolCallNormalization'; import { isTruncationReason } from '@/utils/finishReasons'; import { isLikelyContextOverflowError } from '@/utils/errors'; import { detectDocuments, shouldInjectMultiDocHint, buildMultiDocHintContent, buildPostPruneNote, hasTaskTool, } from '@/utils/contextPressure'; import { createSchemaOnlyTools } from '@/tools/schema'; import { prepareSchemaForProvider } from '@/schemas/validate'; import { AgentContext } from '@/agents/AgentContext'; import { StructuredOutputRefusalError, StructuredOutputTruncatedError, } from '@/types/graph'; import { createFakeStreamingLLM } from '@/llm/fake'; import { handleToolCalls } from '@/tools/handlers'; import { ChatModelStreamHandler } from '@/stream'; import { HandlerRegistry } from '@/events'; import { StreamingToolCallBuffer } from '@/tools/StreamingToolCallBuffer'; const { AGENT, TOOLS } = GraphNodeKeys; export abstract class Graph< T extends t.BaseGraphState = t.BaseGraphState, _TNodeName extends string = string, > { abstract resetValues(): void; abstract initializeTools({ currentTools, currentToolMap, }: { currentTools?: t.GraphTools; currentToolMap?: t.ToolMap; }): CustomToolNode | ToolNode; abstract initializeModel({ currentModel, tools, clientOptions, }: { currentModel?: t.ChatModel; tools?: t.GraphTools; clientOptions?: t.ClientOptions; }): Runnable; abstract getRunMessages(): BaseMessage[] | undefined; abstract getContentParts(): t.MessageContentComplex[] | undefined; abstract generateStepId(stepKey: string): [string, number]; abstract getKeyList( metadata: Record | undefined ): (string | number | undefined)[]; abstract getStepKey(metadata: Record | undefined): string; abstract checkKeyList(keyList: (string | number | undefined)[]): boolean; abstract getStepIdByKey(stepKey: string, index?: number): string; abstract getRunStep(stepId: string): t.RunStep | undefined; abstract dispatchRunStep( stepKey: string, stepDetails: t.StepDetails, metadata?: Record ): Promise; abstract dispatchRunStepDelta( id: string, delta: t.ToolCallDelta ): Promise; abstract dispatchMessageDelta( id: string, delta: t.MessageDelta ): Promise; abstract dispatchReasoningDelta( stepId: string, delta: t.ReasoningDelta ): Promise; abstract handleToolCallCompleted( data: t.ToolEndData, metadata?: Record, omitOutput?: boolean ): Promise; abstract createCallModel( agentId?: string, currentModel?: t.ChatModel ): (state: T, config?: RunnableConfig) => Promise>; messageStepHasToolCalls: Map = new Map(); messageIdsByStepKey: Map = new Map(); prelimMessageIdsByStepKey: Map = new Map(); config: RunnableConfig | undefined; contentData: t.RunStep[] = []; stepKeyIds: Map = new Map(); contentIndexMap: Map = new Map(); toolCallStepIds: Map = new Map(); signal?: AbortSignal; /** Set of invoked tool call IDs from non-message run steps completed mid-run, if any */ invokedToolIds?: Set; handlerRegistry: HandlerRegistry | undefined; /** Optional hook registry threaded down from Run; consumed by ToolNode for tool-lifecycle hooks. */ hookRegistry?: import('@/hooks').HookRegistry; /** * Optional tool-output reference registry threaded down from Run * (upstream PR #114). When set, every ToolNode built by this graph * stores successful outputs here and resolves `{{toolturn}}` * placeholders in args before invoking the tool. * * @deprecated Prefer `toolOutputReferences` config + the lazy * `getOrCreateToolOutputRegistry()` accessor — keeps the registry * lifecycle explicit (cleared in `clearHeavyState()`). Existing * direct-set callers still work for backwards compat. */ toolOutputRegistry?: import('@/tools/toolOutputReferences').ToolOutputReferenceRegistry; /** * Run-scoped tool output reference configuration (upstream PR #117). * When `enabled` is true, the graph lazily allocates a single * `ToolOutputReferenceRegistry` on first `getOrCreateToolOutputRegistry()` * call and shares it with every ToolNode the graph compiles, so * cross-agent `{{toolturn}}` substitutions resolve. */ toolOutputReferences?: import('@/types/tools').ToolOutputReferencesConfig; /** * Lazy single-instance registry for the run. Constructed on first * `getOrCreateToolOutputRegistry()` call when * `toolOutputReferences.enabled` is true. Cleared (and recreated on * next access) by `clearHeavyState()`. */ private _toolOutputRegistry?: import('@/tools/toolOutputReferences').ToolOutputReferenceRegistry; /** * Returns the shared `ToolOutputReferenceRegistry` for this run, * constructing it on first access. Returns `undefined` when the * feature is disabled. All ToolNodes compiled from this graph share * this single instance so cross-agent `{{...}}` references resolve. * * @internal Public so `attemptInvoke` can read it through the typed * `InvokeContext` and project ToolMessages into LLM-facing annotated * copies right before each provider call (see `annotateMessagesForLLM`). * Host code should not call this directly — registry mutations outside * the ToolNode lifecycle break the partitioning, eviction, and * turn-counter invariants. */ public getOrCreateToolOutputRegistry(): | import('@/tools/toolOutputReferences').ToolOutputReferenceRegistry | undefined { // Direct-set instance (legacy path) takes precedence so existing // Run plumbing keeps working. if (this.toolOutputRegistry != null) { return this.toolOutputRegistry; } if (this.toolOutputReferences?.enabled !== true) { return undefined; } if (this._toolOutputRegistry == null) { this._toolOutputRegistry = new ToolOutputReferenceRegistryClass({ maxOutputSize: this.toolOutputReferences.maxOutputSize, maxTotalSize: this.toolOutputReferences.maxTotalSize, }); } return this._toolOutputRegistry; } /** * Tool session contexts for automatic state persistence across tool invocations. * Keyed by tool name (e.g., Constants.EXECUTE_CODE). * Currently supports code execution session tracking (session_id, files). */ sessions: t.ToolSessionMap = new Map(); /** * Streaming tool call buffer — accumulates raw arg strings during streaming * so that truncated tool call content can be recovered by the ToolNode. * Fed by handleToolCallChunks, consumed by ToolNode.run when args are incomplete. */ streamingToolCallBuffer: StreamingToolCallBuffer = new StreamingToolCallBuffer(); /** * Clears heavy references to allow GC to reclaim memory held by * LangGraph's internal config / AsyncLocalStorage RunTree chain. * Call after a run completes and content has been extracted. */ clearHeavyState(): void { this.config = undefined; this.signal = undefined; this.contentData = []; this.contentIndexMap = new Map(); this.stepKeyIds = new Map(); this.toolCallStepIds.clear(); this.messageIdsByStepKey = new Map(); this.messageStepHasToolCalls = new Map(); this.prelimMessageIdsByStepKey = new Map(); this.invokedToolIds = undefined; this.handlerRegistry = undefined; this.sessions.clear(); this.streamingToolCallBuffer.clearAll(); } } export class StandardGraph extends Graph { overrideModel?: t.ChatModel; /** Optional compile options passed into workflow.compile() */ compileOptions?: t.CompileOptions | undefined; messages: BaseMessage[] = []; runId: string | undefined; startIndex: number = 0; signal?: AbortSignal; /** Cached summary from the first prune in this run. * Reused for subsequent prunes to avoid blocking LLM calls on every tool iteration. */ private _cachedRunSummary: string | undefined; /** EMA-based pruning calibration state — smooths token budget adjustments across iterations */ private _pruneCalibration: PruneCalibrationState; /** Run-scoped tool discovery cache — avoids re-parsing conversation history on every iteration */ private _toolDiscoveryCache: ToolDiscoveryCache; /** * SCALE: Tracks whether a summary call is already in-flight for this Graph instance. * Prevents multiple concurrent summary LLM calls when rapid tool iterations each * trigger pruning. At 2000 users with 3+ tool calls per turn, this prevents * 6000+ summary calls/turn from becoming 2000. */ private _summaryInFlight: boolean = false; /** Messages accumulated across tool iterations while a summary call is in-flight */ private _pendingMessagesToRefine: BaseMessage[] = []; /** Map of agent contexts by agent ID */ agentContexts: Map = new Map(); /** Default agent ID to use */ defaultAgentId: string; /** Normalized finish/stop reason from the last LLM invocation */ lastFinishReason: string | undefined; constructor({ // parent-level graph inputs runId, signal, agents, tokenCounter, indexTokenCountMap, }: t.StandardGraphInput) { super(); this.runId = runId; this.signal = signal; if (agents.length === 0) { throw new Error('At least one agent configuration is required'); } for (const agentConfig of agents) { const agentContext = AgentContext.fromConfig( agentConfig, tokenCounter, indexTokenCountMap ); this.agentContexts.set(agentConfig.agentId, agentContext); } this.defaultAgentId = agents[0].agentId; // Seed cached summary from persisted storage so the first prune in a // resumed conversation can also skip the synchronous LLM summarization call const primaryContext = this.agentContexts.get(this.defaultAgentId); if (primaryContext?.persistedSummary) { this._cachedRunSummary = primaryContext.persistedSummary; } // Initialize EMA pruning calibration this._pruneCalibration = createPruneCalibration(); // Initialize tool discovery cache, seeded with any pre-existing discoveries this._toolDiscoveryCache = new ToolDiscoveryCache(); if (primaryContext?.discoveredToolNames.size) { this._toolDiscoveryCache.seed([...primaryContext.discoveredToolNames]); } } /* Init */ resetValues(keepContent?: boolean): void { this.messages = []; this.lastFinishReason = undefined; this.config = resetIfNotEmpty(this.config, undefined); if (keepContent !== true) { this.contentData = resetIfNotEmpty(this.contentData, []); this.contentIndexMap = resetIfNotEmpty(this.contentIndexMap, new Map()); /** * Clear in-place instead of replacing with a new Map to preserve the * shared reference held by ToolNode (passed at construction time). * Using resetIfNotEmpty would create a new Map, leaving ToolNode with * a stale reference on 2nd+ processStream calls. * * Gated by `keepContent` because HITL resume calls processStream a * second time (`Command({resume})`) with `keepContent: true` to * preserve the in-progress message's contentParts. The toolCallId → * stepId map must survive the same boundary so that when LangGraph * replays the interrupted ToolNode, `handleRunToolCompletions` can * still resolve the resumed tool call's stepId and dispatch * ON_RUN_STEP_COMPLETED with the tool's output. Without this guard, * HITL tools (e.g. `ask_user`) lose their `output` field on every * persisted message — the result is in the LLM context but never * makes it into the saved tool_call entry. */ this.toolCallStepIds.clear(); } this.messageIdsByStepKey = resetIfNotEmpty( this.messageIdsByStepKey, new Map() ); this.messageStepHasToolCalls = resetIfNotEmpty( this.messageStepHasToolCalls, new Map() ); this.prelimMessageIdsByStepKey = resetIfNotEmpty( this.prelimMessageIdsByStepKey, new Map() ); this.invokedToolIds = resetIfNotEmpty(this.invokedToolIds, undefined); // Reset EMA calibration, tool discovery cache, and summary debounce for fresh run this._pruneCalibration = createPruneCalibration(); this._toolDiscoveryCache.reset(); this._summaryInFlight = false; this._pendingMessagesToRefine = []; for (const context of this.agentContexts.values()) { context.reset(); } } override clearHeavyState(): void { super.clearHeavyState(); this.messages = []; this.overrideModel = undefined; for (const context of this.agentContexts.values()) { context.reset(); } } /** * Returns clientOptions with a reduced thinking budget for subsequent * ReAct loop iterations (tool-result turns). * * **Rationale:** The first LLM call in a conversation processes the user's * original query and may benefit from deep extended thinking. Subsequent * iterations — where the model receives tool results and decides whether * to call another tool or produce a final response — require minimal * reasoning. Reducing the thinking budget from the user's configured * value to TOOL_TURN_THINKING_BUDGET (1024 tokens) cuts wall-clock * latency by ~15-20s per iteration, compounding across multi-tool flows. * * Provider handling: * - **Anthropic (direct):** Reduces `thinking.budget_tokens` if > threshold * - **Bedrock (Anthropic models):** Reduces `additionalModelRequestFields.thinking.budget_tokens` * - **VertexAI / Google:** Reduces `thinkingConfig.thinkingBudget` if > threshold * - **All others:** Returns clientOptions unchanged (no-op) * * @param clientOptions - The original client options from AgentContext * @param provider - The LLM provider enum value * @returns Shallow-cloned clientOptions with reduced thinking budget, or the original if no reduction needed */ getAdaptiveClientOptions( clientOptions: t.ClientOptions, provider: Providers ): t.ClientOptions { if (provider === Providers.ANTHROPIC) { const anthropicOpts = clientOptions as t.AnthropicClientOptions; if ( anthropicOpts.thinking != null && typeof anthropicOpts.thinking === 'object' && 'type' in anthropicOpts.thinking && (anthropicOpts.thinking.type === 'enabled' || anthropicOpts.thinking.type === 'adaptive') && 'budget_tokens' in anthropicOpts.thinking && (anthropicOpts.thinking.budget_tokens as number) > TOOL_TURN_THINKING_BUDGET ) { return { ...anthropicOpts, thinking: { ...anthropicOpts.thinking, budget_tokens: TOOL_TURN_THINKING_BUDGET, }, } as t.AnthropicClientOptions; } } if (provider === Providers.BEDROCK) { const bedrockOpts = clientOptions as t.BedrockAnthropicClientOptions; const thinkingField = bedrockOpts.additionalModelRequestFields?.thinking; if ( thinkingField != null && typeof thinkingField === 'object' && 'budget_tokens' in thinkingField && (thinkingField.budget_tokens as number) > TOOL_TURN_THINKING_BUDGET ) { return { ...bedrockOpts, additionalModelRequestFields: { ...((bedrockOpts.additionalModelRequestFields ?? {}) as Record< string, unknown >), thinking: { ...thinkingField, budget_tokens: TOOL_TURN_THINKING_BUDGET, }, }, } as t.BedrockAnthropicClientOptions; } } if (provider === Providers.VERTEXAI || provider === Providers.GOOGLE) { const googleOpts = clientOptions as t.GoogleClientOptions; if ( googleOpts.thinkingConfig?.thinkingBudget != null && googleOpts.thinkingConfig.thinkingBudget > TOOL_TURN_THINKING_BUDGET ) { return { ...googleOpts, thinkingConfig: { ...googleOpts.thinkingConfig, thinkingBudget: TOOL_TURN_THINKING_BUDGET, }, } as t.GoogleClientOptions; } } return clientOptions; } /** * Determines whether summarization should trigger based on SummarizationConfig. * * Supports three trigger strategies: * - contextPercentage (default): Trigger when context utilization >= threshold% * - messageCount: Trigger when pruned message count >= threshold * - tokenThreshold: Trigger when total estimated tokens >= threshold * * When no config is provided, always triggers (preserves backward compatibility). * * @param prunedMessageCount - Number of messages that were pruned * @param maxContextTokens - Maximum context token budget * @param indexTokenCountMap - Token count map by message index * @param instructionTokens - Token count for instructions/system message * @param config - Optional SummarizationConfig * @returns Whether summarization should be triggered */ private shouldTriggerSummarization( prunedMessageCount: number, maxContextTokens: number, indexTokenCountMap: Record, instructionTokens: number, config?: t.SummarizationConfig ): boolean { // No pruned messages means nothing to summarize if (prunedMessageCount === 0) { return false; } // No config = backward compatible (always summarize when messages are pruned) if (!config || !config.triggerType) { return true; } const threshold = config.triggerThreshold; switch (config.triggerType) { case 'contextPercentage': { if (maxContextTokens <= 0) return true; const effectiveThreshold = threshold ?? SUMMARIZATION_CONTEXT_THRESHOLD; let totalTokens = instructionTokens; for (const key in indexTokenCountMap) { totalTokens += indexTokenCountMap[key] ?? 0; } const utilization = (totalTokens / maxContextTokens) * 100; return utilization >= effectiveThreshold; } case 'messageCount': { const effectiveThreshold = threshold ?? 5; return prunedMessageCount >= effectiveThreshold; } case 'tokenThreshold': { if (threshold == null) return true; let totalTokens = instructionTokens; for (const key in indexTokenCountMap) { totalTokens += indexTokenCountMap[key] ?? 0; } return totalTokens >= threshold; } default: return true; } } /** * Returns the normalized finish/stop reason from the last LLM invocation. * Used by callers to detect when the response was truncated due to max_tokens. */ getLastFinishReason(): string | undefined { return this.lastFinishReason; } /** * Estimates a human-friendly description of the conversation timeframe based on message count. * Uses rough heuristics to provide context about how much history is available. * * @param messageCount - Number of messages in the remaining context * @returns A friendly description like "the last few minutes", "the past hour", etc. */ getContextTimeframeDescription(messageCount: number): string { // Rough heuristics based on typical conversation patterns: // - Very active chat: ~20-30 messages per hour // - Normal chat: ~10-15 messages per hour // - Slow/thoughtful chat: ~5-8 messages per hour // We use a middle estimate of ~12 messages per hour if (messageCount <= 5) { return 'just the last few exchanges'; } else if (messageCount <= 15) { return 'the last several minutes'; } else if (messageCount <= 30) { return 'roughly the past hour'; } else if (messageCount <= 60) { return 'the past couple of hours'; } else if (messageCount <= 150) { return 'the past few hours'; } else if (messageCount <= 300) { return "roughly a day's worth"; } else if (messageCount <= 700) { return 'the past few days'; } else { return 'about a week or more'; } } /* Run Step Processing */ getRunStep(stepId: string): t.RunStep | undefined { const index = this.contentIndexMap.get(stepId); if (index !== undefined) { return this.contentData[index]; } return undefined; } getAgentContext(metadata: Record | undefined): AgentContext { if (!metadata) { throw new Error('No metadata provided to retrieve agent context'); } const currentNode = metadata.langgraph_node as string; if (!currentNode) { throw new Error( 'No langgraph_node in metadata to retrieve agent context' ); } let agentId: string | undefined; if (currentNode.startsWith(AGENT)) { agentId = currentNode.substring(AGENT.length); } else if (currentNode.startsWith(TOOLS)) { agentId = currentNode.substring(TOOLS.length); } const agentContext = this.agentContexts.get(agentId ?? ''); if (!agentContext) { throw new Error(`No agent context found for agent ID ${agentId}`); } return agentContext; } getStepKey(metadata: Record | undefined): string { if (!metadata) return ''; const keyList = this.getKeyList(metadata); if (this.checkKeyList(keyList)) { /** * Missing metadata fields can occur in child subgraphs invoked via * subgraph.invoke() (e.g., handoff children). These don't have the * full LangGraph metadata (checkpoint_ns, langgraph_node, etc.) * because they run outside the parent graph's stream pipeline. * Return a fallback key instead of throwing. */ const available = keyList.filter((k) => k !== undefined); if (available.length === 0) { return ''; } return joinKeys(available as (string | number)[]); } return joinKeys(keyList); } getStepIdByKey(stepKey: string, index?: number): string { const stepIds = this.stepKeyIds.get(stepKey); if (!stepIds) { throw new Error(`No step IDs found for stepKey ${stepKey}`); } if (index === undefined) { return stepIds[stepIds.length - 1]; } return stepIds[index]; } generateStepId(stepKey: string): [string, number] { const stepIds = this.stepKeyIds.get(stepKey); let newStepId: string | undefined; let stepIndex = 0; if (stepIds) { stepIndex = stepIds.length; newStepId = `step_${nanoid()}`; stepIds.push(newStepId); this.stepKeyIds.set(stepKey, stepIds); } else { newStepId = `step_${nanoid()}`; this.stepKeyIds.set(stepKey, [newStepId]); } return [newStepId, stepIndex]; } getKeyList( metadata: Record | undefined ): (string | number | undefined)[] { if (!metadata) return []; const keyList = [ metadata.run_id as string, metadata.thread_id as string, metadata.langgraph_node as string, metadata.langgraph_step as number, metadata.checkpoint_ns as string, ]; const agentContext = this.getAgentContext(metadata); if ( agentContext.currentTokenType === ContentTypes.THINK || agentContext.currentTokenType === 'think_and_text' ) { keyList.push('reasoning'); } else if (agentContext.tokenTypeSwitch === 'content') { keyList.push(`post-reasoning-${agentContext.reasoningTransitionCount}`); } if (this.invokedToolIds != null && this.invokedToolIds.size > 0) { keyList.push(this.invokedToolIds.size + ''); } return keyList; } checkKeyList(keyList: (string | number | undefined)[]): boolean { return keyList.some((key) => key === undefined); } /* Misc.*/ getRunMessages(): BaseMessage[] | undefined { const result = this.messages.slice(this.startIndex); return result; } getContentParts(): t.MessageContentComplex[] | undefined { return convertMessagesToContent(this.messages.slice(this.startIndex)); } /** * Get all run steps, optionally filtered by agent ID */ getRunSteps(agentId?: string): t.RunStep[] { if (agentId == null || agentId === '') { return [...this.contentData]; } return this.contentData.filter((step) => step.agentId === agentId); } /** * Get run steps grouped by agent ID */ getRunStepsByAgent(): Map { const stepsByAgent = new Map(); for (const step of this.contentData) { if (step.agentId == null || step.agentId === '') continue; const steps = stepsByAgent.get(step.agentId) ?? []; steps.push(step); stepsByAgent.set(step.agentId, steps); } return stepsByAgent; } /** * Get agent IDs that participated in this run */ getActiveAgentIds(): string[] { const agentIds = new Set(); for (const step of this.contentData) { if (step.agentId != null && step.agentId !== '') { agentIds.add(step.agentId); } } return Array.from(agentIds); } /** * Maps contentPart indices to agent IDs for post-run analysis * Returns a map where key is the contentPart index and value is the agentId */ getContentPartAgentMap(): Map { const contentPartAgentMap = new Map(); for (const step of this.contentData) { if ( step.agentId != null && step.agentId !== '' && Number.isFinite(step.index) ) { contentPartAgentMap.set(step.index, step.agentId); } } return contentPartAgentMap; } /** * Get the context breakdown from the primary agent for admin token tracking. * Returns detailed token counts for instructions, tools, etc. */ getContextBreakdown(): { instructions: number; artifacts: number; tools: number; toolCount: number; toolContext: number; total: number; toolsDetail: Array<{ name: string; tokens: number }>; toolContextDetail: Array<{ name: string; tokens: number }>; } | null { const primaryContext = this.agentContexts.get(this.defaultAgentId); if (!primaryContext) { return null; } return primaryContext.getContextBreakdown(); } /** * Get the latest context analytics from the graph. * Returns metrics like utilization %, TOON stats, message breakdown. */ getContextAnalytics(): ContextAnalytics | null { return this.lastContextAnalytics ?? null; } /** Store the latest context analytics for retrieval after run */ private lastContextAnalytics: ContextAnalytics | null = null; /* Graph */ createSystemRunnable({ provider, clientOptions, instructions, additional_instructions, }: { provider?: Providers; clientOptions?: t.ClientOptions; instructions?: string; additional_instructions?: string; }): t.SystemRunnable | undefined { let finalInstructions: string | BaseMessageFields | undefined = instructions; if (additional_instructions != null && additional_instructions !== '') { finalInstructions = finalInstructions != null && finalInstructions ? `${finalInstructions}\n\n${additional_instructions}` : additional_instructions; } if ( finalInstructions != null && finalInstructions && provider === Providers.ANTHROPIC && (clientOptions as t.AnthropicClientOptions).promptCache === true ) { finalInstructions = { content: [ { type: 'text', text: instructions, cache_control: { type: 'ephemeral' }, }, ], }; } if (finalInstructions != null && finalInstructions !== '') { const systemMessage = new SystemMessage(finalInstructions); return RunnableLambda.from((messages: BaseMessage[]) => { return [systemMessage, ...messages]; }).withConfig({ runName: 'prompt' }); } } initializeTools({ currentTools, currentToolMap, agentContext, }: { currentTools?: t.GraphTools; currentToolMap?: t.ToolMap; agentContext?: AgentContext; }): CustomToolNode | ToolNode { const toolDefinitions = agentContext?.toolDefinitions; const eventDrivenMode = toolDefinitions != null && toolDefinitions.length > 0; // Extract HITL tool approval config from compile options (if configured) const toolApprovalConfig = this.compileOptions?.toolApprovalConfig; if (eventDrivenMode) { const schemaTools = createSchemaOnlyTools(toolDefinitions); const toolDefMap = new Map(toolDefinitions.map((def) => [def.name, def])); const graphTools = agentContext?.graphTools as | t.GenericTool[] | undefined; const directToolNames = new Set(); const allTools = [...schemaTools] as t.GenericTool[]; const allToolMap: t.ToolMap = new Map( schemaTools.map((tool) => [tool.name, tool]) ); /** * Include built-in tools (task, content, project, askUser, etc.) as direct tools. * These have full instances and don't need on-demand loading via ON_TOOL_EXECUTE. * Without this, event-driven mode would only have schema stubs + graph tools, * causing "Tool not found" errors when the LLM calls any built-in tool. */ const builtInTools = (currentTools as t.GenericTool[] | undefined) ?? []; for (const tool of builtInTools) { if ('name' in tool) { allTools.push(tool); allToolMap.set(tool.name, tool); directToolNames.add(tool.name); } } if (graphTools && graphTools.length > 0) { for (const tool of graphTools) { if ('name' in tool) { allTools.push(tool); allToolMap.set(tool.name, tool); directToolNames.add(tool.name); } } } return new CustomToolNode({ tools: allTools, toolMap: allToolMap, eventDrivenMode: true, sessions: this.sessions, toolDefinitions: toolDefMap, agentId: agentContext?.agentId, toolCallStepIds: this.toolCallStepIds, toolRegistry: agentContext?.toolRegistry, directToolNames: directToolNames.size > 0 ? directToolNames : undefined, streamingToolCallBuffer: this.streamingToolCallBuffer, errorHandler: (data, metadata) => StandardGraph.handleToolCallErrorStatic(this, data, metadata), toolApprovalConfig, hookRegistry: this.hookRegistry, toolOutputRegistry: this.toolOutputRegistry, }); } const graphTools = agentContext?.graphTools as t.GenericTool[] | undefined; const baseTools = (currentTools as t.GenericTool[] | undefined) ?? []; const allTraditionalTools = graphTools && graphTools.length > 0 ? [...baseTools, ...graphTools] : baseTools; /** * Build tool map from all sources: agent's toolMap, agent's tools array, and graph tools. * Previously, baseTools were missing from the map when no explicit toolMap was provided, * causing ToolNode to not find agent-defined tools (e.g., custom DynamicStructuredTools). */ const traditionalToolMap = graphTools && graphTools.length > 0 ? new Map([ ...(currentToolMap ?? new Map()), ...baseTools .filter((t): t is t.GenericTool & { name: string } => 'name' in t) .map((t) => [t.name, t] as [string, t.GenericTool]), ...graphTools .filter((t): t is t.GenericTool & { name: string } => 'name' in t) .map((t) => [t.name, t] as [string, t.GenericTool]), ]) : currentToolMap; /** Build directToolNames from graph-managed tools (handoff/transfer) so HITL can bypass them */ let directToolNames: Set | undefined; if (graphTools && graphTools.length > 0) { directToolNames = new Set(); for (const tool of graphTools) { if ('name' in tool) { directToolNames.add(tool.name); } } if (directToolNames.size === 0) { directToolNames = undefined; } } return new CustomToolNode({ tools: allTraditionalTools, toolMap: traditionalToolMap, toolCallStepIds: this.toolCallStepIds, streamingToolCallBuffer: this.streamingToolCallBuffer, errorHandler: (data, metadata) => StandardGraph.handleToolCallErrorStatic(this, data, metadata), toolRegistry: agentContext?.toolRegistry, sessions: this.sessions, directToolNames, toolApprovalConfig, hookRegistry: this.hookRegistry, toolOutputRegistry: this.toolOutputRegistry, }); } initializeModel({ provider, tools, clientOptions, }: { provider: Providers; tools?: t.GraphTools; clientOptions?: t.ClientOptions; }): Runnable { const ChatModelClass = getChatModelClass(provider); const model = new ChatModelClass(clientOptions ?? {}); if ( isOpenAILike(provider) && (model instanceof ChatOpenAI || model instanceof AzureChatOpenAI) ) { model.temperature = (clientOptions as t.OpenAIClientOptions) .temperature as number; model.topP = (clientOptions as t.OpenAIClientOptions).topP as number; model.frequencyPenalty = (clientOptions as t.OpenAIClientOptions) .frequencyPenalty as number; model.presencePenalty = (clientOptions as t.OpenAIClientOptions) .presencePenalty as number; model.n = (clientOptions as t.OpenAIClientOptions).n as number; } else if ( provider === Providers.VERTEXAI && model instanceof ChatVertexAI ) { model.temperature = (clientOptions as t.VertexAIClientOptions) .temperature as number; model.topP = (clientOptions as t.VertexAIClientOptions).topP as number; model.topK = (clientOptions as t.VertexAIClientOptions).topK as number; model.topLogprobs = (clientOptions as t.VertexAIClientOptions) .topLogprobs as number; model.frequencyPenalty = (clientOptions as t.VertexAIClientOptions) .frequencyPenalty as number; model.presencePenalty = (clientOptions as t.VertexAIClientOptions) .presencePenalty as number; model.maxOutputTokens = (clientOptions as t.VertexAIClientOptions) .maxOutputTokens as number; } if (!tools || tools.length === 0) { return model as unknown as Runnable; } return (model as t.ModelWithTools).bindTools(tools); } overrideTestModel( responses: string[], sleep?: number, toolCalls?: ToolCall[] ): void { this.overrideModel = createFakeStreamingLLM({ responses, sleep, toolCalls, }); } getNewModel({ provider, clientOptions, }: { provider: Providers; clientOptions?: t.ClientOptions; }): t.ChatModelInstance { const ChatModelClass = getChatModelClass(provider); return new ChatModelClass(clientOptions ?? {}); } getUsageMetadata( finalMessage?: BaseMessage ): Partial | undefined { if ( finalMessage && 'usage_metadata' in finalMessage && finalMessage.usage_metadata != null ) { return finalMessage.usage_metadata as Partial; } } /** Execute model invocation with streaming support */ private async attemptInvoke( { currentModel, finalMessages, provider, tools: _tools, }: { currentModel?: t.ChatModel; finalMessages: BaseMessage[]; provider: Providers; tools?: t.GraphTools; }, config?: RunnableConfig ): Promise> { const model = this.overrideModel ?? currentModel; if (!model) { throw new Error('No model found'); } /** * Lazy tool-output reference annotation (upstream PR #117). Right * before the message array hits the provider, walk it and apply * `[ref: toolturn]` prefixes / `_ref` JSON fields to a * transient copy. The persisted ToolMessages stay clean — only what * the LLM sees gets the annotation, and only when the registry * actually has the referenced output. No-op when the registry is * absent (the common case until a host opts in). */ const annotatedRunId = ( config?.configurable as { run_id?: string } | undefined )?.run_id; finalMessages = annotateMessagesForLLM( finalMessages, this.toolOutputRegistry, annotatedRunId ); if (model.stream) { /** * Process all model output through a local ChatModelStreamHandler in the * graph execution context. Each chunk is awaited before the next one is * consumed, so by the time the stream is exhausted every run step * (MESSAGE_CREATION, TOOL_CALLS) has been created and toolCallStepIds is * fully populated — the graph will not transition to ToolNode until this * is done. * * This replaces the previous pattern where ChatModelStreamHandler lived * in the for-await stream consumer (handler registry). That consumer * runs concurrently with graph execution, so the graph could advance to * ToolNode before the consumer had processed all events. By handling * chunks here, inside the agent node, the race is eliminated. * * The for-await consumer no longer needs a ChatModelStreamHandler; its * on_chat_model_stream events are simply ignored (no handler registered). * The dispatched custom events (ON_RUN_STEP, ON_MESSAGE_DELTA, etc.) * still reach the content aggregator and SSE handlers through the custom * event callback in Run.createCustomEventCallback. */ const metadata = config?.metadata as Record | undefined; const streamHandler = new ChatModelStreamHandler(); const stream = await model.stream(finalMessages, config); let finalChunk: AIMessageChunk | undefined; for await (const chunk of stream) { await streamHandler.handle( GraphEvents.CHAT_MODEL_STREAM, { chunk }, metadata, this ); finalChunk = finalChunk ? concat(finalChunk, chunk) : chunk; } if (manualToolStreamProviders.has(provider)) { finalChunk = modifyDeltaProperties(provider, finalChunk); } if ((finalChunk?.tool_calls?.length ?? 0) > 0) { finalChunk!.tool_calls = finalChunk!.tool_calls?.filter( (tool_call: ToolCall) => !!tool_call.name ); } return { messages: [finalChunk as AIMessageChunk] }; } else { /** Fallback for models without stream support. */ const finalMessage = await model.invoke(finalMessages, config); if ((finalMessage.tool_calls?.length ?? 0) > 0) { finalMessage.tool_calls = finalMessage.tool_calls?.filter( (tool_call: ToolCall) => !!tool_call.name ); } return { messages: [finalMessage] }; } } /** * Execute model invocation with structured output. * Uses native constrained decoding (jsonSchema method) for supported providers, * or falls back to withStructuredOutput with functionCalling/jsonMode. * * Native mode uses provider APIs directly: * - Anthropic: output_config.format via LangChain's method: 'json_schema' * - OpenAI/Azure: response_format.json_schema via LangChain's method: 'jsonSchema' * - Bedrock: falls back to functionCalling (LangChain doesn't support native yet) */ private async attemptStructuredInvoke( { currentModel, finalMessages, schema, structuredOutputConfig, provider, agentContext, }: { currentModel: t.ChatModelInstance; finalMessages: BaseMessage[]; schema: Record; structuredOutputConfig: t.StructuredOutputConfig; provider?: Providers; agentContext?: AgentContext; }, config?: RunnableConfig ): Promise<{ structuredResponse: Record; rawMessage?: AIMessageChunk; }> { const model = this.overrideModel ?? currentModel; // Check if model supports withStructuredOutput // eslint-disable-next-line @typescript-eslint/no-explicit-any if (typeof (model as any).withStructuredOutput !== 'function') { throw new Error( 'The selected model does not support structured output. ' + 'Please use a model that supports JSON schema output (e.g., OpenAI GPT-4, Anthropic Claude, Google Gemini) ' + 'or disable structured output for this agent.' ); } const { name = 'StructuredResponse', includeRaw: _includeRaw = false, handleErrors = true, maxRetries = 2, } = structuredOutputConfig; // Resolve the structured output method using AgentContext's provider-aware logic let method: t.ResolvedStructuredOutputMethod; if (agentContext) { const resolved = agentContext.resolveStructuredOutputMode(); method = resolved.method; if (resolved.warnings.length > 0) { mwarn('[Graph] Structured output mode warnings:', resolved.warnings); } } else { // Legacy fallback: use the old mode-based resolution const mode = structuredOutputConfig.mode ?? 'auto'; if (mode === 'tool') { method = 'functionCalling'; } else if (mode === 'provider') { method = provider === Providers.BEDROCK ? 'functionCalling' : 'jsonMode'; } else { method = undefined; } } // Prepare schema for provider-specific constraints when using native/jsonSchema mode let preparedSchema = schema; if (method === 'jsonSchema' && provider != null) { const { schema: prepared, warnings } = prepareSchemaForProvider( schema, provider, structuredOutputConfig.strict !== false ); preparedSchema = prepared; if (warnings.length > 0) { mwarn('[Graph] Schema preparation warnings:', warnings); } } // Use withStructuredOutput to bind the schema // Always use includeRaw: true internally so we can debug what's returned // eslint-disable-next-line @typescript-eslint/no-explicit-any const structuredModel = (model as any).withStructuredOutput( preparedSchema, { name, method: method === 'native' ? undefined : method, includeRaw: true, // Always true internally for debugging strict: structuredOutputConfig.strict !== false, } ); let lastError: Error | undefined; let attempts = 0; while (attempts <= maxRetries) { try { // Note: We pass the original config here. The stream aggregator will filter out // the synthetic "response" tool call events from withStructuredOutput() const result = await structuredModel.invoke(finalMessages, config); // Check for refusal or truncation in the raw message if (result?.raw != null) { const rawMsg = result.raw as AIMessageChunk; // Check stop reason for refusal or truncation const responseMetadata = rawMsg.response_metadata; const stopReason = responseMetadata.stop_reason ?? // Anthropic responseMetadata.finish_reason ?? // OpenAI responseMetadata.stopReason; // Bedrock if (stopReason === 'max_tokens' || stopReason === 'length') { throw new StructuredOutputTruncatedError(stopReason); } // Check for Anthropic refusal (stop_reason won't be 'refusal' but content may indicate it) // OpenAI uses message.refusal field const refusal = (rawMsg as AIMessageChunk & { refusal?: string }) .refusal; if (refusal != null && refusal !== '') { throw new StructuredOutputRefusalError(refusal); } } // Handle response - we always use includeRaw internally if (result?.raw != null && result?.parsed !== undefined) { return { structuredResponse: result.parsed as Record, rawMessage: result.raw as AIMessageChunk, }; } // Fallback for models that don't support includeRaw return { structuredResponse: result as Record, }; } catch (error) { // Don't retry on refusal or truncation errors — they need user action if ( error instanceof StructuredOutputRefusalError || error instanceof StructuredOutputTruncatedError ) { throw error; } lastError = error as Error; attempts++; // If error handling is disabled, throw immediately if (handleErrors === false) { throw error; } // If we've exhausted retries, throw if (attempts > maxRetries) { throw new Error( `Structured output failed after ${maxRetries + 1} attempts: ${lastError.message}` ); } // Add error message to conversation for retry const errorMessage = typeof handleErrors === 'string' ? handleErrors : `The response did not match the expected schema. Error: ${lastError.message}. Please try again with a valid response.`; mwarn( `[Graph] Structured output attempt ${attempts} failed: ${lastError.message}. Retrying...` ); // Add the error as a human message for context finalMessages = [ ...finalMessages, new HumanMessage({ content: `[VALIDATION ERROR]\n${errorMessage}`, }), ]; } } throw lastError ?? new Error('Structured output failed'); } cleanupSignalListener(currentModel?: t.ChatModel): void { if (!this.signal) { return; } const model = this.overrideModel ?? currentModel; if (!model) { return; } const client = (model as ChatOpenAI | undefined)?.exposedClient; if (!client?.abortHandler) { return; } this.signal.removeEventListener('abort', client.abortHandler); client.abortHandler = undefined; } /** * Perform structured output invocation: creates a fresh model without tools bound, * removes thinking configuration, invokes with the schema, emits the event, * and returns a clean AIMessageChunk without tool_calls. * * Used by both the immediate path (no tools) and the deferred path (after tool use). */ private async performStructuredOutput({ agentContext, finalMessages, config, }: { agentContext: AgentContext; finalMessages: BaseMessage[]; config: RunnableConfig; }): Promise> { const schema = agentContext.getStructuredOutputSchema(); if (!schema) { throw new Error('Structured output schema is not configured'); } // Get a fresh model WITHOUT tools bound // bindTools() returns RunnableBinding which lacks withStructuredOutput // Also disable thinking mode - Anthropic/Bedrock doesn't allow tool_choice with thinking enabled const structuredClientOptions = { ...agentContext.clientOptions, } as t.ClientOptions; // Determine if streaming is possible for this structured output mode // Native/jsonSchema modes can stream; tool/functionCalling modes cannot (synthetic tool calls break UX) const resolved = agentContext.resolveStructuredOutputMode(); const canStream = resolved.method === 'jsonSchema' || resolved.method === 'jsonMode'; if (!canStream) { // Disable streaming for function calling mode (synthetic tool calls break streaming UX) // eslint-disable-next-line @typescript-eslint/no-explicit-any (structuredClientOptions as any).streaming = false; } // For native/jsonSchema mode, Anthropic's constrained decoding works with thinking enabled // (grammar only applies to final output, not thinking blocks). For function calling mode, // thinking must be disabled because forced tool_choice is incompatible with thinking. const needsThinkingDisabled = resolved.method !== 'jsonSchema'; if (needsThinkingDisabled) { // Remove thinking configuration for Bedrock if (agentContext.provider === Providers.BEDROCK) { const bedrockOpts = structuredClientOptions as t.BedrockAnthropicClientOptions; if (bedrockOpts.additionalModelRequestFields != null) { const additionalFields = Object.assign( {}, bedrockOpts.additionalModelRequestFields ) as Record; delete additionalFields.thinking; delete additionalFields.budgetTokens; bedrockOpts.additionalModelRequestFields = additionalFields as t.BedrockAnthropicInput['additionalModelRequestFields']; } } // Remove thinking configuration for Anthropic direct API if (agentContext.provider === Providers.ANTHROPIC) { const anthropicOpts = structuredClientOptions as t.AnthropicClientOptions; if (anthropicOpts.thinking) { delete anthropicOpts.thinking; } } } const structuredModel = this.getNewModel({ provider: agentContext.provider, clientOptions: structuredClientOptions, }); const { structuredResponse, rawMessage } = await this.attemptStructuredInvoke( { currentModel: structuredModel, finalMessages, schema, structuredOutputConfig: agentContext.structuredOutput!, provider: agentContext.provider, agentContext, }, config ); // Emit structured output event await safeDispatchCustomEvent( GraphEvents.ON_STRUCTURED_OUTPUT, { structuredResponse, schema, raw: rawMessage, }, config ); // Create a clean message WITHOUT tool_calls for structured output. // The rawMessage contains a tool_call for the structured output schema (e.g., "response"), // which would cause the graph router to send it to the tool node. // We return a clean AI message that ends the graph. let cleanMessage: AIMessageChunk | undefined; if (rawMessage) { cleanMessage = new AIMessageChunk({ content: JSON.stringify(structuredResponse, null, 2), id: rawMessage.id, response_metadata: rawMessage.response_metadata, usage_metadata: rawMessage.usage_metadata, }); } return { messages: cleanMessage ? [cleanMessage] : [], structuredResponse, }; } createCallModel(agentId = 'default') { return async ( state: t.BaseGraphState, config?: RunnableConfig ): Promise> => { /** * Get agent context - it must exist by this point */ const agentContext = this.agentContexts.get(agentId); if (!agentContext) { throw new Error(`Agent context not found for agentId: ${agentId}`); } if (!config) { throw new Error('No config provided'); } let { messages } = state; // CACHE OPTIMIZATION: Inject dynamicContext as a HumanMessage at the start of conversation // This keeps the system message static (cacheable) while providing dynamic context // (timestamps, user info, tool context) as conversation content instead. // Only inject on the first turn when messages don't already have the context marker. if ( agentContext.dynamicContext != null && agentContext.dynamicContext !== '' && messages.length > 0 && !messages.some( (m) => m instanceof HumanMessage && typeof m.content === 'string' && m.content.startsWith('[SESSION_CONTEXT]') ) ) { const dynamicContextMessage = new HumanMessage({ content: `[SESSION_CONTEXT]\n${agentContext.dynamicContext}`, }); const ackMessage = new AIMessageChunk({ content: 'Understood. I have noted the session context including the current date/time (CST) and will apply it appropriately.', }); messages = [dynamicContextMessage, ackMessage, ...messages]; } // Tool discovery caching: only scan new messages since last iteration // instead of re-parsing the full history via extractToolDiscoveries() const cachedDiscoveries = this._toolDiscoveryCache.getNewDiscoveries(messages); if (cachedDiscoveries.length > 0) { agentContext.markToolsAsDiscovered(cachedDiscoveries); mlog( `[Graph:ToolDiscovery] Cached ${cachedDiscoveries.length} new tools (total: ${this._toolDiscoveryCache.size})` ); } const toolsForBinding = agentContext.getToolsForBinding(); // PERF: Detect subsequent ReAct iterations (tool results present in messages) // and reduce thinking budget to minimize per-iteration latency. // First iteration gets the user's configured budget; follow-up turns // use TOOL_TURN_THINKING_BUDGET (1024) since they only need to route // "call next tool" or "produce final response". const isSubsequentIteration = messages.some( (m) => m._getType() === 'tool' ); const effectiveClientOptions = isSubsequentIteration && agentContext.clientOptions ? this.getAdaptiveClientOptions( agentContext.clientOptions, agentContext.provider ) : agentContext.clientOptions; let model = this.overrideModel ?? this.initializeModel({ tools: toolsForBinding, provider: agentContext.provider, clientOptions: effectiveClientOptions, }); if (agentContext.systemRunnable) { model = agentContext.systemRunnable.pipe(model as Runnable); } if (agentContext.tokenCalculationPromise) { await agentContext.tokenCalculationPromise; } if (!config.signal) { config.signal = this.signal; } // First-writer-wins: `this.config` is used ONLY as a "has a run started" // existence flag by the dispatch* methods (they never read its value — // they read the current RunnableConfig from LangChain AsyncLocalStorage). // Unconditionally reassigning here races across concurrent child // subgraph.invoke() calls under parallel multi-agent handoffs; the last // writer wins, and any dispatch firing between writes would historically // have been tagged with the wrong child's metadata. Keeping the first // write pinned makes this a true flag, eliminating the race. this.config ??= config; let messagesToUse = messages; // ==================================================================== // PRE-PRUNING DELEGATION CHECK // ==================================================================== // Context management is now fully mechanical: // - Pruning always runs when needed (no delegation-based skip) // - Auto-continuation in client.js handles max_tokens finish reason // - LLM never sees raw token numbers (prevents voluntary bail-out) // ==================================================================== if ( !agentContext.pruneMessages && agentContext.tokenCounter && agentContext.maxContextTokens != null && agentContext.indexTokenCountMap[0] != null ) { const isAnthropicWithThinking = (agentContext.provider === Providers.ANTHROPIC && (agentContext.clientOptions as t.AnthropicClientOptions).thinking != null) || (agentContext.provider === Providers.BEDROCK && (agentContext.clientOptions as t.BedrockAnthropicInput) .additionalModelRequestFields?.['thinking'] != null) || (agentContext.provider === Providers.OPENAI && ( (agentContext.clientOptions as t.OpenAIClientOptions).modelKwargs ?.thinking as t.AnthropicClientOptions['thinking'] )?.type === 'enabled'); // Apply EMA calibration to max token budget — smooths pruning across iterations const calibratedMaxTokens = applyCalibration( agentContext.maxContextTokens, this._pruneCalibration ); agentContext.pruneMessages = createPruneMessages({ startIndex: this.startIndex, provider: agentContext.provider, tokenCounter: agentContext.tokenCounter, maxTokens: calibratedMaxTokens, thinkingEnabled: isAnthropicWithThinking, indexTokenCountMap: agentContext.indexTokenCountMap, }); } // Update EMA calibration with actual token usage from API response if ( agentContext.currentUsage?.input_tokens && agentContext.maxContextTokens ) { const estimatedTokens = Object.values( agentContext.indexTokenCountMap ).reduce((sum, v) => (sum ?? 0) + (v ?? 0), 0) as number; if (estimatedTokens > 0) { this._pruneCalibration = updatePruneCalibration( this._pruneCalibration, agentContext.currentUsage.input_tokens, estimatedTokens ); } } // ── Proactive summarization at context pressure ─────────────────── // Inspired by VS Code Copilot Chat's 3-tier strategy: // 80% → fire proactive background summary (BEFORE pruning needed) // 90% → pruning kicks in (summary already cached from 80% trigger) // 100% → graceful: use existing summary + recent messages, NEVER block // // This ensures the summary is READY by the time pruning actually occurs, // so the user never waits and never sees a context cliff. if ( agentContext.maxContextTokens != null && agentContext.maxContextTokens > 0 && agentContext.summarizeCallback && !this._summaryInFlight && !this._cachedRunSummary ) { const utilization = getContextUtilization( agentContext.indexTokenCountMap, agentContext.instructionTokens, agentContext.maxContextTokens ); const threshold = agentContext.summarizationConfig?.triggerThreshold ?? PROACTIVE_SUMMARY_THRESHOLD * 100; if (utilization >= threshold) { // Identify older messages to summarize proactively. // Keep the last N messages (recent turns) intact — only summarize older history. // This is incremental: the callback checks for existing summary and updates it. const recentTurnCount = Math.max( 4, Math.floor(messages.length * 0.3) ); const oldMessages = messages.slice( messages[0]?.getType() === 'system' ? 1 : 0, Math.max(1, messages.length - recentTurnCount) ); if (oldMessages.length > 0) { this._summaryInFlight = true; mlog( `[Graph:ProactiveSummary] Context at ${utilization.toFixed(1)}% (threshold ${threshold}%) — summarizing ${oldMessages.length} older msgs in background` ); /** * Fire PreCompact (#103) before the LLM call. Observational — * hosts may use the input to schedule auxiliary work, but a * deny/ask decision does NOT block compaction here because * Ranger's summarizer runs in the background and a refusal * has nowhere to go: pruning still needs to happen at 100% * regardless. Errors are swallowed so a hook bug cannot * masquerade as a context-overflow failure. */ const sessionId = this.runId ?? ''; const preCompactPromise = this.hookRegistry?.hasHookFor('PreCompact', sessionId) === true ? executeHooks({ registry: this.hookRegistry, input: { hook_event_name: 'PreCompact', runId: sessionId, agentId: agentContext.agentId, messagesBeforeCount: oldMessages.length, trigger: agentContext.summarizationConfig?.trigger?.type ?? agentContext.summarizationConfig?.triggerType ?? 'default', }, sessionId, }).catch(() => undefined) : Promise.resolve(undefined); preCompactPromise .then(() => agentContext.summarizeCallback!(oldMessages)) .then((updated) => { if (updated != null && updated !== '') { this._cachedRunSummary = updated; mlog( `[Graph:ProactiveSummary] Background summary ready (len=${updated.length})` ); if ( this.hookRegistry?.hasHookFor('PostCompact', sessionId) === true ) { void executeHooks({ registry: this.hookRegistry, input: { hook_event_name: 'PostCompact', runId: sessionId, agentId: agentContext.agentId, summary: updated, messagesAfterCount: 0, }, sessionId, }).catch(() => { /* PostCompact is observational — swallow errors */ }); } } }) .catch((err) => { console.error( '[Graph:ProactiveSummary] Background summary failed (non-fatal):', err ); }) .finally(() => { this._summaryInFlight = false; }); } } } if (agentContext.pruneMessages) { // ── Context Compaction (Copilot-style: never delete messages) ───── // // DESIGN: Original messages are NEVER removed from the array. // Instead, we build a "windowed view" for the LLM: // [system prompt] + [summary of older turns] + [recent turns that fit] // // This ensures: // - No context is ever lost (summary covers older turns) // - We can always re-summarize from originals if summary is stale // - Conversation chaining works naturally across turns // // Flow: // 1. Resolve best available summary (cached > persisted > seed) // 2. Calculate token budget available for recent messages // 3. Walk newest→oldest, build view of messages that fit // 4. Assemble: [system] + [summary] + [recent window] // 5. Fire background summary update for messages outside the window const sumConfig = agentContext.summarizationConfig; const tokenCounter = agentContext.tokenCounter; const maxTokens = agentContext.maxContextTokens ?? 0; // Step 1: Resolve best available summary let summary: string | undefined; if (this._cachedRunSummary != null) { summary = this._cachedRunSummary; } else if ( agentContext.persistedSummary != null && agentContext.persistedSummary !== '' ) { summary = agentContext.persistedSummary; this._cachedRunSummary = summary; } else if ( sumConfig?.initialSummary != null && sumConfig.initialSummary !== '' ) { summary = sumConfig.initialSummary; this._cachedRunSummary = summary; } // Step 2: Calculate token budget // Apply EMA calibration for accuracy across iterations const calibratedMax = applyCalibration( maxTokens, this._pruneCalibration ); const systemMsg = messages[0]?.getType() === 'system' ? messages[0] : null; const systemTokens = systemMsg != null ? (agentContext.indexTokenCountMap[0] ?? 0) : 0; const summaryMsg = summary != null && summary !== '' ? new SystemMessage(`[Conversation Summary]\n${summary}`) : null; const summaryTokens = summaryMsg != null && tokenCounter != null ? tokenCounter(summaryMsg) : 0; // Budget for recent messages = total - system - summary - 3 (assistant priming) const recentBudget = calibratedMax - systemTokens - summaryTokens - 3; // Step 3: Determine window of recent messages to include. // // Two modes: // A) No summary available → fill the budget (all messages that fit) // B) Summary available → keep last 2 conversation rounds (H+A pairs) // + any trailing tool messages. The summary covers everything else. // This avoids wasting tokens on raw messages the summary already covers. // // A "round" = one human message + one AI response (+ any tool messages between). const contentStart = systemMsg != null ? 1 : 0; let usedTokens = 0; let windowStart = messages.length; // index where the recent window begins let fileManifestTokens = 0; // populated in Step 4 if file manifest is injected if (summary == null || summary === '') { // Mode A: No summary — include as many recent messages as fit in budget for (let i = messages.length - 1; i >= contentStart; i--) { const msgTokens = agentContext.indexTokenCountMap[i] ?? 0; if (usedTokens + msgTokens > recentBudget) { break; } usedTokens += msgTokens; windowStart = i; } } else { // Mode B: Summary exists — keep last 2 rounds (4 core messages: H+A+H+A) // Walk backward counting human messages as round boundaries. const MAX_RECENT_ROUNDS = COMPACTION_RECENT_ROUNDS; let roundsSeen = 0; for (let i = messages.length - 1; i >= contentStart; i--) { const msgType = messages[i]?.getType(); const msgTokens = agentContext.indexTokenCountMap[i] ?? 0; // Budget guard — even in round-limited mode, don't exceed budget if (usedTokens + msgTokens > recentBudget) { break; } usedTokens += msgTokens; windowStart = i; // Count a human message as a round boundary if (msgType === 'human') { roundsSeen++; if (roundsSeen >= MAX_RECENT_ROUNDS) { break; } } } } // Ensure we don't split tool-call / tool-result pairs. // If windowStart lands on a ToolMessage, walk back to include its AI message. while ( windowStart > contentStart && messages[windowStart]?.getType() === 'tool' ) { windowStart--; usedTokens += agentContext.indexTokenCountMap[windowStart] ?? 0; } const recentMessages = messages.slice(windowStart); const compactedMessages = messages.slice(contentStart, windowStart); const hasSummary = summaryMsg != null; // Step 4: Assemble the windowed view // [system] + [summary] + [file manifest] + [recent window] // // File manifest is injected ONLY when compaction is active (messages behind summary). // It provides the LLM with awareness of all conversation files so it can // retrieve content on demand via file_search or content_tool read. const viewParts: BaseMessage[] = []; if (systemMsg != null) { viewParts.push(systemMsg); } if (summaryMsg != null) { viewParts.push(summaryMsg); } // Inject file manifest when files exist and messages are being compacted const fileManifest = agentContext.fileManifest; if ( fileManifest && fileManifest.length > 0 && compactedMessages.length > 0 ) { const manifestBlock = buildFileManifestBlock(fileManifest); if (manifestBlock) { const manifestMsg = new SystemMessage(manifestBlock); viewParts.push(manifestMsg); // Account for manifest tokens in the view token map const manifestTokens = tokenCounter != null ? tokenCounter(manifestMsg) : 0; // Will be inserted at the correct index when rebuilding viewTokenMap below fileManifestTokens = manifestTokens; } } viewParts.push(...recentMessages); messagesToUse = viewParts; // Rebuild indexTokenCountMap for the windowed view so downstream // analytics and summarization triggers see accurate token counts. const viewTokenMap: Record = {}; let viewIdx = 0; if (systemMsg != null) { viewTokenMap[viewIdx] = systemTokens; viewIdx++; } if (summaryMsg != null) { viewTokenMap[viewIdx] = summaryTokens; viewIdx++; } if (fileManifestTokens > 0) { viewTokenMap[viewIdx] = fileManifestTokens; viewIdx++; } for (let i = windowStart; i < messages.length; i++) { viewTokenMap[viewIdx] = agentContext.indexTokenCountMap[i]; viewIdx++; } agentContext.indexTokenCountMap = viewTokenMap; // Step 5: Fire background summary update (non-blocking) // Summarize messages outside the window so next iteration has a fresh summary. // Only trigger if there are compacted messages worth summarizing. if (compactedMessages.length > 0 && agentContext.summarizeCallback) { const shouldSummarize = this.shouldTriggerSummarization( compactedMessages.length, maxTokens, agentContext.indexTokenCountMap, agentContext.instructionTokens, sumConfig ); if (shouldSummarize) { if (this._summaryInFlight) { this._pendingMessagesToRefine.push(...compactedMessages); mlog( `[Graph:Compaction] Summary in-flight, queued ${compactedMessages.length} msgs (pending=${this._pendingMessagesToRefine.length})` ); } else { this._summaryInFlight = true; const allMessages = this._pendingMessagesToRefine.length > 0 ? [...this._pendingMessagesToRefine, ...compactedMessages] : compactedMessages; this._pendingMessagesToRefine = []; agentContext .summarizeCallback(allMessages) .then((updated) => { if (updated != null && updated !== '') { this._cachedRunSummary = updated; } }) .catch((err) => { console.error( '[Graph:Compaction] Background summary update failed (non-fatal):', err ); }) .finally(() => { this._summaryInFlight = false; }); } } } // Post-compaction context note for task-tool-enabled agents if (compactedMessages.length > 0 && hasTaskTool(agentContext.tools)) { const postPruneNote = buildPostPruneNote( compactedMessages.length, hasSummary ); if (postPruneNote) { messagesToUse = [ ...messagesToUse, new SystemMessage(postPruneNote), ]; } } } // Deduplicate system messages — ALWAYS runs, not just during compaction. // Duplicate system messages accumulate from repeated tool iterations, // summary injections, and context notes across turns. const { messages: dedupedMessages, removedCount } = deduplicateSystemMessages(messagesToUse); if (removedCount > 0) { messagesToUse = dedupedMessages; mlog( `[Graph:Dedup] Removed ${removedCount} duplicate system message(s)` ); } let finalMessages = messagesToUse; if (agentContext.useLegacyContent) { finalMessages = formatContentStrings(finalMessages); } const lastMessageX = finalMessages.length >= 2 ? finalMessages[finalMessages.length - 2] : null; const lastMessageY = finalMessages.length >= 1 ? finalMessages[finalMessages.length - 1] : null; if ( agentContext.provider === Providers.BEDROCK && lastMessageX instanceof AIMessageChunk && lastMessageY?.getType() === MessageTypes.TOOL && typeof lastMessageX.content === 'string' ) { finalMessages[finalMessages.length - 2].content = ''; } // Use getType() instead of instanceof to avoid module mismatch issues const isLatestToolMessage = lastMessageY?.getType() === MessageTypes.TOOL; if ( isLatestToolMessage && agentContext.provider === Providers.ANTHROPIC ) { formatAnthropicArtifactContent(finalMessages); } else if ( isLatestToolMessage && ((isOpenAILike(agentContext.provider) && agentContext.provider !== Providers.DEEPSEEK) || isGoogleLike(agentContext.provider)) ) { formatArtifactPayload(finalMessages); } /** * Handle edge case: when switching from a non-thinking agent to a thinking-enabled agent, * convert AI messages with tool calls to HumanMessages to avoid thinking block requirements. * This is required by Anthropic/Bedrock when thinking is enabled. * * IMPORTANT: This MUST happen BEFORE cache control is applied. * If we add cachePoint to an AI message first, then convert that AI message to a HumanMessage, * the cachePoint is lost. By converting first, we ensure cache control is applied to the * final message structure that will be sent to the API. */ const isAnthropicWithThinking = (agentContext.provider === Providers.ANTHROPIC && (agentContext.clientOptions as t.AnthropicClientOptions).thinking != null) || (agentContext.provider === Providers.BEDROCK && (agentContext.clientOptions as t.BedrockAnthropicInput) .additionalModelRequestFields?.['thinking'] != null); if (isAnthropicWithThinking) { finalMessages = ensureThinkingBlockInMessages( finalMessages, agentContext.provider ); } // Apply cache control AFTER thinking block handling to ensure cachePoints aren't lost // when AI messages are converted to HumanMessages if (agentContext.provider === Providers.ANTHROPIC) { const anthropicOptions = agentContext.clientOptions as | t.AnthropicClientOptions | undefined; if (anthropicOptions?.promptCache === true) { finalMessages = addCacheControl(finalMessages); } } else if (agentContext.provider === Providers.BEDROCK) { const bedrockOptions = agentContext.clientOptions as | t.BedrockAnthropicClientOptions | undefined; // Both Claude and Nova models support cachePoint in system and messages // (Llama, Titan, and other models do NOT support cachePoint) const modelId = bedrockOptions?.model?.toLowerCase() ?? ''; const supportsCaching = modelId.includes('claude') || modelId.includes('anthropic') || modelId.includes('nova'); if (bedrockOptions?.promptCache === true && supportsCaching) { finalMessages = addBedrockCacheControl(finalMessages); } } if ( agentContext.lastStreamCall != null && agentContext.streamBuffer != null ) { const timeSinceLastCall = Date.now() - agentContext.lastStreamCall; if (timeSinceLastCall < agentContext.streamBuffer) { const timeToWait = Math.ceil((agentContext.streamBuffer - timeSinceLastCall) / 1000) * 1000; await sleep(timeToWait); } } agentContext.lastStreamCall = Date.now(); let result: Partial | undefined; const fallbacks = (agentContext.clientOptions as t.LLMConfig | undefined)?.fallbacks ?? []; if (finalMessages.length === 0) { throw new Error( JSON.stringify({ type: 'empty_messages', info: 'Message pruning removed all messages as none fit in the context window. Please increase the context window size or make your message shorter.', }) ); } // Get model info for analytics const bedrockOpts = agentContext.clientOptions as | t.BedrockAnthropicClientOptions | undefined; const modelId = bedrockOpts?.model ?? (agentContext.clientOptions as t.AnthropicClientOptions | undefined) ?.modelName; const thinkingConfig = bedrockOpts?.additionalModelRequestFields?.['thinking'] ?? (agentContext.clientOptions as t.AnthropicClientOptions | undefined) ?.thinking; // Build and emit context analytics for traces const contextAnalytics = buildContextAnalytics(finalMessages, { tokenCounter: agentContext.tokenCounter, maxContextTokens: agentContext.maxContextTokens, instructionTokens: agentContext.instructionTokens, indexTokenCountMap: agentContext.indexTokenCountMap, }); // Store for retrieval via getContextAnalytics() after run completes this.lastContextAnalytics = contextAnalytics; await safeDispatchCustomEvent( GraphEvents.ON_CONTEXT_ANALYTICS, { provider: agentContext.provider, model: modelId, thinkingEnabled: thinkingConfig != null, cacheEnabled: bedrockOpts?.promptCache === true, analytics: contextAnalytics, }, config ); // ==================================================================== // MULTI-DOCUMENT DELEGATION (task-driven, not budget-driven) // // Token-based pressure hints have been removed — the LLM never sees // raw token numbers. Context overflow is handled mechanically by // pruning (Graph) + auto-continuation (client.js max_tokens detection). // See: docs/context-overflow-architecture.md // ==================================================================== if (hasTaskTool(agentContext.tools)) { const { count: documentCount, names: documentNames } = detectDocuments(finalMessages); // Multi-document delegation: first iteration only (before AI has responded) const hasAiResponse = finalMessages.some( (m) => m._getType() === 'ai' || m._getType() === 'tool' ); if (shouldInjectMultiDocHint(documentCount, hasAiResponse)) { const pressureMsg = new HumanMessage({ content: buildMultiDocHintContent(documentCount, documentNames), }); finalMessages = [...finalMessages, pressureMsg]; console.info( `[Graph] Multi-document delegation hint injected for ${documentCount} documents: ` + `${documentNames.join(', ')}` ); } } // Structured output mode: when the agent has NO tools, produce structured JSON immediately. // When the agent HAS tools, we defer structured output until after tool use completes // (see the deferred structured output block after attemptInvoke below). const hasTools = (toolsForBinding?.length ?? 0) > 0; if ( agentContext.isStructuredOutputMode && agentContext.structuredOutput && !hasTools ) { try { const structuredResult = await this.performStructuredOutput({ agentContext, finalMessages, config, }); agentContext.currentUsage = this.getUsageMetadata( structuredResult.messages?.[0] ); this.cleanupSignalListener(); return structuredResult; } catch (structuredError) { console.error('[Graph] Structured output failed:', structuredError); throw structuredError; } } try { result = await this.attemptInvoke( { currentModel: model, finalMessages, provider: agentContext.provider, tools: agentContext.tools, }, config ); } catch (primaryError) { const errorMessage = (primaryError as Error).message; const isInputTooLongError = isLikelyContextOverflowError(errorMessage); // Log when we detect the error if (isInputTooLongError) { mwarn( '[Graph] Detected input too long error:', errorMessage.substring(0, 200) ); mwarn('[Graph] Checking emergency pruning conditions:', { hasPruneMessages: !!agentContext.pruneMessages, hasTokenCounter: !!agentContext.tokenCounter, maxContextTokens: agentContext.maxContextTokens, indexTokenMapKeys: Object.keys(agentContext.indexTokenCountMap) .length, }); } // If input too long and we have pruning capability OR tokenCounter, retry with progressively more aggressive pruning // Note: We can create emergency pruneMessages dynamically if we have tokenCounter and maxContextTokens const canPrune = agentContext.tokenCounter != null && agentContext.maxContextTokens != null && agentContext.maxContextTokens > 0; if (isInputTooLongError && canPrune) { // Progressive reduction: 50% -> 25% -> 10% of original context const reductionLevels = [0.5, 0.25, 0.1]; for (const reductionFactor of reductionLevels) { if (result) break; // Exit if we got a result const reducedMaxTokens = Math.floor( agentContext.maxContextTokens! * reductionFactor ); mwarn( `[Graph] Input too long. Retrying with ${reductionFactor * 100}% context (${reducedMaxTokens} tokens)...` ); // Build fresh indexTokenCountMap if missing/incomplete // This is needed when messages were dynamically added without updating the token map let tokenMapForPruning = agentContext.indexTokenCountMap; if (Object.keys(tokenMapForPruning).length < messages.length) { mwarn( '[Graph] Building fresh token count map for emergency pruning...' ); tokenMapForPruning = {}; for (let i = 0; i < messages.length; i++) { tokenMapForPruning[i] = agentContext.tokenCounter!(messages[i]); } } const emergencyPrune = createPruneMessages({ startIndex: this.startIndex, provider: agentContext.provider, tokenCounter: agentContext.tokenCounter!, maxTokens: reducedMaxTokens, thinkingEnabled: false, // Disable thinking for emergency prune indexTokenCountMap: tokenMapForPruning, }); const { context: reducedMessages } = emergencyPrune({ messages, usageMetadata: agentContext.currentUsage, }); // Skip if we can't fit any messages if (reducedMessages.length === 0) { mwarn( `[Graph] Cannot fit any messages at ${reductionFactor * 100}% reduction, trying next level...` ); continue; } // Calculate how many messages were pruned and estimate context timeframe const prunedCount = finalMessages.length - reducedMessages.length; const remainingCount = reducedMessages.length; const estimatedContextDescription = this.getContextTimeframeDescription(remainingCount); // Inject a personalized context message to inform the agent about pruning const pruneNoticeMessage = new HumanMessage({ content: `[CONTEXT NOTICE] Our conversation has grown quite long, so I've focused on ${estimatedContextDescription} of our chat (${remainingCount} recent messages). ${prunedCount} earlier messages are no longer in my immediate memory. If I seem to be missing something we discussed earlier, just give me a quick reminder and I'll pick right back up! I'm still fully engaged and ready to help with whatever you need.`, }); // Insert the notice after the system message (if any) but before conversation const hasSystemMessage = reducedMessages[0]?.getType() === 'system'; const insertIndex = hasSystemMessage ? 1 : 0; // Create new array with the pruning notice const messagesWithNotice = [ ...reducedMessages.slice(0, insertIndex), pruneNoticeMessage, ...reducedMessages.slice(insertIndex), ]; let retryMessages = agentContext.useLegacyContent ? formatContentStrings(messagesWithNotice) : messagesWithNotice; // Apply thinking block handling first (before cache control) // This ensures AI+Tool sequences are converted to HumanMessages // before we add cache points that could be lost in the conversion if (isAnthropicWithThinking) { retryMessages = ensureThinkingBlockInMessages( retryMessages, agentContext.provider ); } // Apply Bedrock cache control if needed (after thinking block handling) if (agentContext.provider === Providers.BEDROCK) { const bedrockOptions = agentContext.clientOptions as | t.BedrockAnthropicClientOptions | undefined; const modelId = bedrockOptions?.model?.toLowerCase() ?? ''; const supportsCaching = modelId.includes('claude') || modelId.includes('anthropic') || modelId.includes('nova'); if (bedrockOptions?.promptCache === true && supportsCaching) { retryMessages = addBedrockCacheControl(retryMessages); } } try { result = await this.attemptInvoke( { currentModel: model, finalMessages: retryMessages, provider: agentContext.provider, tools: agentContext.tools, }, config ); // Success with reduced context console.info( `[Graph] ✅ Retry successful at ${reductionFactor * 100}% with ${reducedMessages.length} messages (reduced from ${finalMessages.length})` ); } catch (retryError) { const retryErrorMsg = (retryError as Error).message; const stillTooLong = isLikelyContextOverflowError(retryErrorMsg); if (stillTooLong && reductionFactor > 0.1) { mwarn( `[Graph] Still too long at ${reductionFactor * 100}%, trying more aggressive pruning...` ); } else { console.error( `[Graph] Retry at ${reductionFactor * 100}% failed:`, (retryError as Error).message ); } } } } // If we got a result from retry, skip fallbacks if (result) { // result already set from retry } else { let lastError: unknown = primaryError; for (const fb of fallbacks) { try { let model = this.getNewModel({ provider: fb.provider, clientOptions: fb.clientOptions, }); const bindableTools = agentContext.tools; model = ( !bindableTools || bindableTools.length === 0 ? model : model.bindTools(bindableTools) ) as t.ChatModelInstance; result = await this.attemptInvoke( { currentModel: model, finalMessages, provider: fb.provider, tools: agentContext.tools, }, config ); lastError = undefined; break; } catch (e) { lastError = e; continue; } } if (lastError !== undefined) { throw lastError; } } } if (!result) { throw new Error('No result after model invocation'); } /** * Fallback: populate toolCallStepIds in the graph execution context. * * When model.stream() is available (the common case), attemptInvoke * processes all chunks through a local ChatModelStreamHandler which * creates run steps and populates toolCallStepIds before returning. * The code below is a fallback for the rare case where model.stream * is unavailable and model.invoke() was used instead. * * Text content is dispatched FIRST so that MESSAGE_CREATION is the * current step when handleToolCalls runs. handleToolCalls then creates * TOOL_CALLS on top of it. The dedup in getMessageId and * toolCallStepIds.has makes this safe when attemptInvoke already * handled everything — both paths become no-ops. */ const responseMessage = result.messages?.[0]; // Tool-call name normalization — catches LLM output that names tools // with wrong delimiters (outlook/operations), prefixes // (functions.outlook_operations), case drift, counter suffixes, or // empty names recoverable from the tool_call id. Mutates in place so // the downstream ToolNode dispatch sees the corrected names. if (responseMessage && agentContext.toolMap) { const allowedNames = new Set(Object.keys(agentContext.toolMap)); if (allowedNames.size > 0) { const rewrote = normalizeMessageToolCalls( responseMessage, allowedNames ); if (rewrote) { mlog( `[Graph] normalized tool_call names on agent "${agentId}" response` ); } } } const toolCalls = (responseMessage as AIMessageChunk | undefined) ?.tool_calls; const hasToolCalls = Array.isArray(toolCalls) && toolCalls.length > 0; if (hasToolCalls) { const metadata = config.metadata as Record; const stepKey = this.getStepKey(metadata); const content = responseMessage?.content as MessageContent | undefined; const hasTextContent = content != null && (typeof content === 'string' ? content !== '' : Array.isArray(content) && content.length > 0); /** * Dispatch text content BEFORE creating TOOL_CALLS steps. * getMessageId returns a new ID only on the first call for a step key; * if the for-await consumer already claimed it, this is a no-op. */ if (hasTextContent) { const messageId = getMessageId(stepKey, this) ?? ''; if (messageId) { await this.dispatchRunStep( stepKey, { type: StepTypes.MESSAGE_CREATION, message_creation: { message_id: messageId }, }, metadata ); const stepId = this.getStepIdByKey(stepKey); if (typeof content === 'string') { await this.dispatchMessageDelta(stepId, { content: [{ type: ContentTypes.TEXT, text: content }], }); } else if ( Array.isArray(content) && content.every( (c) => typeof c === 'object' && 'type' in c && typeof c.type === 'string' && c.type.startsWith('text') ) ) { await this.dispatchMessageDelta(stepId, { content: content as t.MessageDelta['content'], }); } } } await handleToolCalls(toolCalls as ToolCall[], metadata, this); } /** * When streaming is disabled, on_chat_model_stream events are never * emitted so ChatModelStreamHandler never fires. Dispatch the text * content as MESSAGE_CREATION + MESSAGE_DELTA here. */ const disableStreaming = (agentContext.clientOptions as t.OpenAIClientOptions | undefined) ?.disableStreaming === true; if ( disableStreaming && !hasToolCalls && responseMessage != null && (responseMessage.content as MessageContent | undefined) != null ) { const metadata = config.metadata as Record; const stepKey = this.getStepKey(metadata); const messageId = getMessageId(stepKey, this) ?? ''; if (messageId) { await this.dispatchRunStep( stepKey, { type: StepTypes.MESSAGE_CREATION, message_creation: { message_id: messageId }, }, metadata ); const stepId = this.getStepIdByKey(stepKey); const content = responseMessage.content; if (typeof content === 'string') { await this.dispatchMessageDelta(stepId, { content: [{ type: ContentTypes.TEXT, text: content }], }); } else if ( Array.isArray(content) && content.every( (c) => typeof c === 'object' && 'type' in c && typeof c.type === 'string' && c.type.startsWith('text') ) ) { await this.dispatchMessageDelta(stepId, { content: content as t.MessageDelta['content'], }); } } } agentContext.currentUsage = this.getUsageMetadata(result.messages?.[0]); // Extract and normalize the LLM's finish/stop reason for auto-continuation support const finalMsg = result.messages?.[0]; if (finalMsg && 'response_metadata' in finalMsg) { const meta = finalMsg.response_metadata as Record; // Bedrock streaming nests stopReason inside messageStop: { stopReason: '...' } const messageStop = meta.messageStop as | Record | undefined; const nextReason = (meta.finish_reason as string | undefined) ?? // OpenAI/Azure (meta.stop_reason as string | undefined) ?? // Anthropic direct API (meta.stopReason as string | undefined) ?? // Bedrock invoke (non-streaming) (messageStop?.stopReason as string | undefined) ?? // Bedrock streaming (meta.finishReason as string | undefined); // VertexAI/Google // Sticky on truncation: a single Graph instance is reused across // every scoped-subgraph inner node invocation (see MultiAgentGraph // buildScopedSubgraph). If an earlier inner node hit max_tokens // but a later inner node finished cleanly, the host's continuation layer // would miss the truncation signal unless we preserve it. Keep the // truncation reason pinned so the outer caller can retry. if (!isTruncationReason(this.lastFinishReason)) { this.lastFinishReason = nextReason; } } this.cleanupSignalListener(); // DEFERRED STRUCTURED OUTPUT: When the agent has tools AND structured output configured, // we let the agent use tools normally via attemptInvoke(). Once the agent's response // has NO tool_calls (it's done with tools), we produce the final structured JSON response. if ( agentContext.isStructuredOutputMode && agentContext.structuredOutput != null ) { const lastMessage = result.messages?.[0]; const resultHasToolCalls = lastMessage != null && 'tool_calls' in lastMessage && ((lastMessage as AIMessageChunk).tool_calls?.length ?? 0) > 0; if (resultHasToolCalls !== true) { try { // Build messages for structured output: include the full conversation // plus the agent's text response from attemptInvoke, so the structured // output model has full context (tool results + agent reasoning). const messagesForStructured = [...finalMessages]; if (lastMessage) { messagesForStructured.push(lastMessage); } const structuredResult = await this.performStructuredOutput({ agentContext, finalMessages: messagesForStructured, config, }); // Accumulate token usage from both API calls const structuredUsage = this.getUsageMetadata( structuredResult.messages?.[0] ); if (structuredUsage && agentContext.currentUsage) { agentContext.currentUsage = { input_tokens: (agentContext.currentUsage.input_tokens ?? 0) + (structuredUsage.input_tokens ?? 0), output_tokens: (agentContext.currentUsage.output_tokens ?? 0) + (structuredUsage.output_tokens ?? 0), total_tokens: (agentContext.currentUsage.total_tokens ?? 0) + (structuredUsage.total_tokens ?? 0), }; } else if (structuredUsage) { agentContext.currentUsage = structuredUsage; } return structuredResult; } catch (structuredError) { // Graceful fallback: the agent completed its work with tools, // but we couldn't format the output as structured JSON. // Return the unstructured text response from attemptInvoke. console.error( '[Graph] Deferred structured output failed after successful tool use:', structuredError ); mwarn( '[Graph] Falling back to unstructured response from tool-use phase' ); return result; } } } return result; }; } createAgentNode(agentId: string): t.CompiledAgentWorfklow { const agentContext = this.agentContexts.get(agentId); if (!agentContext) { throw new Error(`Agent context not found for agentId: ${agentId}`); } /** * Subagent injection (upstream Tier 5): when the agent has SubagentConfig * entries and depth budget remaining, inject a `subagent` DynamicStructuredTool * into graphTools. The tool's executor receives this graph's hookRegistry and * a lazy handler-registry getter (Run wires handlerRegistry AFTER createWorkflow, * so direct capture would be undefined at construction time). */ const effectiveSubagentDepth = agentContext.maxSubagentDepth ?? 1; if ( agentContext.subagentConfigs != null && agentContext.subagentConfigs.length > 0 && effectiveSubagentDepth > 0 ) { const resolvedConfigs = resolveSubagentConfigs( agentContext.subagentConfigs, agentContext ); if (resolvedConfigs.length > 0) { const getParentHandlerRegistry = (): HandlerRegistry | undefined => this.handlerRegistry; const executor = new SubagentExecutor({ configs: new Map(resolvedConfigs.map((c) => [c.type, c])), parentSignal: this.signal, hookRegistry: this.hookRegistry, parentHandlerRegistry: getParentHandlerRegistry, parentRunId: this.runId ?? '', parentAgentId: agentContext.agentId, tokenCounter: agentContext.tokenCounter, maxDepth: effectiveSubagentDepth, createChildGraph: (input): StandardGraph => new StandardGraph(input as t.StandardGraphInput), }); const subagentTool = makeStructuredTool(async (rawInput, config) => { const input = rawInput as { description?: string; subagent_type?: string; }; const description = typeof input.description === 'string' && input.description.trim().length > 0 ? input.description : 'No task description provided'; const subagentType = typeof input.subagent_type === 'string' ? input.subagent_type : ''; const threadId = config.configurable?.thread_id as string | undefined; const toolCall = (config as { toolCall?: { id?: string } }).toolCall; const parentToolCallId = typeof toolCall?.id === 'string' ? toolCall.id : undefined; const result = await executor.execute({ description, subagentType, threadId, parentToolCallId, /** * Forward the parent's `configurable` so host-set fields * (`requestBody`, `user`, etc.) propagate into the child * workflow. The executor scrubs run-identity fields before * forwarding — see `SubagentExecuteParams.parentConfigurable`. */ parentConfigurable: config.configurable as | Record | undefined, }); return result.content; }, buildSubagentToolParams(resolvedConfigs)); if (!agentContext.graphTools) { agentContext.graphTools = []; } (agentContext.graphTools as t.GenericTool[]).push(subagentTool); if (agentContext.tokenCounter) { const { tokenCounter, baseIndexTokenCountMap } = agentContext; agentContext.tokenCalculationPromise = agentContext .calculateInstructionTokens(tokenCounter) .then(() => { agentContext.updateTokenMapWithInstructions( baseIndexTokenCountMap ); }) .catch((err) => { console.error( 'Error recalculating instruction tokens after subagent tool injection:', err ); }); } } } const agentNode = `${AGENT}${agentId}` as const; const toolNode = `${TOOLS}${agentId}` as const; const routeMessage = ( state: t.BaseGraphState, config?: RunnableConfig ): string => { // First-writer-wins — see note in createCallModel. `this.config` is an // existence flag only; assigning unconditionally would race under // parallel child subgraph.invoke(). this.config ??= config; return toolsCondition(state, toolNode, this.invokedToolIds); }; const StateAnnotation = Annotation.Root({ messages: Annotation({ reducer: messagesStateReducer, default: () => [], }), }); const workflow = new StateGraph(StateAnnotation) .addNode(agentNode, this.createCallModel(agentId)) .addNode( toolNode, this.initializeTools({ currentTools: agentContext.tools, currentToolMap: agentContext.toolMap, agentContext, }) ) .addEdge(START, agentNode) .addConditionalEdges(agentNode, routeMessage) .addEdge(toolNode, agentContext.toolEnd ? END : agentNode); // Cast to unknown to avoid tight coupling to external types; options are opt-in return workflow.compile(this.compileOptions as unknown as never); } createWorkflow(): t.CompiledStateWorkflow { /** Use the default (first) agent for now */ const agentNode = this.createAgentNode(this.defaultAgentId); const StateAnnotation = Annotation.Root({ messages: Annotation({ reducer: (a, b) => { if (!a.length) { this.startIndex = a.length + b.length; } const result = messagesStateReducer(a, b); this.messages = result; return result; }, default: () => [], }), }); // Pass compileOptions (including the HITL checkpointer) to the OUTER // workflow — not just the inner agent subgraph. hasInterrupts() calls // getState() on the outer compiled graph; without a checkpointer here, // getState reports zero tasks and the HITL interrupt/resume loop breaks // out immediately even though interrupt() fired correctly inside the // agent subgraph. const workflow = new StateGraph(StateAnnotation) .addNode(this.defaultAgentId, agentNode, { ends: [END] }) .addEdge(START, this.defaultAgentId) .compile(this.compileOptions as unknown as never); return workflow; } /** * Indicates if this is a multi-agent graph. * Override in MultiAgentGraph to return true. * Used to conditionally include agentId in RunStep for frontend rendering. */ protected isMultiAgentGraph(): boolean { return false; } /** * Get the parallel group ID for an agent, if any. * Override in MultiAgentGraph to provide actual group IDs. * Group IDs are incrementing numbers (1, 2, 3...) reflecting execution order. * @param _agentId - The agent ID to look up * @returns undefined for StandardGraph (no parallel groups), or group number for MultiAgentGraph */ protected getParallelGroupIdForAgent(_agentId: string): number | undefined { return undefined; } /* Dispatchers */ /** * Dispatches a run step to the client, returns the step ID */ async dispatchRunStep( stepKey: string, stepDetails: t.StepDetails, metadata?: Record ): Promise { if (!this.config) { throw new Error('No config provided'); } const [stepId, stepIndex] = this.generateStepId(stepKey); if (stepDetails.type === StepTypes.TOOL_CALLS && stepDetails.tool_calls) { for (const tool_call of stepDetails.tool_calls) { const toolCallId = tool_call.id ?? ''; if (!toolCallId || this.toolCallStepIds.has(toolCallId)) { continue; } this.toolCallStepIds.set(toolCallId, stepId); } } const runStep: t.RunStep = { stepIndex, id: stepId, type: stepDetails.type, index: this.contentData.length, stepDetails, usage: null, }; const runId = this.runId ?? ''; if (runId) { runStep.runId = runId; } /** * Extract agentId and parallelGroupId from metadata * Only set agentId for MultiAgentGraph (so frontend knows when to show agent labels) */ if (metadata) { try { const agentContext = this.getAgentContext(metadata); if (this.isMultiAgentGraph() && agentContext.agentId) { // Only include agentId for MultiAgentGraph - enables frontend to show agent labels runStep.agentId = agentContext.agentId; // Set group ID if this agent is part of a parallel group // Group IDs are incrementing numbers (1, 2, 3...) reflecting execution order const groupId = this.getParallelGroupIdForAgent(agentContext.agentId); if (groupId != null) { runStep.groupId = groupId; } } } catch (_e) { /** If we can't get agent context, that's okay - agentId remains undefined */ mlog( `[dispatchRunStep] Could not resolve agentId from metadata.langgraph_node="${(metadata as Record).langgraph_node}": ${(_e as Error).message}` ); } } this.contentData.push(runStep); this.contentIndexMap.set(stepId, runStep.index); // Pass undefined so safeDispatchCustomEvent resolves the runnable config // from LangChain's AsyncLocalStorage. Using the shared `this.config` would // race across concurrent child subgraph.invoke calls under parallel // multi-agent handoffs and tag events with the wrong child's spawnKey. await safeDispatchCustomEvent(GraphEvents.ON_RUN_STEP, runStep); return stepId; } async handleToolCallCompleted( data: t.ToolEndData, metadata?: Record, omitOutput?: boolean ): Promise { if (!this.config) { throw new Error('No config provided'); } if (!data.output) { return; } const { input, output: _output } = data; if ((_output as Command | undefined)?.lg_name === 'Command') { return; } const output = _output as ToolMessage; const { tool_call_id } = output; const stepId = this.toolCallStepIds.get(tool_call_id) ?? ''; if (!stepId) { throw new Error(`No stepId found for tool_call_id ${tool_call_id}`); } const runStep = this.getRunStep(stepId); if (!runStep) { throw new Error(`No run step found for stepId ${stepId}`); } /** * Extract and store code execution session context from artifacts. * Each file is stamped with its source session_id to support multi-session file tracking. * When the same filename appears in a later execution, the newer version replaces the old. */ const toolName = output.name; if ( toolName === Constants.EXECUTE_CODE || toolName === Constants.PROGRAMMATIC_TOOL_CALLING ) { const artifact = output.artifact as t.CodeExecutionArtifact | undefined; const newFiles = artifact?.files ?? []; const hasNewFiles = newFiles.length > 0; if (artifact?.session_id != null && artifact.session_id !== '') { const existingSession = this.sessions.get(Constants.EXECUTE_CODE) as | t.CodeSessionContext | undefined; const existingFiles = existingSession?.files ?? []; if (hasNewFiles) { /** * Stamp each new file with its source session_id. * This enables files from different executions (parallel or sequential) * to be tracked and passed to subsequent calls. */ const filesWithSession: t.FileRefs = newFiles.map((file) => ({ ...file, session_id: artifact.session_id, })); /** * Merge files, preferring latest versions by name. * If a file with the same name exists, replace it with the new version. * This handles cases where files are edited/recreated in subsequent executions. */ const newFileNames = new Set(filesWithSession.map((f) => f.name)); const filteredExisting = existingFiles.filter( (f) => !newFileNames.has(f.name) ); this.sessions.set(Constants.EXECUTE_CODE, { /** Keep latest session_id for reference/fallback */ session_id: artifact.session_id, /** Accumulated files with latest versions preferred */ files: [...filteredExisting, ...filesWithSession], lastUpdated: Date.now(), }); } else { /** * Even when execution produces no files (e.g., error or print-only), * store the session_id so retries can reuse the same workspace * and access any files written to disk during the failed execution. */ this.sessions.set(Constants.EXECUTE_CODE, { session_id: artifact.session_id, files: existingFiles, lastUpdated: Date.now(), }); } } } const dispatchedOutput = typeof output.content === 'string' ? output.content : JSON.stringify(output.content); const args = typeof input === 'string' ? input : input.input; const tool_call = { args: typeof args === 'string' ? args : JSON.stringify(args), name: output.name ?? '', id: output.tool_call_id, output: omitOutput === true ? '' : dispatchedOutput, progress: 1, }; await this.handlerRegistry ?.getHandler(GraphEvents.ON_RUN_STEP_COMPLETED) ?.handle( GraphEvents.ON_RUN_STEP_COMPLETED, { result: { id: stepId, index: runStep.index, type: 'tool_call', tool_call, } as t.ToolCompleteEvent, }, metadata, this ); } /** * Static version of handleToolCallError to avoid creating strong references * that prevent garbage collection */ static async handleToolCallErrorStatic( graph: StandardGraph, data: t.ToolErrorData, metadata?: Record ): Promise { if (!graph.config) { throw new Error('No config provided'); } if (!data.id) { mwarn('No Tool ID provided for Tool Error'); return; } const stepId = graph.toolCallStepIds.get(data.id) ?? ''; if (!stepId) { throw new Error(`No stepId found for tool_call_id ${data.id}`); } const { name, input: args, error } = data; const runStep = graph.getRunStep(stepId); if (!runStep) { throw new Error(`No run step found for stepId ${stepId}`); } const tool_call: t.ProcessedToolCall = { id: data.id, name: name || '', args: typeof args === 'string' ? args : JSON.stringify(args), output: `Error processing tool${error?.message != null ? `: ${error.message}` : ''}`, progress: 1, }; await graph.handlerRegistry ?.getHandler(GraphEvents.ON_RUN_STEP_COMPLETED) ?.handle( GraphEvents.ON_RUN_STEP_COMPLETED, { result: { id: stepId, index: runStep.index, type: 'tool_call', tool_call, } as t.ToolCompleteEvent, }, metadata, graph ); } /** * Instance method that delegates to the static method * Kept for backward compatibility */ async handleToolCallError( data: t.ToolErrorData, metadata?: Record ): Promise { await StandardGraph.handleToolCallErrorStatic(this, data, metadata); } async dispatchRunStepDelta( id: string, delta: t.ToolCallDelta ): Promise { if (!this.config) { throw new Error('No config provided'); } else if (!id) { throw new Error('No step ID found'); } const runStepDelta: t.RunStepDeltaEvent = { id, delta, }; // See dispatchRunStep note: do not pass `this.config`. The implicit // AsyncLocalStorage config is the correct per-async-branch source under // parallel handoffs. await safeDispatchCustomEvent(GraphEvents.ON_RUN_STEP_DELTA, runStepDelta); } async dispatchMessageDelta(id: string, delta: t.MessageDelta): Promise { if (!this.config) { throw new Error('No config provided'); } const messageDelta: t.MessageDeltaEvent = { id, delta, }; // See dispatchRunStep note. await safeDispatchCustomEvent(GraphEvents.ON_MESSAGE_DELTA, messageDelta); } dispatchReasoningDelta = async ( stepId: string, delta: t.ReasoningDelta ): Promise => { if (!this.config) { throw new Error('No config provided'); } const reasoningDelta: t.ReasoningDeltaEvent = { id: stepId, delta, }; // See dispatchRunStep note. await safeDispatchCustomEvent( GraphEvents.ON_REASONING_DELTA, reasoningDelta ); }; }