/** * Optimized ChatBedrockConverse wrapper that fixes content block merging for * streaming responses and adds support for latest @langchain/aws features: * * - Prompt caching support for Bedrock Converse API (Illuma feature) * - Application Inference Profiles (PR #9129) * - Service Tiers (Priority/Standard/Flex) (PR #9785) - requires AWS SDK 3.966.0+ * * Bedrock's `@langchain/aws` library does not include an `index` property on content * blocks (unlike Anthropic/OpenAI), which causes LangChain's `_mergeLists` to append * each streaming chunk as a separate array entry instead of merging by index. * * This wrapper takes full ownership of the stream by directly interfacing with the * AWS SDK client (`this.client`) and using custom handlers from `./utils/` that * include `contentBlockIndex` in response_metadata for every delta type. It then * promotes `contentBlockIndex` to an `index` property on each content block * (mirroring Anthropic's pattern) and strips it from metadata to avoid * `_mergeDicts` conflicts. * * When multiple content block types are present (e.g. reasoning + text), text deltas * are promoted from strings to array form with `index` so they merge correctly once * the accumulated content is already an array. * * PROMPT CACHING: * When promptCache: true is set, this wrapper adds cachePoint markers to the tools array * to enable Bedrock prompt caching for tool definitions. This allows tool schemas to be * cached and reused across requests, reducing latency and costs. * * CACHE TOKEN EXTRACTION: * Cache token extraction is handled in `./utils/message_outputs.ts` and added * to usage_metadata with input_token_details (cacheReadInputTokens/cacheWriteInputTokens). */ import { ChatBedrockConverse } from '@langchain/aws'; import type { BaseMessage } from '@langchain/core/messages'; import { ChatGenerationChunk, ChatResult } from '@langchain/core/outputs'; import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager'; import type { ChatBedrockConverseInput } from '@langchain/aws'; /** * Service tier type for Bedrock invocations. * Requires AWS SDK >= 3.966.0 to actually work. * @see https://docs.aws.amazon.com/bedrock/latest/userguide/service-tiers-inference.html */ export type ServiceTierType = 'priority' | 'default' | 'flex' | 'reserved'; /** * Extended input interface with additional features: * - promptCache: Enable Bedrock prompt caching for tool definitions * - applicationInferenceProfile: Use an inference profile ARN instead of model ID * - serviceTier: Specify service tier (Priority, Standard, Flex, Reserved) */ export interface IllumaBedrockConverseInput extends ChatBedrockConverseInput { /** * Enable Bedrock prompt caching for tool definitions. * When true, adds cachePoint markers to tools array. */ promptCache?: boolean; /** * Application Inference Profile ARN to use for the model. * For example, "arn:aws:bedrock:eu-west-1:123456789102:application-inference-profile/fm16bt65tzgx" * When provided, this ARN will be used for the actual inference calls instead of the model ID. * Must still provide `model` as normal modelId to benefit from all the metadata. * @see https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-create.html */ applicationInferenceProfile?: string; /** * Service tier for model invocation. * Specifies the processing tier type used for serving the request. * Supported values are 'priority', 'default', 'flex', and 'reserved'. * * - 'priority': Prioritized processing for lower latency * - 'default': Standard processing tier * - 'flex': Flexible processing tier with lower cost * - 'reserved': Reserved capacity for consistent performance * * If not provided, AWS uses the default tier. * Note: Requires AWS SDK >= 3.966.0 to work. * @see https://docs.aws.amazon.com/bedrock/latest/userguide/service-tiers-inference.html */ serviceTier?: ServiceTierType; } /** * Extended call options with serviceTier override support. */ export interface IllumaBedrockConverseCallOptions { serviceTier?: ServiceTierType; } export declare class IllumaBedrockConverse extends ChatBedrockConverse { /** Enable Bedrock prompt caching for tool definitions */ promptCache: boolean; /** Application Inference Profile ARN to use instead of model ID */ applicationInferenceProfile?: string; /** Service tier for model invocation */ serviceTier?: ServiceTierType; constructor(fields?: IllumaBedrockConverseInput); static lc_name(): string; /** * Get the model ID to use for API calls. * Returns applicationInferenceProfile if set, otherwise returns this.model. */ protected getModelId(): string; /** * Override invocationParams to: * 1. Add cachePoint to tools when promptCache is enabled * 2. Add serviceTier support * * CACHING STRATEGY: Separate cachePoints for core tools and MCP tools * - Core tools (web_search, execute_code, etc.) are stable → cache first * - MCP tools (have '_mcp_' in name) are dynamic → cache separately after * - This allows core tools to stay cached when MCP selection changes * * NOTE: Only Claude models support cachePoint - Nova and other models will reject it. */ invocationParams(options?: this['ParsedCallOptions'] & IllumaBedrockConverseCallOptions): ReturnType & { serviceTier?: { type: ServiceTierType; }; }; /** * Override _generateNonStreaming to use applicationInferenceProfile as modelId. * Uses the same model-swapping pattern as streaming for consistency. */ _generateNonStreaming(messages: BaseMessage[], options: this['ParsedCallOptions'] & IllumaBedrockConverseCallOptions, runManager?: CallbackManagerForLLMRun): Promise; /** * Own the stream end-to-end so we have direct access to every * `contentBlockDelta.contentBlockIndex` from the AWS SDK. * * This replaces the parent's implementation which strips contentBlockIndex * from text and reasoning deltas, making it impossible to merge correctly. */ _streamResponseChunks(messages: BaseMessage[], options: this['ParsedCallOptions'] & IllumaBedrockConverseCallOptions, runManager?: CallbackManagerForLLMRun): AsyncGenerator; /** * Inject `index` on content blocks for proper merge behaviour, then strip * `contentBlockIndex` from response_metadata to prevent `_mergeDicts` conflicts. * * Text string content is promoted to array form only when the stream contains * multiple content block indices (e.g. reasoning at index 0, text at index 1), * ensuring text merges correctly with the already-array accumulated content. */ enrichChunk(chunk: ChatGenerationChunk, seenBlockIndices: Set): ChatGenerationChunk; /** * Extract `contentBlockIndex` from the top level of response_metadata. * Our custom handlers always place it at the top level. */ private extractContentBlockIndex; private removeContentBlockIndex; } /** @deprecated Use IllumaBedrockConverse. Alias kept for backward compatibility with existing imports. */ export declare const CustomChatBedrockConverse: typeof IllumaBedrockConverse; export type { ChatBedrockConverseInput };