/* eslint-disable no-process-env */ /* eslint-disable no-console */ /* eslint-disable @typescript-eslint/no-explicit-any */ /** * Integration tests for structured output with real LLM API calls. * * These tests require actual API credentials. They will be skipped if * the required environment variables are not set. * * To run: * 1. Set env vars directly, or set AGENTS_TEST_ENV_PATH to point at * a host .env file containing the required credentials. * 2. npm test -- --testPathPattern=structured-output.integration * * Environment variables needed: * - BEDROCK_AWS_ACCESS_KEY_ID + BEDROCK_AWS_SECRET_ACCESS_KEY + BEDROCK_AWS_DEFAULT_REGION * - ANTHROPIC_API_KEY (optional, for direct Anthropic tests) * - OPENAI_API_KEY (optional, for direct OpenAI tests) */ import { config } from 'dotenv'; import { resolve } from 'path'; // Load local .env first; optionally fall back to a host-provided env file. config(); if (process.env.AGENTS_TEST_ENV_PATH) { config({ path: resolve(process.env.AGENTS_TEST_ENV_PATH) }); } import { HumanMessage, SystemMessage, AIMessageChunk, } from '@langchain/core/messages'; import { CustomChatBedrockConverse } from '@/llm/bedrock'; import { Providers } from '@/common'; import { validateStructuredOutput, prepareSchemaForProvider, } from '@/schemas/validate'; jest.setTimeout(120000); // ────────────────────────────────────────────────────────────── // Helper: check if credentials are available // ────────────────────────────────────────────────────────────── const hasBedrock = !!process.env.BEDROCK_AWS_ACCESS_KEY_ID && !!process.env.BEDROCK_AWS_SECRET_ACCESS_KEY; const hasAnthropic = !!process.env.ANTHROPIC_API_KEY; const hasOpenAI = !!process.env.OPENAI_API_KEY; // ────────────────────────────────────────────────────────────── // Shared test schema // ────────────────────────────────────────────────────────────── const personSchema = { type: 'object' as const, properties: { name: { type: 'string', description: 'Full name of the person' }, age: { type: 'number', description: 'Age in years' }, occupation: { type: 'string', description: 'Job title or occupation' }, }, required: ['name', 'age', 'occupation'], additionalProperties: false, }; const classificationSchema = { type: 'object' as const, properties: { category: { type: 'string', enum: ['positive', 'negative', 'neutral'], description: 'Sentiment category', }, confidence: { type: 'number', description: 'Confidence score between 0 and 1', }, reasoning: { type: 'string', description: 'Brief explanation of the classification', }, }, required: ['category', 'confidence', 'reasoning'], additionalProperties: false, }; // ────────────────────────────────────────────────────────────── // Bedrock Integration Tests // ────────────────────────────────────────────────────────────── describe('Bedrock structured output integration', () => { const bedrockRegion = process.env.BEDROCK_AWS_DEFAULT_REGION || 'us-east-1'; const createBedrockModel = () => new CustomChatBedrockConverse({ // Use Claude 3 Haiku which is widely available and supports on-demand invocation model: 'anthropic.claude-3-haiku-20240307-v1:0', region: bedrockRegion, credentials: { accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID!, secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY!, ...(process.env.BEDROCK_AWS_SESSION_TOKEN ? { sessionToken: process.env.BEDROCK_AWS_SESSION_TOKEN } : {}), }, maxTokens: 1024, }); const conditionalTest = hasBedrock ? test : test.skip; conditionalTest( 'withStructuredOutput functionCalling - basic person extraction', async () => { const model = createBedrockModel(); const structuredModel = model.withStructuredOutput(personSchema, { name: 'PersonInfo', method: 'functionCalling' as any, includeRaw: true, strict: true, }); const result = await structuredModel.invoke([ new SystemMessage('Extract person information from the text.'), new HumanMessage( 'John Smith is a 35 year old software engineer from Seattle.' ), ]); console.log( '[Integration] Bedrock functionCalling result:', JSON.stringify(result.parsed ?? result, null, 2) ); const parsed = result.parsed ?? result; expect(parsed).toBeDefined(); expect(typeof parsed.name).toBe('string'); expect(typeof parsed.age).toBe('number'); expect(typeof parsed.occupation).toBe('string'); // Validate against schema const validation = validateStructuredOutput(parsed, personSchema); expect(validation.success).toBe(true); } ); conditionalTest( 'withStructuredOutput functionCalling - classification with enum', async () => { const model = createBedrockModel(); const structuredModel = model.withStructuredOutput(classificationSchema, { name: 'Classification', method: 'functionCalling' as any, includeRaw: true, strict: true, }); const result = await structuredModel.invoke([ new SystemMessage('Classify the sentiment of the following text.'), new HumanMessage( 'I absolutely love this product! It exceeded all my expectations.' ), ]); const parsed = result.parsed ?? result; console.log( '[Integration] Bedrock classification result:', JSON.stringify(parsed, null, 2) ); expect(parsed).toBeDefined(); expect(['positive', 'negative', 'neutral']).toContain(parsed.category); expect(typeof parsed.confidence).toBe('number'); expect(typeof parsed.reasoning).toBe('string'); } ); conditionalTest('withStructuredOutput with prepared schema', async () => { const rawSchema = { type: 'object', properties: { summary: { type: 'string', maxLength: 500 }, topics: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: 5, }, wordCount: { type: 'number', minimum: 0 }, }, }; // Prepare schema for Bedrock const { schema: prepared, warnings } = prepareSchemaForProvider( rawSchema, Providers.BEDROCK ); console.log('[Integration] Schema preparation warnings:', warnings); // Verify preparation worked expect(prepared.additionalProperties).toBe(false); expect(prepared.required).toEqual( expect.arrayContaining(['summary', 'topics', 'wordCount']) ); const summaryProp = (prepared.properties as any).summary; expect(summaryProp.maxLength).toBeUndefined(); expect(summaryProp.description).toContain('maxLength'); // Now use the prepared schema with the model const model = createBedrockModel(); const structuredModel = model.withStructuredOutput(prepared, { name: 'Analysis', method: 'functionCalling' as any, includeRaw: true, strict: true, }); const result = await structuredModel.invoke([ new SystemMessage( 'Analyze the following text and provide a summary, key topics, and word count.' ), new HumanMessage( 'Artificial intelligence has transformed the technology industry in recent years. ' + 'Machine learning models are now capable of generating text, images, and even code. ' + 'Companies are investing heavily in AI research and development.' ), ]); const parsed = result.parsed ?? result; console.log( '[Integration] Bedrock analysis result:', JSON.stringify(parsed, null, 2) ); expect(parsed).toBeDefined(); expect(typeof parsed.summary).toBe('string'); expect(Array.isArray(parsed.topics)).toBe(true); expect(typeof parsed.wordCount).toBe('number'); }); conditionalTest( 'withStructuredOutput with tools - deferred structured output pattern', async () => { const model = createBedrockModel(); // Step 1: Regular invocation with tools (simulate tool use) const messages = [ new SystemMessage( 'You are a helpful assistant. When asked to analyze text, provide a structured analysis.' ), new HumanMessage( 'Analyze the sentiment: "This movie was absolutely terrible, worst I have ever seen."' ), ]; // Step 2: Get structured output (this simulates the deferred structured output pattern) const structuredModel = model.withStructuredOutput(classificationSchema, { name: 'SentimentAnalysis', method: 'functionCalling' as any, includeRaw: true, strict: true, }); const result = await structuredModel.invoke(messages); const parsed = result.parsed ?? result; console.log( '[Integration] Bedrock deferred structured result:', JSON.stringify(parsed, null, 2) ); expect(parsed).toBeDefined(); expect(['positive', 'negative', 'neutral']).toContain(parsed.category); // The movie review is clearly negative expect(parsed.category).toBe('negative'); expect(typeof parsed.confidence).toBe('number'); } ); conditionalTest( 'validates response against schema and detects failures', async () => { // Test schema validation on a known good response const goodResponse = { name: 'Alice', age: 30, occupation: 'Engineer', }; const goodResult = validateStructuredOutput(goodResponse, personSchema); expect(goodResult.success).toBe(true); // Test with a bad response const badResponse = { name: 123, // Wrong type age: 'thirty', // Wrong type }; const badResult = validateStructuredOutput(badResponse, personSchema); expect(badResult.success).toBe(false); expect(badResult.error).toBeDefined(); } ); }); // ────────────────────────────────────────────────────────────── // Complex Tool + Structured Output Integration Tests (Bedrock) // ────────────────────────────────────────────────────────────── describe('Bedrock complex tool + structured output integration', () => { const bedrockRegion = process.env.BEDROCK_AWS_DEFAULT_REGION || 'us-east-1'; const conditionalTest = hasBedrock ? test : test.skip; const createBedrockModel = () => new CustomChatBedrockConverse({ model: 'anthropic.claude-3-haiku-20240307-v1:0', region: bedrockRegion, credentials: { accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID!, secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY!, ...(process.env.BEDROCK_AWS_SESSION_TOKEN ? { sessionToken: process.env.BEDROCK_AWS_SESSION_TOKEN } : {}), }, maxTokens: 2048, }); conditionalTest( 'complex schema with nested objects and arrays - multi-step analysis', async () => { const complexSchema = { type: 'object' as const, properties: { analysis: { type: 'object', properties: { overall_sentiment: { type: 'string', enum: [ 'very_positive', 'positive', 'neutral', 'negative', 'very_negative', ], }, confidence: { type: 'number' }, key_themes: { type: 'array', items: { type: 'object', properties: { theme: { type: 'string' }, sentiment: { type: 'string', enum: ['positive', 'negative', 'neutral'], }, evidence: { type: 'string' }, }, required: ['theme', 'sentiment', 'evidence'], additionalProperties: false, }, }, }, required: ['overall_sentiment', 'confidence', 'key_themes'], additionalProperties: false, }, entities: { type: 'array', items: { type: 'object', properties: { name: { type: 'string' }, type: { type: 'string', enum: ['person', 'organization', 'location', 'product'], }, role: { type: 'string' }, }, required: ['name', 'type', 'role'], additionalProperties: false, }, }, summary: { type: 'string' }, action_items: { type: 'array', items: { type: 'string' }, }, }, required: ['analysis', 'entities', 'summary', 'action_items'], additionalProperties: false, }; const model = createBedrockModel(); const structuredModel = model.withStructuredOutput(complexSchema, { name: 'DetailedAnalysis', method: 'functionCalling' as any, includeRaw: true, strict: true, }); const result = await structuredModel.invoke([ new SystemMessage( 'You are an expert analyst. Analyze the following business communication and extract detailed structured information.' ), new HumanMessage( 'From: Sarah Johnson, VP of Engineering at TechCorp\n' + 'To: All Engineering Teams\n\n' + 'I am pleased to announce that our new AI platform, CodeAssist, has exceeded Q3 targets by 45%. ' + 'The Seattle development team, led by Marcus Chen, deserves special recognition for their outstanding work. ' + 'However, we need to address the performance issues reported by our European clients. ' + 'The London office has flagged latency problems that must be resolved before the Q4 launch. ' + 'Action items: 1) Marcus to lead a performance task force. 2) Schedule a meeting with the London team. ' + '3) Update the deployment pipeline for EU regions.' ), ]); const parsed = result.parsed ?? result; console.log( '[Integration] Complex analysis result:', JSON.stringify(parsed, null, 2) ); // Verify top-level structure // Note: functionCalling mode is best-effort, so some fields may be missing. // Native constrained decoding (when available) would guarantee all fields. expect(parsed).toBeDefined(); expect(typeof parsed.summary).toBe('string'); // Analysis is the most complex nested field — verify if present if (parsed.analysis) { expect([ 'very_positive', 'positive', 'neutral', 'negative', 'very_negative', ]).toContain(parsed.analysis.overall_sentiment); expect(typeof parsed.analysis.confidence).toBe('number'); if (Array.isArray(parsed.analysis.key_themes)) { expect(parsed.analysis.key_themes.length).toBeGreaterThan(0); for (const theme of parsed.analysis.key_themes) { expect(typeof theme.theme).toBe('string'); expect(['positive', 'negative', 'neutral']).toContain( theme.sentiment ); expect(typeof theme.evidence).toBe('string'); } } } // Verify entities if present if (Array.isArray(parsed.entities)) { expect(parsed.entities.length).toBeGreaterThan(0); for (const entity of parsed.entities) { expect(typeof entity.name).toBe('string'); expect(['person', 'organization', 'location', 'product']).toContain( entity.type ); } } // Verify action items if (Array.isArray(parsed.action_items)) { expect(parsed.action_items.length).toBeGreaterThan(0); } // At minimum, we expect the model returned something parseable with the key fields expect(parsed.summary.length).toBeGreaterThan(0); } ); conditionalTest( 'tool-like multi-turn conversation with structured output at the end', async () => { // Simulate a multi-turn conversation where the agent has gathered info // and now needs to produce structured output const reportSchema = { type: 'object' as const, properties: { report_title: { type: 'string' }, findings: { type: 'array', items: { type: 'object', properties: { category: { type: 'string' }, description: { type: 'string' }, severity: { type: 'string', enum: ['low', 'medium', 'high', 'critical'], }, recommendation: { type: 'string' }, }, required: [ 'category', 'description', 'severity', 'recommendation', ], additionalProperties: false, }, }, overall_risk_level: { type: 'string', enum: ['low', 'medium', 'high', 'critical'], }, next_steps: { type: 'array', items: { type: 'string' }, }, }, required: [ 'report_title', 'findings', 'overall_risk_level', 'next_steps', ], additionalProperties: false, }; const model = createBedrockModel(); // Multi-turn conversation simulating tool results being fed in const messages = [ new SystemMessage( 'You are a security analyst. You have performed a security audit and gathered the following data from various scans. ' + 'Produce a structured security report.' ), new HumanMessage( 'Here are the scan results:\n\n' + 'Port Scan: Ports 22, 80, 443, 8080 are open. Port 8080 is running an outdated Apache Tomcat 8.5.\n\n' + 'Vulnerability Scan: Found SQL injection in /api/users endpoint. XSS vulnerability in search form. ' + 'Outdated OpenSSL 1.1.1 (should be 3.x). No rate limiting on login endpoint.\n\n' + 'Configuration Review: Default admin credentials found on Tomcat manager. ' + 'CORS is set to wildcard (*). No CSP headers. Cookies without HttpOnly flag.\n\n' + 'Provide your structured security report.' ), ]; const structuredModel = model.withStructuredOutput(reportSchema, { name: 'SecurityReport', method: 'functionCalling' as any, includeRaw: true, strict: true, }); const result = await structuredModel.invoke(messages); const parsed = result.parsed ?? result; console.log( '[Integration] Security report result:', JSON.stringify(parsed, null, 2) ); // Verify structure expect(typeof parsed.report_title).toBe('string'); expect(Array.isArray(parsed.findings)).toBe(true); expect(parsed.findings.length).toBeGreaterThanOrEqual(3); // At least SQL injection, XSS, outdated software expect(['low', 'medium', 'high', 'critical']).toContain( parsed.overall_risk_level ); expect(Array.isArray(parsed.next_steps)).toBe(true); expect(parsed.next_steps.length).toBeGreaterThan(0); // Verify each finding for (const finding of parsed.findings) { expect(typeof finding.category).toBe('string'); expect(typeof finding.description).toBe('string'); expect(['low', 'medium', 'high', 'critical']).toContain( finding.severity ); expect(typeof finding.recommendation).toBe('string'); } // Given the scan results, risk should be high or critical expect(['high', 'critical']).toContain(parsed.overall_risk_level); // Validate entire response const validation = validateStructuredOutput(parsed, reportSchema); expect(validation.success).toBe(true); } ); conditionalTest( 'prepared schema with constraint stripping works end-to-end', async () => { // A schema with MANY unsupported constraints that need stripping const rawSchema = { type: 'object' as const, properties: { title: { type: 'string', minLength: 5, maxLength: 100 }, rating: { type: 'number', minimum: 1, maximum: 5, multipleOf: 0.5 }, tags: { type: 'array', items: { type: 'string', minLength: 1, maxLength: 50 }, minItems: 1, maxItems: 5, uniqueItems: true, }, metadata: { type: 'object', properties: { author: { type: 'string', pattern: '^[A-Za-z ]+$' }, word_count: { type: 'integer', minimum: 0, maximum: 100000 }, language: { type: 'string', enum: ['en', 'es', 'fr', 'de'] }, }, }, }, }; // Prepare for Bedrock const { schema: prepared, warnings } = prepareSchemaForProvider( rawSchema, Providers.BEDROCK ); console.log( '[Integration] Complex schema warnings count:', warnings.length ); expect(warnings.length).toBeGreaterThan(5); // Should have many warnings // Verify ALL constraints were stripped const titleProp = (prepared.properties as any).title; expect(titleProp.minLength).toBeUndefined(); expect(titleProp.maxLength).toBeUndefined(); const ratingProp = (prepared.properties as any).rating; expect(ratingProp.minimum).toBeUndefined(); expect(ratingProp.maximum).toBeUndefined(); expect(ratingProp.multipleOf).toBeUndefined(); const tagsProp = (prepared.properties as any).tags; expect(tagsProp.minItems).toBeUndefined(); expect(tagsProp.maxItems).toBeUndefined(); expect(tagsProp.uniqueItems).toBeUndefined(); // Now use with actual model const model = createBedrockModel(); const structuredModel = model.withStructuredOutput(prepared, { name: 'BookReview', method: 'functionCalling' as any, includeRaw: true, strict: true, }); const result = await structuredModel.invoke([ new SystemMessage( 'You are a book reviewer. Review the book described below.' ), new HumanMessage( 'Review "The Great Gatsby" by F. Scott Fitzgerald. ' + 'It is a classic American novel written in English about the decline of the American Dream.' ), ]); const parsed = result.parsed ?? result; console.log( '[Integration] Book review result:', JSON.stringify(parsed, null, 2) ); expect(typeof parsed.title).toBe('string'); expect(typeof parsed.rating).toBe('number'); expect(Array.isArray(parsed.tags)).toBe(true); expect(parsed.metadata).toBeDefined(); expect(['en', 'es', 'fr', 'de']).toContain(parsed.metadata.language); } ); }); // ────────────────────────────────────────────────────────────── // Anthropic Direct Integration Tests // ────────────────────────────────────────────────────────────── describe('Anthropic direct structured output integration', () => { const conditionalTest = hasAnthropic ? test : test.skip; conditionalTest( 'withStructuredOutput jsonSchema (native) - person extraction', async () => { // Dynamic import to avoid errors when @langchain/anthropic is not installed const { CustomAnthropic } = await import('@/llm/anthropic'); const model = new CustomAnthropic({ anthropicApiKey: process.env.ANTHROPIC_API_KEY, model: 'claude-sonnet-4-20250514', maxTokens: 1024, }); const structuredModel = model.withStructuredOutput(personSchema, { name: 'PersonInfo', method: 'jsonSchema' as any, includeRaw: true, strict: true, }); const result = await structuredModel.invoke([ new SystemMessage('Extract person information from the text.'), new HumanMessage( 'Marie Curie was a 66 year old physicist and chemist.' ), ]); const parsed = result.parsed ?? result; console.log( '[Integration] Anthropic native result:', JSON.stringify(parsed, null, 2) ); expect(parsed).toBeDefined(); expect(typeof parsed.name).toBe('string'); expect(typeof parsed.age).toBe('number'); expect(typeof parsed.occupation).toBe('string'); const validation = validateStructuredOutput(parsed, personSchema); expect(validation.success).toBe(true); } ); conditionalTest( 'withStructuredOutput jsonSchema - classification with enum', async () => { const { CustomAnthropic } = await import('@/llm/anthropic'); const model = new CustomAnthropic({ anthropicApiKey: process.env.ANTHROPIC_API_KEY, model: 'claude-sonnet-4-20250514', maxTokens: 1024, }); const structuredModel = model.withStructuredOutput(classificationSchema, { name: 'Sentiment', method: 'jsonSchema' as any, includeRaw: true, strict: true, }); const result = await structuredModel.invoke([ new SystemMessage('Classify the sentiment.'), new HumanMessage('This is the best day of my life!'), ]); const parsed = result.parsed ?? result; console.log( '[Integration] Anthropic classification:', JSON.stringify(parsed, null, 2) ); expect(['positive', 'negative', 'neutral']).toContain(parsed.category); expect(parsed.category).toBe('positive'); } ); }); // ────────────────────────────────────────────────────────────── // OpenAI Integration Tests // ────────────────────────────────────────────────────────────── describe('OpenAI structured output integration', () => { const conditionalTest = hasOpenAI ? test : test.skip; conditionalTest( 'withStructuredOutput jsonSchema (native) - person extraction', async () => { const { ChatOpenAI } = await import('@/llm/openai'); const model = new ChatOpenAI({ openAIApiKey: process.env.OPENAI_API_KEY, model: 'gpt-4o-mini', maxTokens: 1024, }); const structuredModel = model.withStructuredOutput(personSchema, { name: 'PersonInfo', method: 'jsonSchema' as any, includeRaw: true, strict: true, }); const result = await structuredModel.invoke([ new SystemMessage('Extract person information from the text.'), new HumanMessage( 'Albert Einstein was a 76 year old theoretical physicist.' ), ]); const parsed = result.parsed ?? result; console.log( '[Integration] OpenAI native result:', JSON.stringify(parsed, null, 2) ); expect(parsed).toBeDefined(); expect(typeof parsed.name).toBe('string'); expect(typeof parsed.age).toBe('number'); const validation = validateStructuredOutput(parsed, personSchema); expect(validation.success).toBe(true); } ); }); // ────────────────────────────────────────────────────────────── // Schema Preparation Integration Tests // ────────────────────────────────────────────────────────────── describe('Schema preparation produces valid schemas for all providers', () => { const complexSchema = { type: 'object', properties: { analysis: { type: 'object', properties: { score: { type: 'number', minimum: 0, maximum: 100 }, label: { type: 'string', enum: ['low', 'medium', 'high'] }, details: { type: 'string', maxLength: 1000 }, }, }, tags: { type: 'array', items: { type: 'string', minLength: 1 }, minItems: 1, maxItems: 10, }, metadata: { type: 'object', properties: { source: { type: 'string' }, timestamp: { type: 'string' }, }, }, }, }; test('prepares valid schema for Anthropic', () => { const { schema, warnings } = prepareSchemaForProvider( complexSchema, Providers.ANTHROPIC ); // Root object expect(schema.additionalProperties).toBe(false); expect(schema.required).toEqual( expect.arrayContaining(['analysis', 'tags', 'metadata']) ); // Nested object: analysis const analysis = (schema.properties as any).analysis; expect(analysis.additionalProperties).toBe(false); expect(analysis.required).toEqual( expect.arrayContaining(['score', 'label', 'details']) ); // Numeric constraints stripped expect(analysis.properties.score.minimum).toBeUndefined(); expect(analysis.properties.score.maximum).toBeUndefined(); // String constraints stripped expect(analysis.properties.details.maxLength).toBeUndefined(); // Array constraints stripped const tags = (schema.properties as any).tags; expect(tags.minItems).toBeUndefined(); expect(tags.maxItems).toBeUndefined(); // Array item string constraints stripped expect(tags.items.minLength).toBeUndefined(); // Nested object: metadata const metadata = (schema.properties as any).metadata; expect(metadata.additionalProperties).toBe(false); expect(warnings.length).toBeGreaterThan(0); console.log('[Integration] Anthropic schema warnings:', warnings); }); test('prepares valid schema for OpenAI', () => { const { schema } = prepareSchemaForProvider( complexSchema, Providers.OPENAI ); expect(schema.additionalProperties).toBe(false); const analysis = (schema.properties as any).analysis; expect(analysis.properties.score.minimum).toBeUndefined(); }); test('prepares valid schema for Bedrock', () => { const { schema } = prepareSchemaForProvider( complexSchema, Providers.BEDROCK ); expect(schema.additionalProperties).toBe(false); const analysis = (schema.properties as any).analysis; expect(analysis.properties.score.minimum).toBeUndefined(); }); });