/** * Integration tests for ai-providers * * These tests make real API calls and are skipped if credentials are not available. * Set AI_GATEWAY_URL and AI_GATEWAY_TOKEN (or individual provider keys) to run these tests. * * Tests cover: * - End-to-end provider usage * - Gateway integration * - Model generation * - Embedding generation */ import { describe, it, expect } from 'vitest' import { model, embeddingModel, createRegistry } from '../src/index.js' // Skip tests if no credentials configured const hasGateway = !!( process.env.AI_GATEWAY_URL || process.env.ANTHROPIC_API_KEY || process.env.OPENAI_API_KEY || process.env.OPENROUTER_API_KEY ) const hasCloudflare = !!(process.env.CLOUDFLARE_ACCOUNT_ID && process.env.CLOUDFLARE_API_TOKEN) describe.skipIf(!hasGateway)('model() integration', () => { it('gets a language model from registry', async () => { const m = await model('sonnet') expect(m).toBeDefined() }) it('resolves aliases to full model IDs', async () => { const m = await model('opus') expect(m).toBeDefined() }) it('works with full model IDs', async () => { const m = await model('anthropic/claude-sonnet-4.5') expect(m).toBeDefined() }) it('routes OpenAI models to OpenAI SDK', async () => { try { const m = await model('openai/gpt-4o') expect(m).toBeDefined() } catch (error) { // May not be configured expect(error).toBeDefined() } }) it('routes Anthropic models to Anthropic SDK', async () => { try { const m = await model('anthropic/claude-sonnet-4.5') expect(m).toBeDefined() } catch (error) { // May not be configured expect(error).toBeDefined() } }) it('routes Google models to Google SDK', async () => { try { const m = await model('google/gemini-2.5-flash') expect(m).toBeDefined() } catch (error) { // May not be configured expect(error).toBeDefined() } }) it('routes other models through OpenRouter', async () => { try { const m = await model('meta-llama/llama-3.3-70b-instruct') expect(m).toBeDefined() } catch (error) { // May not be configured expect(error).toBeDefined() } }) }) describe.skipIf(!hasGateway)('embeddingModel() integration', () => { it('gets an OpenAI embedding model', async () => { try { const em = await embeddingModel('openai:text-embedding-3-small') expect(em).toBeDefined() expect(em.provider).toBe('openai') } catch (error) { // May not be configured expect(error).toBeDefined() } }) it.skipIf(!hasCloudflare)('gets a Cloudflare embedding model', async () => { const em = await embeddingModel('cloudflare:@cf/baai/bge-m3') expect(em).toBeDefined() expect(em.provider).toBe('cloudflare') }) }) describe.skipIf(!hasGateway)('createRegistry() integration', () => { it('creates registry with gateway config', async () => { const registry = await createRegistry({ gatewayUrl: process.env.AI_GATEWAY_URL, gatewayToken: process.env.AI_GATEWAY_TOKEN, }) expect(registry).toBeDefined() expect(typeof registry.languageModel).toBe('function') expect(typeof registry.textEmbeddingModel).toBe('function') }) it('creates registry with direct API keys', async () => { const registry = await createRegistry({ openaiApiKey: process.env.OPENAI_API_KEY, anthropicApiKey: process.env.ANTHROPIC_API_KEY, openrouterApiKey: process.env.OPENROUTER_API_KEY, }) expect(registry).toBeDefined() }) it('can use registry to get models', async () => { const registry = await createRegistry() try { const m = registry.languageModel('openrouter:anthropic/claude-sonnet-4.5') expect(m).toBeDefined() } catch (error) { // May not be configured expect(error).toBeDefined() } }) }) describe.skipIf(!hasGateway)('gateway authentication', () => { it('works with gateway stored secrets', async () => { if (!process.env.AI_GATEWAY_URL || !process.env.AI_GATEWAY_TOKEN) { return } const registry = await createRegistry({ gatewayUrl: process.env.AI_GATEWAY_URL, gatewayToken: process.env.AI_GATEWAY_TOKEN, // No individual API keys needed }) expect(registry).toBeDefined() }) it('custom fetch adds cf-aig-authorization header', async () => { // This is tested indirectly through successful API calls with gateway expect(true).toBe(true) }) }) describe.skipIf(!hasCloudflare)('Cloudflare embeddings integration', () => { it('generates embeddings via REST API', async () => { const { cloudflareEmbedding } = await import('../src/providers/cloudflare.js') const model = cloudflareEmbedding('@cf/baai/bge-m3', { accountId: process.env.CLOUDFLARE_ACCOUNT_ID, apiToken: process.env.CLOUDFLARE_API_TOKEN, }) const result = await model.doEmbed({ values: ['hello world', 'test embedding'], }) expect(result.embeddings).toHaveLength(2) expect(result.embeddings[0].length).toBeGreaterThan(0) expect(Array.isArray(result.embeddings[0])).toBe(true) }) it('supports different embedding models', async () => { const { cloudflareEmbedding } = await import('../src/providers/cloudflare.js') const models = ['@cf/baai/bge-small-en-v1.5', '@cf/baai/bge-base-en-v1.5', '@cf/baai/bge-m3'] for (const modelId of models) { const model = cloudflareEmbedding(modelId, { accountId: process.env.CLOUDFLARE_ACCOUNT_ID, apiToken: process.env.CLOUDFLARE_API_TOKEN, }) const result = await model.doEmbed({ values: ['test'], }) expect(result.embeddings).toHaveLength(1) } }, 30000) // Allow extra time for multiple API calls }) describe.skipIf(!hasGateway)('provider-specific features', () => { it.skip('supports Anthropic MCP with direct routing', async () => { // Test MCP features when using direct Anthropic SDK routing // Requires actual MCP implementation expect(true).toBe(true) }) it.skip('supports OpenAI function calling with direct routing', async () => { // Test function calling when using direct OpenAI SDK routing expect(true).toBe(true) }) it.skip('supports Google grounding with direct routing', async () => { // Test grounding when using direct Google SDK routing expect(true).toBe(true) }) }) describe.skipIf(!hasGateway)('error handling', () => { it('handles invalid model IDs gracefully', async () => { try { await model('invalid-model-that-does-not-exist-12345') } catch (error) { expect(error).toBeDefined() } }) it('handles network errors gracefully', async () => { const registry = await createRegistry({ baseUrls: { openrouter: 'https://invalid-url-that-does-not-exist.example.com', }, }) try { const m = registry.languageModel('openrouter:test') // Attempting to use this model should fail expect(m).toBeDefined() } catch (error) { expect(error).toBeDefined() } }) it('handles missing credentials gracefully', async () => { const registry = await createRegistry({ // No credentials provided gatewayUrl: undefined, gatewayToken: undefined, openaiApiKey: undefined, anthropicApiKey: undefined, }) // Registry should be created, but using models may fail expect(registry).toBeDefined() }) }) describe.skipIf(!hasGateway)('concurrent requests', () => { it('handles multiple concurrent model resolutions', async () => { const promises = [model('sonnet'), model('opus'), model('gpt-4o')] try { const models = await Promise.all(promises) expect(models).toHaveLength(3) models.forEach((m) => expect(m).toBeDefined()) } catch (error) { // Some models may not be configured expect(error).toBeDefined() } }) it('caches registry to avoid duplicate initialization', async () => { const { getRegistry } = await import('../src/index.js') const promises = [getRegistry(), getRegistry(), getRegistry()] const registries = await Promise.all(promises) // All should be the same instance expect(registries[0]).toBe(registries[1]) expect(registries[1]).toBe(registries[2]) }) }) describe.skipIf(!hasGateway)('performance', () => { it('resolves models quickly after initial setup', async () => { const start = Date.now() await model('sonnet') const duration = Date.now() - start // After registry is initialized, should be fast // Allow up to 1 second for initialization expect(duration).toBeLessThan(1000) }) it('caches registry for subsequent calls', async () => { // First call initializes await model('sonnet') // Second call should use cached registry const start = Date.now() await model('opus') const duration = Date.now() - start // Should be very fast (< 100ms) when cached expect(duration).toBeLessThan(100) }) })