// @ts-nocheck import { describe, it, expect, beforeEach, afterEach, jest } from '@jest/globals'; // Mock huggingface transformers before importing anything else jest.mock('@huggingface/transformers', () => ({ pipeline: jest.fn(), })); import { SemanticAnalyzer } from '../src/semantic-analyzer'; describe('SemanticAnalyzer', () => { let analyzer: SemanticAnalyzer; let mockPipeline: jest.MockedFunction; beforeEach(async () => { const transformers = await import('@huggingface/transformers'); mockPipeline = transformers.pipeline as jest.MockedFunction; analyzer = new SemanticAnalyzer(); }); afterEach(() => { jest.clearAllMocks(); }); describe('initialization', () => { it('should initialize successfully with both embedding and NLI models', async () => { const mockEmbeddingModel = jest.fn(); const mockNLIModel = jest.fn(); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); mockPipeline.mockResolvedValueOnce(mockNLIModel); await analyzer.initialize(); expect(mockPipeline).toHaveBeenCalledWith( 'feature-extraction', 'Xenova/all-MiniLM-L6-v2', expect.objectContaining({ cache_dir: expect.any(String), }) ); expect(mockPipeline).toHaveBeenCalledWith( 'zero-shot-classification', 'MoritzLaurer/deberta-v3-base-zeroshot-v1.1-all-33', expect.objectContaining({ cache_dir: expect.any(String), }) ); expect(analyzer.isReady()).toBe(true); }); it('should handle initialization errors', async () => { // First model (embedding) loads successfully, second model (NLI) fails const mockEmbeddingModel = jest.fn(); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); mockPipeline.mockRejectedValueOnce(new Error('Model loading failed')); await expect(analyzer.initialize()).rejects.toThrow('Model loading failed'); expect(analyzer.isReady()).toBe(false); }); it('should not reinitialize if already initialized', async () => { const mockEmbeddingModel = jest.fn(); const mockNLIModel = jest.fn(); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); mockPipeline.mockResolvedValueOnce(mockNLIModel); await analyzer.initialize(); const firstCallCount = mockPipeline.mock.calls.length; // Try to initialize again await analyzer.initialize(); // Should not have made additional pipeline calls expect(mockPipeline.mock.calls.length).toBe(firstCallCount); }); }); describe('analyzeTextPair', () => { let mockNLIModel: jest.MockedFunction; beforeEach(async () => { const mockEmbeddingModel = jest.fn(); mockNLIModel = jest.fn(); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); mockPipeline.mockResolvedValueOnce(mockNLIModel); await analyzer.initialize(); // Store the mock in the analyzer instance so all tests can access it (analyzer as any).nliClassifier = mockNLIModel; }); it('should analyze text pairs and return NLI results', async () => { const mockResult = { labels: ['ENTAILMENT', 'NEUTRAL', 'CONTRADICTION'], scores: [0.8, 0.15, 0.05], }; mockNLIModel.mockResolvedValueOnce(mockResult); const result = await analyzer.analyzeTextPair( 'The user clicked a button', 'Button was clicked' ); expect(result).toEqual({ label: 'ENTAILMENT', score: 0.8, confidence: 0.8, }); }); it('should handle array results from NLI model', async () => { const mockResult = [ { labels: ['CONTRADICTION', 'NEUTRAL', 'ENTAILMENT'], scores: [0.7, 0.2, 0.1], }, ]; mockNLIModel.mockResolvedValueOnce(mockResult); const result = await analyzer.analyzeTextPair('User failed', 'Task succeeded'); expect(result).toEqual({ label: 'CONTRADICTION', score: 0.7, confidence: 0.7, }); }); it('should throw error if not initialized', async () => { const uninitializedAnalyzer = new SemanticAnalyzer(); await expect(uninitializedAnalyzer.analyzeTextPair('text1', 'text2')).rejects.toThrow( 'SemanticAnalyzer not initialized' ); }); it('should handle NLI model errors', async () => { mockNLIModel.mockRejectedValueOnce(new Error('NLI processing failed')); await expect(analyzer.analyzeTextPair('text1', 'text2')).rejects.toThrow( 'NLI processing failed' ); }); }); describe('assessActionOutcome', () => { let mockNLIModel: jest.MockedFunction; beforeEach(async () => { const mockEmbeddingModel = jest.fn(); mockNLIModel = jest.fn(); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); mockPipeline.mockResolvedValueOnce(mockNLIModel); await analyzer.initialize(); // Store the mock in the analyzer instance so all tests can access it (analyzer as any).nliClassifier = mockNLIModel; }); it('should assess successful action outcomes', async () => { const mockResult = { labels: ['ENTAILMENT', 'NEUTRAL', 'CONTRADICTION'], scores: [0.8, 0.15, 0.05], }; mockNLIModel.mockResolvedValueOnce(mockResult); const result = await analyzer.assessActionOutcome('Click button', 'Button was clicked'); expect(result).toEqual({ category: 'success', confidence: 0.8, reasoning: expect.stringContaining('aligns with expected outcome'), }); }); it('should assess failed action outcomes', async () => { const mockResult = { labels: ['CONTRADICTION', 'NEUTRAL', 'ENTAILMENT'], scores: [0.85, 0.1, 0.05], }; mockNLIModel.mockResolvedValueOnce(mockResult); const result = await analyzer.assessActionOutcome('Click button', 'Button was not found'); expect(result).toEqual({ category: 'failure', confidence: 0.85, reasoning: expect.stringContaining('contradicts expected outcome'), }); }); it('should handle neutral outcomes', async () => { const mockResult = { labels: ['NEUTRAL', 'ENTAILMENT', 'CONTRADICTION'], scores: [0.6, 0.25, 0.15], }; mockNLIModel.mockResolvedValueOnce(mockResult); const result = await analyzer.assessActionOutcome('Click button', 'Something happened'); expect(result).toEqual({ category: 'neutral', confidence: 0.6, reasoning: expect.stringContaining('Uncertain relationship'), }); }); }); describe('calculateSemanticSimilarity', () => { beforeEach(async () => { const mockEmbeddingModel = jest.fn(); mockEmbeddingModel.mockImplementation((text) => ({ data: new Array(384).fill(0.5), })); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); await analyzer.initialize(); }); it('should calculate semantic similarity using embeddings', async () => { const result = await analyzer.calculateSemanticSimilarity('Hello world', 'Hi there'); expect(result).toEqual({ similarity: expect.any(Number), confidence: 1, reasoning: 'Fast embedding-based semantic similarity', }); expect(result.similarity).toBeGreaterThanOrEqual(0); expect(result.similarity).toBeLessThanOrEqual(1); }); it('should throw error if not initialized', async () => { const uninitializedAnalyzer = new SemanticAnalyzer(); await expect( uninitializedAnalyzer.calculateSemanticSimilarity('text1', 'text2') ).rejects.toThrow('SemanticAnalyzer not initialized'); }); it('should handle embedding errors', async () => { const mockEmbeddingModel = jest.fn(); mockEmbeddingModel.mockRejectedValueOnce(new Error('Embedding failed')); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); const failingAnalyzer = new SemanticAnalyzer(); await failingAnalyzer.initialize(); await expect(failingAnalyzer.calculateSemanticSimilarity('text1', 'text2')).rejects.toThrow( 'Embedding failed' ); }); }); describe('cache management and LRU eviction', () => { let mockEmbeddingModel: jest.MockedFunction; beforeEach(async () => { mockEmbeddingModel = jest.fn(); mockEmbeddingModel.mockImplementation((text) => ({ data: new Array(384).fill(Math.random()), dims: [1, 384], })); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); await analyzer.initialize(); }); it('should evict oldest entries when cache exceeds maxCacheSize', async () => { // Generate 501 texts to exceed the maxCacheSize of 500 const texts = []; for (let i = 0; i < 501; i++) { texts.push(`text_${i}`); } // Process all texts to fill cache beyond limit await analyzer.getBatchEmbeddings(texts); // Process again with new texts - should require model calls mockEmbeddingModel.mockClear(); // Test with completely new entries (should require model calls) await analyzer.getBatchEmbeddings(['new_text_1', 'new_text_2', 'new_text_3']); // Should have made calls for new texts expect(mockEmbeddingModel).toHaveBeenCalled(); }); it('should handle cache key deletion during LRU eviction', async () => { // This test specifically targets the LRU eviction logic const manyTexts = []; for (let i = 0; i < 502; i++) { manyTexts.push(`cache_test_${i}`); } // This should trigger LRU eviction multiple times await analyzer.getBatchEmbeddings(manyTexts); // Verify embeddings were still generated expect(mockEmbeddingModel).toHaveBeenCalledTimes(502); }); }); describe('getBatchEmbeddings', () => { let mockEmbeddingModel: jest.MockedFunction; beforeEach(async () => { mockEmbeddingModel = jest.fn(); mockEmbeddingModel.mockImplementation((text) => ({ data: new Array(384).fill(Math.random()), dims: [1, 384], })); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); await analyzer.initialize(); }); it('should process multiple texts and return embeddings', async () => { const texts = ['text1', 'text2', 'text3']; const embeddings = await analyzer.getBatchEmbeddings(texts); expect(embeddings).toHaveLength(3); expect(embeddings[0]).toHaveLength(384); expect(mockEmbeddingModel).toHaveBeenCalledTimes(3); }); it('should use cache for previously processed texts', async () => { const texts = ['text1', 'text2']; // First call await analyzer.getBatchEmbeddings(texts); expect(mockEmbeddingModel).toHaveBeenCalledTimes(2); mockEmbeddingModel.mockClear(); // Second call with same texts - should use cache await analyzer.getBatchEmbeddings(texts); expect(mockEmbeddingModel).toHaveBeenCalledTimes(0); }); it('should handle invalid embedding dimensions', async () => { mockEmbeddingModel.mockImplementationOnce(() => ({ data: new Array(100).fill(0.5), // Wrong dimension dims: [1, 100], })); await expect(analyzer.getBatchEmbeddings(['test'])).rejects.toThrow( 'Invalid embedding dimension: 100. Expected 384' ); }); it('should handle missing embedding data', async () => { mockEmbeddingModel.mockImplementationOnce(() => ({ dims: [1, 384], // Missing data property })); await expect(analyzer.getBatchEmbeddings(['test'])).rejects.toThrow( 'Unable to process embedding for text: test' ); }); it('should throw error if embedding model not initialized', async () => { const uninitializedAnalyzer = new SemanticAnalyzer(); await expect(uninitializedAnalyzer.getBatchEmbeddings(['test'])).rejects.toThrow( 'Embedding model not initialized' ); }); }); describe('cosineSimilarity', () => { beforeEach(async () => { const mockEmbeddingModel = jest.fn(); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); await analyzer.initialize(); }); it('should calculate cosine similarity for identical vectors', async () => { const vec1 = [1, 0, 0]; const vec2 = [1, 0, 0]; // Access private method for testing const similarity = (analyzer as any).cosineSimilarity(vec1, vec2); expect(similarity).toBeCloseTo(1.0); }); it('should calculate cosine similarity for orthogonal vectors', async () => { const vec1 = [1, 0, 0]; const vec2 = [0, 1, 0]; const similarity = (analyzer as any).cosineSimilarity(vec1, vec2); expect(similarity).toBeCloseTo(0.0); }); it('should handle zero magnitude vectors', async () => { const vec1 = [0, 0, 0]; const vec2 = [1, 2, 3]; const similarity = (analyzer as any).cosineSimilarity(vec1, vec2); expect(similarity).toBe(0); }); it('should throw error for mismatched vector dimensions', async () => { const vec1 = [1, 2, 3]; const vec2 = [1, 2]; // Different length expect(() => { (analyzer as any).cosineSimilarity(vec1, vec2); }).toThrow('Vector dimension mismatch'); }); }); describe('computeSimilarityMatrix', () => { beforeEach(async () => { const mockEmbeddingModel = jest.fn(); mockEmbeddingModel.mockImplementation((text) => ({ data: new Array(384).fill(Math.random()), dims: [1, 384], })); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); await analyzer.initialize(); }); it('should compute similarity matrix for multiple texts', async () => { const texts = ['text1', 'text2', 'text3']; const matrix = await analyzer.computeSimilarityMatrix(texts); expect(matrix).toHaveLength(3); expect(matrix[0]).toHaveLength(3); // Diagonal should be 1.0 (self-similarity) expect(matrix[0][0]).toBe(1.0); expect(matrix[1][1]).toBe(1.0); expect(matrix[2][2]).toBe(1.0); // Matrix should be symmetric expect(matrix[0][1]).toBe(matrix[1][0]); expect(matrix[0][2]).toBe(matrix[2][0]); expect(matrix[1][2]).toBe(matrix[2][1]); }); }); describe('extractSemanticFeatures', () => { let mockNLIModel: jest.MockedFunction; beforeEach(async () => { const mockEmbeddingModel = jest.fn(); mockEmbeddingModel.mockImplementation((text, options) => ({ data: new Array(384).fill(Math.random()), dims: [1, 384], })); mockNLIModel = jest.fn(); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); mockPipeline.mockResolvedValueOnce(mockNLIModel); await analyzer.initialize(); // Store the mock in the analyzer instance so all tests can access it (analyzer as any).nliClassifier = mockNLIModel; }); it('should extract semantic features with custom intents', async () => { // Mock intent classification - this method calls analyzeTextPair which we need to mock const mockIntentResult = { labels: ['ENTAILMENT', 'NEUTRAL', 'CONTRADICTION'], scores: [0.8, 0.15, 0.05], }; // Mock sentiment analysis const mockSentimentResult = { labels: ['positive outcome', 'neutral outcome', 'negative outcome'], scores: [0.7, 0.2, 0.1], }; // First call for intent analysis (via analyzeTextPair) mockNLIModel .mockResolvedValueOnce(mockIntentResult) // Second call for sentiment analysis .mockResolvedValueOnce(mockSentimentResult); const customIntents = ['navigating', 'clicking']; const result = await analyzer.extractSemanticFeatures('Click button', customIntents); // The method should return results, but may not match exactly due to complex logic expect(result).toBeDefined(); expect(result.intents).toBeDefined(); expect(result.sentiment).toBeDefined(); expect(typeof result.confidence).toBe('number'); }); it('should handle negative sentiment', async () => { const mockNLIModel = jest.fn(); const mockIntentResult = { labels: ['ENTAILMENT', 'NEUTRAL', 'CONTRADICTION'], scores: [0.8, 0.15, 0.05], }; const mockSentimentResult = { labels: ['negative outcome', 'neutral outcome', 'positive outcome'], scores: [0.8, 0.15, 0.05], }; mockNLIModel .mockResolvedValueOnce(mockIntentResult) .mockResolvedValueOnce(mockSentimentResult); mockPipeline.mockResolvedValueOnce(mockNLIModel); const result = await analyzer.extractSemanticFeatures('Failed to load page'); // The method processes sentiment but may return default values in error cases expect(result).toBeDefined(); expect(['positive', 'negative', 'neutral']).toContain(result.sentiment); }); it('should handle errors gracefully', async () => { const mockNLIModel = jest.fn(); mockNLIModel.mockRejectedValueOnce(new Error('Feature extraction failed')); mockPipeline.mockResolvedValueOnce(mockNLIModel); const result = await analyzer.extractSemanticFeatures('test text'); expect(result).toEqual({ intents: [], sentiment: 'neutral', confidence: 0, }); }); it('should handle sentiment analysis edge cases', async () => { const mockNLIModel = jest.fn(); // Test with different sentiment outcomes const mockIntentResult = { labels: ['NEUTRAL', 'ENTAILMENT', 'CONTRADICTION'], scores: [0.6, 0.3, 0.1], }; const mockSentimentResult = { labels: ['neutral outcome', 'positive outcome', 'negative outcome'], scores: [0.8, 0.15, 0.05], }; mockNLIModel .mockResolvedValueOnce(mockIntentResult) .mockResolvedValueOnce(mockSentimentResult); mockPipeline.mockResolvedValueOnce(mockNLIModel); const result = await analyzer.extractSemanticFeatures('neutral statement'); expect(result.sentiment).toBe('neutral'); expect(typeof result.confidence).toBe('number'); }); it('should throw error if not initialized', async () => { const uninitializedAnalyzer = new SemanticAnalyzer(); await expect(uninitializedAnalyzer.extractSemanticFeatures('test')).rejects.toThrow( 'SemanticAnalyzer not initialized' ); }); }); describe('classifyActionIntent', () => { let mockNLIModel: jest.MockedFunction; beforeEach(async () => { const mockEmbeddingModel = jest.fn(); mockNLIModel = jest.fn(); mockPipeline.mockResolvedValueOnce(mockEmbeddingModel); mockPipeline.mockResolvedValueOnce(mockNLIModel); await analyzer.initialize(); // Store the mock in the analyzer instance so all tests can access it (analyzer as any).nliClassifier = mockNLIModel; }); it('should classify action intents and rank them', async () => { // Clear previous mock calls but keep the mock function mockNLIModel.mockClear(); // Mock different responses for different intents mockNLIModel .mockResolvedValueOnce({ labels: ['ENTAILMENT', 'NEUTRAL', 'CONTRADICTION'], scores: [0.9, 0.08, 0.02], }) .mockResolvedValueOnce({ labels: ['NEUTRAL', 'ENTAILMENT', 'CONTRADICTION'], scores: [0.6, 0.3, 0.1], }) .mockResolvedValueOnce({ labels: ['CONTRADICTION', 'NEUTRAL', 'ENTAILMENT'], scores: [0.8, 0.15, 0.05], }); const intents = ['clicking', 'scrolling', 'typing']; const result = await analyzer.classifyActionIntent('click button', intents); expect(result.bestMatch).toBe('clicking'); expect(result.confidence).toBeGreaterThan(0.8); // ENTAILMENT with high score expect(result.allScores).toHaveLength(3); expect(result.allScores[0].score).toBeGreaterThanOrEqual(result.allScores[1].score); }); it('should handle neutral classifications', async () => { // Clear previous mock calls but keep the mock function mockNLIModel.mockClear(); mockNLIModel.mockResolvedValue({ labels: ['NEUTRAL', 'ENTAILMENT', 'CONTRADICTION'], scores: [0.7, 0.2, 0.1], }); const result = await analyzer.classifyActionIntent('ambiguous action', ['intent1']); expect(result.confidence).toBeCloseTo(0.35); // 0.7 * 0.5 for neutral }); it('should handle contradiction classifications', async () => { // Clear previous mock calls but keep the mock function mockNLIModel.mockClear(); mockNLIModel.mockResolvedValue({ labels: ['CONTRADICTION', 'NEUTRAL', 'ENTAILMENT'], scores: [0.8, 0.15, 0.05], }); const result = await analyzer.classifyActionIntent('opposite action', ['intent1']); expect(result.confidence).toBeCloseTo(0.08); // 0.8 * 0.1 for contradiction }); }); });