import { ASTNode, TagNode } from '../types'; import { LLMXMLError } from '../errors'; import logger from '../utils/logger'; import { ERROR_MESSAGES } from '../constants'; interface ExtractOptions { /** Title to search for */ title: string; /** Optional heading level to match */ level?: number; /** Fuzzy match threshold (0-1) */ threshold?: number; /** Whether to include subsections */ includeSubsections?: boolean; /** Whether to return fuzzy matches */ fuzzyMatches?: boolean; } interface MatchScore { /** Whether this is a case-insensitive exact match */ exactMatch: boolean; /** Fuzzy match score (0-1) */ fuzzyScore: number; /** Whether this title appears in a higher level section */ isContainedInHigherLevel: boolean; /** Whether this section has more specific child sections */ hasMoreSpecificChild: boolean; /** Heading level (1-6) */ level: number; /** Position in document (0-based) */ position: number; /** Score adjusted based on context */ contextualScore: number; } interface Match { /** The matched node */ node: TagNode; /** Detailed scoring information */ score: MatchScore; /** Parent node if any */ parent?: TagNode; } /** * Results from heading similarity calculation */ export interface MatchResult { /** Title of the heading */ title: string; /** Heading level (1-6) */ level: number; /** Match similarity score (0-1) */ similarity: number; } /** * Extracts sections from AST based on fuzzy title matching */ export class SectionExtractor { /** * Find sections matching the given criteria * * @param ast - The AST to search * @param options - Search options * @returns Matching sections * @throws {LLMXMLError} If no matches found */ public findSections(ast: ASTNode[], options: ExtractOptions): TagNode[] { const matches: Match[] = []; const threshold = options.threshold || 0.7; // First pass: collect all potential matches with their context this.findMatches(ast, options, matches); if (matches.length === 0) { throw new LLMXMLError( ERROR_MESSAGES.SECTION_NOT_FOUND, 'SECTION_NOT_FOUND', { title: options.title } ); } // Second pass: analyze relationships and adjust scores this.analyzeRelationships(matches); // Filter by threshold and sort by final score const validMatches = matches .filter(m => m.score.contextualScore >= threshold) .sort((a, b) => { // First by exact match if (a.score.exactMatch !== b.score.exactMatch) { return b.score.exactMatch ? 1 : -1; } // Then by contextual score const scoreDiff = b.score.contextualScore - a.score.contextualScore; if (scoreDiff !== 0) return scoreDiff; // Then by level (prefer more specific) const levelDiff = a.score.level - b.score.level; if (levelDiff !== 0) return levelDiff; // Finally by position return a.score.position - b.score.position; }); if (validMatches.length === 0) { throw new LLMXMLError( ERROR_MESSAGES.SECTION_NOT_FOUND, 'SECTION_NOT_FOUND', { title: options.title } ); } // Check for ambiguous matches if (validMatches.length > 1 && !validMatches[0].score.exactMatch && Math.abs(validMatches[0].score.contextualScore - validMatches[1].score.contextualScore) < 0.1) { logger.warn('Ambiguous match', { title: options.title, matches: validMatches.map(m => ({ title: m.node.attributes.title, score: m.score, })), code: 'AMBIGUOUS_MATCH', }); } // Return only the best match if: // 1. Fuzzy matches are disabled // 2. We have an exact match // 3. The best match is significantly better than others // 4. The matches are not clearly related sections at the same level if (options.fuzzyMatches === false || validMatches[0].score.exactMatch || (validMatches.length > 1 && Math.abs(validMatches[0].score.contextualScore - validMatches[1].score.contextualScore) >= 0.2) || (validMatches.length > 1 && validMatches[0].score.level !== validMatches[1].score.level)) { return [validMatches[0].node]; } // For fuzzy matches with similar scores, return all matches at the same level const bestScore = validMatches[0].score.contextualScore; const bestLevel = validMatches[0].score.level; // If we have matches at different levels and a higher level contains the search term, // prefer the more specific (lower level) matches const hasHigherLevelMatch = validMatches.some(m => m.score.level < bestLevel && options.title.toLowerCase().split(/\s+/).some(word => m.node.attributes.title.toLowerCase().includes(word) ) ); const closeMatches = validMatches.filter(m => { // Must be within score threshold const scoreClose = Math.abs(m.score.contextualScore - bestScore) < 0.2; // If we have a higher level match containing the search term, // only include matches at the most specific level if (hasHigherLevelMatch) { return scoreClose && m.score.level === bestLevel; } // Otherwise include all close matches that contain the search term return scoreClose && options.title.toLowerCase().split(/\s+/).some(word => m.node.attributes.title.toLowerCase().includes(word) ); }); // Return all close matches, including subsections if requested const result = closeMatches.map(m => m.node); if (options.includeSubsections) { // For each match, include all its children closeMatches.forEach(match => { const collectChildren = (node: TagNode) => { if (!node.children) return; node.children.forEach(child => { if (child.type === 'tag') { result.push(child as TagNode); collectChildren(child as TagNode); } }); }; collectChildren(match.node); }); } return result; } /** * Recursively find matching sections */ private findMatches( nodes: ASTNode[], options: ExtractOptions, matches: Match[], parent?: TagNode, position: number = 0 ): number { let currentPosition = position; for (const node of nodes) { if (node.type === 'tag') { const tagNode = node as TagNode; const initialScore = this.calculateInitialScore(tagNode, options, currentPosition); if (initialScore.fuzzyScore > 0) { matches.push({ node: tagNode, score: initialScore, parent, }); } if (node.children) { currentPosition = this.findMatches( node.children, options, matches, tagNode, currentPosition + 1 ); } } currentPosition++; } return currentPosition; } /** * Calculate similarity score between section and search criteria */ private calculateInitialScore(node: TagNode, options: ExtractOptions, position: number): MatchScore { const nodeTitle = node.attributes.title.toLowerCase(); const searchTitle = options.title.toLowerCase(); const level = Number(node.attributes.hlevel); // Start with base score const score: MatchScore = { exactMatch: nodeTitle === searchTitle, fuzzyScore: 0, isContainedInHigherLevel: false, hasMoreSpecificChild: false, level, position, contextualScore: 0, }; // Exact match if (score.exactMatch) { score.fuzzyScore = 1.0; // Adjust if level constraint doesn't match if (options.level && level !== options.level) { score.fuzzyScore = 0.8; } } // Word boundary match (case insensitive) else if (nodeTitle.match(new RegExp(`\\b${searchTitle}\\b`, 'i'))) { score.fuzzyScore = 0.95; } // Contains match (case insensitive) else if (nodeTitle.includes(searchTitle)) { score.fuzzyScore = 0.85; } // Partial word matches else { const nodeWords = nodeTitle.split(/\s+/); const searchWords = searchTitle.split(/\s+/); // Calculate word-by-word match scores const wordScores = searchWords.map(searchWord => { let bestScore = 0; nodeWords.forEach(nodeWord => { // Exact word match if (nodeWord === searchWord) { bestScore = Math.max(bestScore, 1.0); } // Word boundary match else if (nodeWord.match(new RegExp(`\\b${searchWord}\\b`, 'i'))) { bestScore = Math.max(bestScore, 0.9); } // Starts with match else if (nodeWord.startsWith(searchWord) || searchWord.startsWith(nodeWord)) { bestScore = Math.max(bestScore, 0.8); } // Significant substring match else if (searchWord.length > 3 && nodeWord.includes(searchWord)) { bestScore = Math.max(bestScore, 0.7); } // Partial word match with high similarity else if (searchWord.length > 2 && ( nodeWord.includes(searchWord) || searchWord.includes(nodeWord) || this.calculateLevenshteinRatio(searchWord, nodeWord) > 0.8 )) { bestScore = Math.max(bestScore, 0.6); } }); return bestScore; }); // Calculate overall score based on word matches if (wordScores.length > 0) { // Average of word scores const avgScore = wordScores.reduce((sum, score) => sum + score, 0) / wordScores.length; // Boost if all words have some match const allWordsMatch = wordScores.every(score => score > 0); // Final fuzzy score score.fuzzyScore = avgScore * (allWordsMatch ? 1.2 : 1.0); // Cap at 0.95 to ensure exact matches still win score.fuzzyScore = Math.min(0.95, score.fuzzyScore); } } // Start with fuzzy score as contextual score score.contextualScore = score.fuzzyScore; return score; } private calculateLevenshteinRatio(s1: string, s2: string): number { const track = Array(s2.length + 1).fill(null).map(() => Array(s1.length + 1).fill(null)); for (let i = 0; i <= s1.length; i += 1) { track[0][i] = i; } for (let j = 0; j <= s2.length; j += 1) { track[j][0] = j; } for (let j = 1; j <= s2.length; j += 1) { for (let i = 1; i <= s1.length; i += 1) { const indicator = s1[i - 1] === s2[j - 1] ? 0 : 1; track[j][i] = Math.min( track[j][i - 1] + 1, // deletion track[j - 1][i] + 1, // insertion track[j - 1][i - 1] + indicator // substitution ); } } const distance = track[s2.length][s1.length]; const maxLength = Math.max(s1.length, s2.length); return 1 - distance / maxLength; } private analyzeRelationships(matches: Match[]): void { // First pass: mark containment relationships for (const match of matches) { const title = match.node.attributes.title.toLowerCase(); // Check if this title is contained in any higher level sections match.score.isContainedInHigherLevel = matches.some(other => other.score.level < match.score.level && other.node.attributes.title.toLowerCase().includes(title) ); // Check if this section has more specific children match.score.hasMoreSpecificChild = matches.some(other => other.score.level > match.score.level && other.parent === match.node ); } // Second pass: adjust contextual scores for (const match of matches) { let contextualScore = match.score.fuzzyScore; // Exact matches get highest priority if (match.score.exactMatch) { contextualScore = 1.0; } else { // For fuzzy matches, boost scores based on word boundaries and order const nodeTitle = match.node.attributes.title.toLowerCase(); const nodeWords = nodeTitle.split(/\s+/); const searchTitle = match.node.attributes.title.toLowerCase(); const searchWords = searchTitle.split(/\s+/); // Check for complete word matches in the right order const hasOrderedWordMatch = searchWords.some(word => { const wordIndex = nodeWords.indexOf(word); if (wordIndex === -1) return false; // Check if surrounding words also match in order const prevWord = searchWords[searchWords.indexOf(word) - 1]; const nextWord = searchWords[searchWords.indexOf(word) + 1]; return (!prevWord || nodeWords[wordIndex - 1] === prevWord) && (!nextWord || nodeWords[wordIndex + 1] === nextWord); }); if (hasOrderedWordMatch) { contextualScore *= 1.2; } // Boost more specific matches when they contain all search terms in order const searchTerms = searchTitle.split(/\s+/); let lastFoundIndex = -1; const containsAllTermsInOrder = searchTerms.every(term => { const index = nodeWords.findIndex((word, i) => i > lastFoundIndex && word.includes(term)); if (index === -1) return false; lastFoundIndex = index; return true; }); if (containsAllTermsInOrder && match.score.level > 1) { contextualScore *= 1.3; } // Penalize higher levels when same term appears in children if (matches.some(other => other.score.level > match.score.level && other.score.fuzzyScore >= match.score.fuzzyScore )) { contextualScore *= 0.8; } } match.score.contextualScore = Math.min(1, contextualScore); } } /** * Find closest matching headings for a search term * * @param searchTitle - Title to search for * @param headings - Available headings to search within * @param threshold - Minimum similarity threshold (0-1) * @returns Array of closest matches with similarity scores */ public findClosestMatches( searchTitle: string, headings: Array<{ title: string; level: number }>, threshold: number = 0.7 ): MatchResult[] { const searchTitleLower = searchTitle.toLowerCase(); const searchWords = searchTitleLower.split(/\s+/); // Calculate similarity for each heading const matches = headings.map(heading => { const titleLower = heading.title.toLowerCase(); let similarity = 0; // Exact match if (titleLower === searchTitleLower) { similarity = 1.0; } // Word boundary match else if (titleLower.match(new RegExp(`\\b${searchTitleLower}\\b`, 'i'))) { similarity = 0.95; } // Contains match else if (titleLower.includes(searchTitleLower)) { similarity = 0.85; } // Partial word matches else { const titleWords = titleLower.split(/\s+/); // Calculate word-by-word match scores const wordScores = searchWords.map(searchWord => { let bestScore = 0; titleWords.forEach(titleWord => { // Exact word match if (titleWord === searchWord) { bestScore = Math.max(bestScore, 1.0); } // Word boundary match else if (titleWord.match(new RegExp(`\\b${searchWord}\\b`, 'i'))) { bestScore = Math.max(bestScore, 0.9); } // Starts with match else if (titleWord.startsWith(searchWord) || searchWord.startsWith(titleWord)) { bestScore = Math.max(bestScore, 0.8); } // Significant substring match else if (searchWord.length > 3 && titleWord.includes(searchWord)) { bestScore = Math.max(bestScore, 0.7); } // Partial word match with high similarity else if (searchWord.length > 2 && ( titleWord.includes(searchWord) || searchWord.includes(titleWord) || this.calculateLevenshteinRatio(searchWord, titleWord) > 0.8 )) { bestScore = Math.max(bestScore, 0.6); } }); return bestScore; }); if (wordScores.length > 0) { // Average of word scores const avgScore = wordScores.reduce((sum, score) => sum + score, 0) / wordScores.length; // Boost if all words have some match const allWordsMatch = wordScores.every(score => score > 0); // Final similarity score similarity = avgScore * (allWordsMatch ? 1.2 : 1.0); // Cap at 0.95 to ensure exact matches still win similarity = Math.min(0.95, similarity); } } return { title: heading.title, level: heading.level, similarity }; }); // Filter by threshold and sort by similarity return matches .filter(match => match.similarity >= threshold) .sort((a, b) => b.similarity - a.similarity); } } // Export a singleton instance export const sectionExtractor = new SectionExtractor();