{"version":3,"sources":["../src/worker.ts","../src/tokenizer.ts"],"sourcesContent":["import { parentPort } from 'worker_threads';\r\nimport { Tokenizer } from './tokenizer';\r\nimport type { Document, BM25Options, FieldBoosts } from './types';\r\n\r\nfunction processDocuments(\r\n docs: Document[],\r\n options: BM25Options & { fieldBoosts?: FieldBoosts },\r\n) {\r\n const tokenizer = new Tokenizer(options);\r\n const documentLengths = new Uint32Array(docs.length);\r\n const termToIndex = new Map();\r\n const termDocs = new Map>();\r\n const termFrequencies = new Map>();\r\n let totalLength = 0;\r\n let nextTermIndex = 0;\r\n\r\n docs.forEach((doc, docIndex) => {\r\n let docLength = 0;\r\n Object.entries(doc).forEach(([field, content]) => {\r\n const { tokens } = tokenizer.tokenize(content);\r\n const fieldBoost = options.fieldBoosts?.[field] || 1;\r\n docLength += tokens.length * fieldBoost;\r\n\r\n const uniqueTerms = new Set(tokens);\r\n uniqueTerms.forEach((term) => {\r\n if (!termToIndex.has(term)) {\r\n termToIndex.set(term, nextTermIndex++);\r\n }\r\n const termIndex = termToIndex.get(term)!;\r\n\r\n if (!termDocs.has(term)) {\r\n termDocs.set(term, new Set());\r\n }\r\n termDocs.get(term)!.add(docIndex);\r\n\r\n if (!termFrequencies.has(termIndex)) {\r\n termFrequencies.set(termIndex, new Map());\r\n }\r\n const freq = tokens.filter((t) => t === term).length * fieldBoost;\r\n const existingFreq = termFrequencies.get(termIndex)!.get(docIndex) || 0;\r\n termFrequencies.get(termIndex)!.set(docIndex, existingFreq + freq);\r\n });\r\n });\r\n\r\n documentLengths[docIndex] = docLength;\r\n totalLength += docLength;\r\n });\r\n\r\n const documentFrequency = new Uint32Array(termToIndex.size);\r\n termDocs.forEach((docs, term) => {\r\n const termIndex = termToIndex.get(term)!;\r\n documentFrequency[termIndex] = docs.size;\r\n });\r\n\r\n // Convert Maps to serializable format\r\n const serializedTermToIndex = Array.from(termToIndex.entries());\r\n const serializedTermFrequencies = Array.from(termFrequencies.entries()).map(\r\n ([termIndex, docFreqs]) => [termIndex, Array.from(docFreqs.entries())],\r\n );\r\n\r\n // Modify return format for better serialization\r\n return {\r\n documentLengths: Array.from(documentLengths),\r\n termToIndex: serializedTermToIndex,\r\n documentFrequency: Array.from(documentFrequency),\r\n averageDocLength: totalLength / docs.length,\r\n termFrequencies: serializedTermFrequencies,\r\n documentCount: docs.length,\r\n };\r\n}\r\n\r\nparentPort?.on('message', ({ docs, options }) => {\r\n const result = processDocuments(docs, options);\r\n parentPort?.postMessage(result);\r\n});\r\n","import { stem } from 'porter2';\r\nimport type {\r\n TokenizerOptions,\r\n TokenizationResult,\r\n TokenizationStats,\r\n StemmingRule,\r\n} from './types';\r\n\r\n/**\r\n * Text tokenization with configurable filters and stemming\r\n */\r\nexport class Tokenizer {\r\n private readonly stopWords: Set;\r\n private readonly minLength: number;\r\n private readonly stemming: boolean;\r\n private readonly stemmingRules: StemmingRule[];\r\n\r\n private static readonly DEFAULT_OPTIONS: Required = {\r\n stopWords: new Set(),\r\n minLength: 2,\r\n stemming: false,\r\n stemmingRules: [],\r\n };\r\n\r\n /**\r\n * Creates a new tokenizer instance\r\n * @param options - Tokenization options including stop words and stemming\r\n */\r\n constructor(options: TokenizerOptions = {}) {\r\n const mergedOptions = { ...Tokenizer.DEFAULT_OPTIONS, ...options };\r\n this.stopWords = mergedOptions.stopWords;\r\n this.minLength = mergedOptions.minLength;\r\n this.stemming = mergedOptions.stemming;\r\n this.stemmingRules = mergedOptions.stemmingRules.map((rule) => ({\r\n ...rule,\r\n pattern:\r\n typeof rule.pattern === 'string'\r\n ? new RegExp(rule.pattern)\r\n : rule.pattern,\r\n }));\r\n }\r\n /**\r\n * Tokenize text into an array of terms with optional statistics\r\n * @param text - Input text to tokenize\r\n * @param includeStats - Whether to include tokenization statistics\r\n * @returns TokenizationResult containing tokens and optional stats\r\n * @throws {Error} If input text is null or empty\r\n */\r\n public tokenize(text: string, includeStats = false): TokenizationResult {\r\n if (!text) {\r\n throw new Error('Input text cannot be null or empty');\r\n }\r\n\r\n const startTime = Date.now();\r\n const originalWords = text.split(/\\s+/).filter((word) => word.length > 0);\r\n\r\n const cleaned = this.cleanText(text);\r\n const tokens = cleaned\r\n .split(/\\s+/)\r\n .filter((token) => this.isValidToken(token))\r\n .map((token) => (this.stemming ? this.stemWord(token) : token));\r\n\r\n const stats: TokenizationStats = includeStats\r\n ? {\r\n originalWordCount: originalWords.length,\r\n stopWordsRemoved: originalWords.length - tokens.length,\r\n stemmedWords: this.stemming ? tokens.length : 0,\r\n processingTimeMs: Date.now() - startTime,\r\n }\r\n : {\r\n originalWordCount: 0,\r\n stopWordsRemoved: 0,\r\n stemmedWords: 0,\r\n processingTimeMs: 0,\r\n };\r\n\r\n return { tokens, stats };\r\n }\r\n\r\n /**\r\n * Cleans and normalizes text by removing unwanted characters while preserving meaningful content.\r\n * Handles Unicode, emojis, symbols, accents, and multiple writing systems automatically.\r\n *\r\n * @param text - Input text to clean\r\n * @returns Cleaned and normalized text\r\n *\r\n * @example\r\n * cleanText(\"Hello, World™!\") // \"hello world\"\r\n * cleanText(\"héllo 👋\") // \"hello\"\r\n * cleanText(\"Hello 世界!\") // \"hello 世界\"\r\n * cleanText(\"I'm don't\") // \"i'm don't\"\r\n * cleanText(\"test©2023\") // \"test 2023\"\r\n */\r\n private cleanText(text: string): string {\r\n return (\r\n text\r\n .toLowerCase()\r\n // Normalize Unicode characters to their canonical form\r\n .normalize('NFKD')\r\n // Remove control characters and zero-width characters\r\n .replace(/[\\u0000-\\u001F\\u007F-\\u009F\\u200B-\\u200D\\uFEFF]/g, '')\r\n // Remove combining diacritical marks\r\n .replace(/[\\u0300-\\u036f]/g, '')\r\n // Remove emojis and symbols while preserving basic punctuation\r\n // .replace(\r\n // /[\\p{Extended_Pictographic}\\p{Emoji}\\p{Emoji_Component}\\p{Symbol}\\p{So}]/gu,\r\n // '',\r\n // )\r\n .replace(/[\\p{Emoji_Presentation}\\p{Extended_Pictographic}]/gu, '')\r\n // Remove trademark, copyright, and similar symbols\r\n .replace(/[™®©℠‼]/g, '')\r\n // Replace punctuation with space\r\n .replace(/[\\p{P}]/gu, ' ')\r\n // Keep alphanumeric, CJK, Hangul intact and replace others with space\r\n .replace(\r\n /[^a-z0-9\\u3040-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uAC00-\\uD7AF\\s]/gu,\r\n ' ',\r\n )\r\n // Normalize whitespace: collapse multiple spaces and trim\r\n .replace(/\\s+/g, ' ')\r\n .trim()\r\n );\r\n }\r\n\r\n /**\r\n * Check if a token is valid based on length and stop words\r\n * @param token - Token to validate\r\n * @returns Whether the token is valid\r\n */\r\n private isValidToken(token: string): boolean {\r\n const isNumeric = /^\\d+$/.test(token);\r\n return (\r\n (token.length >= this.minLength || isNumeric) &&\r\n !this.stopWords.has(token)\r\n );\r\n }\r\n\r\n /**\r\n * Apply stemming rules to a word\r\n * @param word - Word to stem\r\n * @returns Stemmed word\r\n */\r\n private stemWord(word: string): string {\r\n if (word.length < 3) return word;\r\n let customRule = false;\r\n\r\n let stemmed = word;\r\n\r\n // Apply custom stemming rules first\r\n for (const rule of this.stemmingRules) {\r\n const match = stemmed.match(rule.pattern);\r\n if (match) {\r\n customRule = true;\r\n if (!rule.minMeasure || this.measure(stemmed) >= rule.minMeasure) {\r\n if (typeof rule.replacement === 'string') {\r\n // Handle string replacement\r\n stemmed = stemmed.replace(rule.pattern, rule.replacement);\r\n } else {\r\n // Handle function replacement\r\n stemmed = stemmed.replace(rule.pattern, rule.replacement);\r\n }\r\n }\r\n }\r\n }\r\n\r\n // Skip default stemming rules if a custom rule matched\r\n if (customRule) return stemmed;\r\n\r\n stemmed = stem(stemmed);\r\n\r\n return stemmed;\r\n }\r\n\r\n private isConsonant(word: string, i: number): boolean {\r\n const char = word[i];\r\n if ('aeiou'.includes(char)) return false;\r\n return char !== 'y' || (i === 0 ? true : !this.isConsonant(word, i - 1));\r\n }\r\n\r\n private measure(word: string): number {\r\n let m = 0;\r\n let vowelSeen = false;\r\n for (let i = 0; i < word.length; i++) {\r\n if (this.isConsonant(word, i)) {\r\n if (vowelSeen) {\r\n m++;\r\n vowelSeen = false;\r\n }\r\n } else {\r\n vowelSeen = true;\r\n }\r\n }\r\n return m;\r\n }\r\n}\r\n"],"mappings":";;;AAAA,4BAA2B;;;ACA3B,qBAAqB;AAWd,IAAM,YAAN,MAAM,WAAU;AAAA,EACJ;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,OAAwB,kBAA8C;AAAA,IACpE,WAAW,oBAAI,IAAY;AAAA,IAC3B,WAAW;AAAA,IACX,UAAU;AAAA,IACV,eAAe,CAAC;AAAA,EAClB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,YAAY,UAA4B,CAAC,GAAG;AAC1C,UAAM,gBAAgB,EAAE,GAAG,WAAU,iBAAiB,GAAG,QAAQ;AACjE,SAAK,YAAY,cAAc;AAC/B,SAAK,YAAY,cAAc;AAC/B,SAAK,WAAW,cAAc;AAC9B,SAAK,gBAAgB,cAAc,cAAc,IAAI,CAAC,UAAU;AAAA,MAC9D,GAAG;AAAA,MACH,SACE,OAAO,KAAK,YAAY,WACpB,IAAI,OAAO,KAAK,OAAO,IACvB,KAAK;AAAA,IACb,EAAE;AAAA,EACJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQO,SAAS,MAAc,eAAe,OAA2B;AACtE,QAAI,CAAC,MAAM;AACT,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AAEA,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,gBAAgB,KAAK,MAAM,KAAK,EAAE,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC;AAExE,UAAM,UAAU,KAAK,UAAU,IAAI;AACnC,UAAM,SAAS,QACZ,MAAM,KAAK,EACX,OAAO,CAAC,UAAU,KAAK,aAAa,KAAK,CAAC,EAC1C,IAAI,CAAC,UAAW,KAAK,WAAW,KAAK,SAAS,KAAK,IAAI,KAAM;AAEhE,UAAM,QAA2B,eAC7B;AAAA,MACE,mBAAmB,cAAc;AAAA,MACjC,kBAAkB,cAAc,SAAS,OAAO;AAAA,MAChD,cAAc,KAAK,WAAW,OAAO,SAAS;AAAA,MAC9C,kBAAkB,KAAK,IAAI,IAAI;AAAA,IACjC,IACA;AAAA,MACE,mBAAmB;AAAA,MACnB,kBAAkB;AAAA,MAClB,cAAc;AAAA,MACd,kBAAkB;AAAA,IACpB;AAEJ,WAAO,EAAE,QAAQ,MAAM;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAgBQ,UAAU,MAAsB;AACtC,WACE,KACG,YAAY,EAEZ,UAAU,MAAM,EAEhB,QAAQ,oDAAoD,EAAE,EAE9D,QAAQ,oBAAoB,EAAE,EAM9B,QAAQ,uDAAuD,EAAE,EAEjE,QAAQ,YAAY,EAAE,EAEtB,QAAQ,aAAa,GAAG,EAExB;AAAA,MACC;AAAA,MACA;AAAA,IACF,EAEC,QAAQ,QAAQ,GAAG,EACnB,KAAK;AAAA,EAEZ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOQ,aAAa,OAAwB;AAC3C,UAAM,YAAY,QAAQ,KAAK,KAAK;AACpC,YACG,MAAM,UAAU,KAAK,aAAa,cACnC,CAAC,KAAK,UAAU,IAAI,KAAK;AAAA,EAE7B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOQ,SAAS,MAAsB;AACrC,QAAI,KAAK,SAAS,EAAG,QAAO;AAC5B,QAAI,aAAa;AAEjB,QAAI,UAAU;AAGd,eAAW,QAAQ,KAAK,eAAe;AACrC,YAAM,QAAQ,QAAQ,MAAM,KAAK,OAAO;AACxC,UAAI,OAAO;AACT,qBAAa;AACb,YAAI,CAAC,KAAK,cAAc,KAAK,QAAQ,OAAO,KAAK,KAAK,YAAY;AAChE,cAAI,OAAO,KAAK,gBAAgB,UAAU;AAExC,sBAAU,QAAQ,QAAQ,KAAK,SAAS,KAAK,WAAW;AAAA,UAC1D,OAAO;AAEL,sBAAU,QAAQ,QAAQ,KAAK,SAAS,KAAK,WAAW;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,QAAI,WAAY,QAAO;AAEvB,kBAAU,qBAAK,OAAO;AAEtB,WAAO;AAAA,EACT;AAAA,EAEQ,YAAY,MAAc,GAAoB;AACpD,UAAM,OAAO,KAAK,CAAC;AACnB,QAAI,QAAQ,SAAS,IAAI,EAAG,QAAO;AACnC,WAAO,SAAS,QAAQ,MAAM,IAAI,OAAO,CAAC,KAAK,YAAY,MAAM,IAAI,CAAC;AAAA,EACxE;AAAA,EAEQ,QAAQ,MAAsB;AACpC,QAAI,IAAI;AACR,QAAI,YAAY;AAChB,aAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAI,KAAK,YAAY,MAAM,CAAC,GAAG;AAC7B,YAAI,WAAW;AACb;AACA,sBAAY;AAAA,QACd;AAAA,MACF,OAAO;AACL,oBAAY;AAAA,MACd;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;;;AD9LA,SAAS,iBACP,MACA,SACA;AACA,QAAM,YAAY,IAAI,UAAU,OAAO;AACvC,QAAM,kBAAkB,IAAI,YAAY,KAAK,MAAM;AACnD,QAAM,cAAc,oBAAI,IAAoB;AAC5C,QAAM,WAAW,oBAAI,IAAyB;AAC9C,QAAM,kBAAkB,oBAAI,IAAiC;AAC7D,MAAI,cAAc;AAClB,MAAI,gBAAgB;AAEpB,OAAK,QAAQ,CAAC,KAAK,aAAa;AAC9B,QAAI,YAAY;AAChB,WAAO,QAAQ,GAAG,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,MAAM;AAChD,YAAM,EAAE,OAAO,IAAI,UAAU,SAAS,OAAO;AAC7C,YAAM,aAAa,QAAQ,cAAc,KAAK,KAAK;AACnD,mBAAa,OAAO,SAAS;AAE7B,YAAM,cAAc,IAAI,IAAI,MAAM;AAClC,kBAAY,QAAQ,CAAC,SAAS;AAC5B,YAAI,CAAC,YAAY,IAAI,IAAI,GAAG;AAC1B,sBAAY,IAAI,MAAM,eAAe;AAAA,QACvC;AACA,cAAM,YAAY,YAAY,IAAI,IAAI;AAEtC,YAAI,CAAC,SAAS,IAAI,IAAI,GAAG;AACvB,mBAAS,IAAI,MAAM,oBAAI,IAAI,CAAC;AAAA,QAC9B;AACA,iBAAS,IAAI,IAAI,EAAG,IAAI,QAAQ;AAEhC,YAAI,CAAC,gBAAgB,IAAI,SAAS,GAAG;AACnC,0BAAgB,IAAI,WAAW,oBAAI,IAAI,CAAC;AAAA,QAC1C;AACA,cAAM,OAAO,OAAO,OAAO,CAAC,MAAM,MAAM,IAAI,EAAE,SAAS;AACvD,cAAM,eAAe,gBAAgB,IAAI,SAAS,EAAG,IAAI,QAAQ,KAAK;AACtE,wBAAgB,IAAI,SAAS,EAAG,IAAI,UAAU,eAAe,IAAI;AAAA,MACnE,CAAC;AAAA,IACH,CAAC;AAED,oBAAgB,QAAQ,IAAI;AAC5B,mBAAe;AAAA,EACjB,CAAC;AAED,QAAM,oBAAoB,IAAI,YAAY,YAAY,IAAI;AAC1D,WAAS,QAAQ,CAACA,OAAM,SAAS;AAC/B,UAAM,YAAY,YAAY,IAAI,IAAI;AACtC,sBAAkB,SAAS,IAAIA,MAAK;AAAA,EACtC,CAAC;AAGD,QAAM,wBAAwB,MAAM,KAAK,YAAY,QAAQ,CAAC;AAC9D,QAAM,4BAA4B,MAAM,KAAK,gBAAgB,QAAQ,CAAC,EAAE;AAAA,IACtE,CAAC,CAAC,WAAW,QAAQ,MAAM,CAAC,WAAW,MAAM,KAAK,SAAS,QAAQ,CAAC,CAAC;AAAA,EACvE;AAGA,SAAO;AAAA,IACL,iBAAiB,MAAM,KAAK,eAAe;AAAA,IAC3C,aAAa;AAAA,IACb,mBAAmB,MAAM,KAAK,iBAAiB;AAAA,IAC/C,kBAAkB,cAAc,KAAK;AAAA,IACrC,iBAAiB;AAAA,IACjB,eAAe,KAAK;AAAA,EACtB;AACF;AAEA,kCAAY,GAAG,WAAW,CAAC,EAAE,MAAM,QAAQ,MAAM;AAC/C,QAAM,SAAS,iBAAiB,MAAM,OAAO;AAC7C,oCAAY,YAAY,MAAM;AAChC,CAAC;","names":["docs"]}