--!native --!optimize 2 -- Compiled with roblox-ts v3.0.0 local TS = _G[script] local GetKeys = TS.import(script, script.Parent.Parent, "Utils", "GetKeys") --[[ * * Creates a term frequency map { word: count } from tokens. ]] local function GetTermFrequency(tokens) -- ▼ ReadonlyArray.reduce ▼ local _result = {} local _callback = function(acc, token) local _acc = acc local _exp = token local _acc_1 = acc local _token = token local _condition = _acc_1[_token] if _condition == nil then _condition = 0 end _acc[_exp] = _condition + 1 return acc end for _i = 1, #tokens do _result = _callback(_result, tokens[_i], _i - 1, tokens) end -- ▲ ReadonlyArray.reduce ▲ return _result end --[[ * * Builds a combined vocabulary from two term frequency maps. ]] local function BuildVocabulary(tf1, tf2) local _array = {} local _length = #_array local _set = {} local _array_1 = {} local _length_1 = #_array_1 local _array_2 = GetKeys(tf1) local _Length = #_array_2 table.move(_array_2, 1, _Length, _length_1 + 1, _array_1) _length_1 += _Length local _array_3 = GetKeys(tf2) table.move(_array_3, 1, #_array_3, _length_1 + 1, _array_1) for _, _v in _array_1 do _set[_v] = true end for _v in _set do _length += 1 _array[_length] = _v end return _array end --[[ * * Converts text to vector based on global vocabulary. ]] local function TextToVector(term_freq, vocabulary) -- ▼ ReadonlyArray.map ▼ local _newValue = table.create(#vocabulary) local _callback = function(term) local _term_freq = term_freq local _term = term local _condition = _term_freq[_term] if _condition == nil then _condition = 0 end return _condition end for _k, _v in vocabulary do _newValue[_k] = _callback(_v, _k - 1, vocabulary) end -- ▲ ReadonlyArray.map ▲ return _newValue end --[[ * * Computes cosine similarity between two vectors. * Returns 0-1 (1 = identical, 0 = no similarity). ]] local function CosineSimilarity(vec_a, vec_b) if #vec_a ~= #vec_b then error("Vector length mismatch") end -- Handle zero vectors -- ▼ ReadonlyArray.reduce ▼ local _result = 0 local _callback = function(sum, x) return sum + x * x end for _i = 1, #vec_a do _result = _callback(_result, vec_a[_i], _i - 1, vec_a) end -- ▲ ReadonlyArray.reduce ▲ local mag_a = math.sqrt(_result) -- ▼ ReadonlyArray.reduce ▼ local _result_1 = 0 local _callback_1 = function(sum, x) return sum + x * x end for _i = 1, #vec_b do _result_1 = _callback_1(_result_1, vec_b[_i], _i - 1, vec_b) end -- ▲ ReadonlyArray.reduce ▲ local mag_b = math.sqrt(_result_1) if mag_a == 0 and mag_b == 0 then return 1 end if mag_a == 0 or mag_b == 0 then return 0 end -- Dot product calculation -- ▼ ReadonlyArray.reduce ▼ local _result_2 = 0 local _callback_2 = function(sum, x, i) return sum + x * vec_b[i + 1] end for _i = 1, #vec_a do _result_2 = _callback_2(_result_2, vec_a[_i], _i - 1, vec_a) end -- ▲ ReadonlyArray.reduce ▲ local dot_product = _result_2 return dot_product / (mag_a * mag_b) end -- Main function local function CosineTextSimilarityScore(tokenized_term, tokenized_query) local tf_a = GetTermFrequency(tokenized_term) local tf_b = GetTermFrequency(tokenized_query) local vocabulary = BuildVocabulary(tf_a, tf_b) local vec_a = TextToVector(tf_a, vocabulary) local vec_b = TextToVector(tf_b, vocabulary) return CosineSimilarity(vec_a, vec_b) end return { CosineTextSimilarityScore = CosineTextSimilarityScore, }