--!native
--!optimize 2
-- Compiled with roblox-ts v3.0.0
--[[
	*
	 * Tokenizes text into lowercase words, ignoring punctuation.
	 
]]
local function WordTokenization(text, options)
	local tokens = {}
	local _result = options
	if _result ~= nil then
		_result = _result.PreserveAccents
	end
	local pattern = if _result then "[%w\x80-\xff]+" else "%w+"
	local normalized = (string.gsub((string.gsub(string.lower(text), "['']", "'")), "[%c%z]", " "))
	local _result_1 = options
	if _result_1 ~= nil then
		_result_1 = _result_1.PreserveHyphens
	end
	if _result_1 then
		normalized = (string.gsub(normalized, "-", " "))
	end
	for token in string.gmatch(normalized, pattern) do
		if (string.find(token, "'")) ~= nil then
			for part in string.gmatch(token, "[^']+") do
				table.insert(tokens, part)
			end
			continue
		end
		table.insert(tokens, token)
	end
	return tokens
end
return {
	WordTokenization = WordTokenization,
}