--!native
--!optimize 2
-- Compiled with roblox-ts v3.0.0
local TS = _G[script]
local SubString = TS.import(script, script.Parent.Parent, "Utils", "SubString")
local function NGramTokenization(text, n)
	local _arg0 = n > 0
	assert(_arg0, "N must be >= 1")
	local tokens = {}
	do
		local i = 0
		local _shouldIncrement = false
		while true do
			if _shouldIncrement then
				i += 1
			else
				_shouldIncrement = true
			end
			if not (i <= #text - n) then
				break
			end
			local _arg0_1 = SubString(text, i, i + n)
			table.insert(tokens, _arg0_1)
		end
	end
	return tokens
end
return {
	NGramTokenization = NGramTokenization,
}