import { buildWordOrNumberPattern as buildWordSplitterPattern, phraseSeparatorRegExp, sentenceSeparatorTrailingPunctuationCharacterRegExp, sentenceSeparatorCharacterRegExp, whitespacePatternRegExp, letterPatternGlobalRegExp, startsWithWhitespacePattern, endsWithWhitespacePattern } from '../patterns/Patterns.js'
import { cldrSuppressions, additionalSuppressions, leadingApostropheContractionSuppressions, nounSuppressions, tldSuppressions } from '../patterns/Suppressions.js'
import { eastAsianCharRangesRegExp } from '../patterns/EastAsianCharacterPatterns.js'
import { WordSequence } from './WordSequence.js'
import { getShortLanguageCode } from '../utilities/Utilities.js'

import { buildRegExp } from 'regexp-composer'

////////////////////////////////////////////////////////////////////////////////////////////////
// Exported methods
////////////////////////////////////////////////////////////////////////////////////////////////

// Splits a text string into words and then segments the resulting word sequence into sentences and phrases
// (this is the main high-level entry point; it is async because splitting may involve ICU postprocessing)
export async function segmentText(text: string, options?: SegmentationOptions) {
	// Split the text into a sequence of individual words (this may await East Asian postprocessing)
	const wordSequence = await splitToWords(text, options)

	// Segment the word sequence into segments (paragraphs), sentences, and phrases, and return the result
	return segmentWordSequence(wordSequence)
}

// Segments an already-tokenized word sequence into segments (paragraphs), sentences, and phrases
// This is the core segmentation algorithm, working purely on the word level
// (options only control how colons/semicolons are treated when splitting phrases)
export async function segmentWordSequence(wordSequence: WordSequence, options?: WordSequenceSegmentationOptions) {
	// Fill in any option values the caller didn't provide with their defaults
	options = { ...defaultWordSequenceSegmentationOptions, ...options }

	// This will hold the start and end word offsets of each paragraph (called a "segment" here)
	const segmentWordRanges: Range[] = []

	// Step 1: find segment (paragraph) boundaries.
	// A new segment begins right after a newline character that follows at least one non-whitespace word.
	{
		// The word offset at which the current segment starts
		let segmentStartWordOffset = 0
		// Whether at least one non-whitespace word has been seen in the current segment
		let nonWhitespaceWordSeen = false
		// Whether a newline has been seen in the current segment (i.e. we may be at a boundary)
		let newlineSeenInCurrentSegment = false

		// Walk through every word of the sequence
		for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
			// Get the text of the word at the current index
			const word = wordSequence.getWordAt(wordIndex)

			// If we haven't seen a newline yet, the segment already has content, and this word is a newline, mark the newline as seen
			if (!newlineSeenInCurrentSegment && nonWhitespaceWordSeen && word.endsWith('\n')) {
				newlineSeenInCurrentSegment = true
			}

			// If a newline was seen, decide whether the segment actually ends at this word
			if (newlineSeenInCurrentSegment) {
				// Look at the word after the current one (or an empty string if the current word is last)
				const nextWord = wordIndex < wordSequence.length - 1 ? wordSequence.getWordAt(wordIndex + 1) : ''

				// End the segment here if the newline is followed by real content (or by nothing at all)
				if (nextWord === '' || !whitespacePatternRegExp.test(nextWord)) {
					// Record the current segment, ending after the current (newline) word
					segmentWordRanges.push({ start: segmentStartWordOffset, end: wordIndex + 1 })
					// The next segment starts at the word after the newline
					segmentStartWordOffset = wordIndex + 1

					// Reset the newline flag for the new segment
					newlineSeenInCurrentSegment = false
				}
			}

			// If no non-whitespace word has been seen yet, check whether the current word is non-whitespace
			if (nonWhitespaceWordSeen === false) {
				nonWhitespaceWordSeen = !whitespacePatternRegExp.test(word)
			}
		}

		// If the last segment didn't reach the end of the sequence, close it with the remaining words
		if (segmentStartWordOffset < wordSequence.length) {
			segmentWordRanges.push({ start: segmentStartWordOffset, end: wordSequence.length })
		}
	}

	// These will hold the word offsets of each sentence, and (per segment) the range of sentence indices in it
	const sentenceWordRanges: Range[] = []
	const segmentSentenceRanges: Range[] = []

	// Step 2: find sentence boundaries inside each segment.
	// A sentence ends at a sentence-separator character (like '.', '!', '?') once it contains a minimum number of letters.
	{
		// A sentence must contain at least this many letters before a separator may end it
		const minimumSentenceLetterCount = 2

		// Process each segment independently
		for (let segmentIndex = 0; segmentIndex < segmentWordRanges.length; segmentIndex++) {
			// Get the word range of the current segment
			const segmentWordRange = segmentWordRanges[segmentIndex]

			// Record the segment's first and last word index
			const segmentStartWordIndex = segmentWordRange.start
			const segmentEndWordIndex = segmentWordRange.end

			// Add a placeholder entry for this segment's sentence range, starting (and initially ending) at the current sentence count
			segmentSentenceRanges.push({ start: sentenceWordRanges.length, end: sentenceWordRanges.length })

			// Whether a non-whitespace word has been seen in the current sentence
			let nonWhitespaceWordSeen = false
			// The word offset at which the current sentence starts
			let sentenceStartWordOffset = segmentStartWordIndex
			// The number of letters accumulated in the current sentence
			let currentSentenceLetterCount = 0

			// Walk through the words of this segment
			for (let wordIndex = segmentStartWordIndex; wordIndex < segmentEndWordIndex; wordIndex++) {
				// Get the text of the current word
				const word = wordSequence.getWordAt(wordIndex)

				// If no non-whitespace word has been seen yet, check whether the current word is non-whitespace
				if (nonWhitespaceWordSeen === false) {
					nonWhitespaceWordSeen = !whitespacePatternRegExp.test(word)
				}

				// Count letters, but only while the sentence hasn't yet reached the minimum (to save work)
				if (currentSentenceLetterCount < minimumSentenceLetterCount) {
					// Find every letter character in the current word
					const matches = word.matchAll(letterPatternGlobalRegExp)

					// Count the letters, stopping early once the minimum is reached
					for (const _ of matches) {
						currentSentenceLetterCount += 1

						if (currentSentenceLetterCount >= minimumSentenceLetterCount) {
							break
						}
					}
				}

				// End the sentence here if it has content, enough letters, and the current word is a sentence-separator character
				if (nonWhitespaceWordSeen && currentSentenceLetterCount >= minimumSentenceLetterCount && sentenceSeparatorCharacterRegExp.test(word)) {
					// Start scanning at the separator; this index marks where the sentence's trailing punctuation ends
					let trailingSequenceEndIndex = wordIndex

					// Consume any trailing punctuation words (closing quotes, brackets, etc.) that belong to this sentence
					while (trailingSequenceEndIndex < segmentEndWordIndex) {
						// Get the text of the candidate trailing-punctuation word
						const trailingWord = wordSequence.getWordAt(trailingSequenceEndIndex)

						// Keep consuming it if it is trailing punctuation; otherwise stop
						if (sentenceSeparatorTrailingPunctuationCharacterRegExp.test(trailingWord)) {
							trailingSequenceEndIndex++
						} else {
							break
						}
					}

					// Record the completed sentence, from its start offset up to (but not including) the trailing punctuation
					sentenceWordRanges.push({
						start: sentenceStartWordOffset,
						end: trailingSequenceEndIndex
					})

					// Extend the current segment's sentence range to include the sentence we just recorded
					segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1

					// The next sentence starts right after the trailing punctuation
					sentenceStartWordOffset = trailingSequenceEndIndex
					// Reset the letter count for the next sentence
					currentSentenceLetterCount = 0

					// Skip the words already consumed by the trailing-punctuation scan (the for loop will increment again)
					wordIndex = trailingSequenceEndIndex - 1
				}
			}

			// If the segment ends with words that don't complete a new sentence, close the final sentence at the segment end
			if (sentenceStartWordOffset < segmentEndWordIndex) {
				sentenceWordRanges.push({ start: sentenceStartWordOffset, end: segmentEndWordIndex })
				segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1
			}
		}
	}

	// This will hold the Sentence objects built from the sentence word ranges
	const sentences: Sentence[] = []

	// Build a Sentence object for every sentence range found above
	for (const wordRange of sentenceWordRanges) {
		// Extract the slice of the word sequence that belongs to this sentence
		const sentenceWordSequence = wordSequence.slice(wordRange.start, wordRange.end)

		// Create the Sentence for this range and append it to the list
		sentences.push(new Sentence(wordRange, sentenceWordSequence))
	}

	// Step 3: split each sentence into phrases at phrase-separator characters (commas, colons, semicolons, etc.)
	for (const sentence of sentences) {
		// This will hold the start and end word offsets of each phrase in the current sentence
		const phraseWordRanges: Range[] = []

		// The word offset just past the end of this sentence
		let sentenceEndWordOffset = sentence.wordRange.end
		// The word offset at which the current phrase starts
		let phraseStartWordOffset = sentence.wordRange.start

		// Walk through the words of this sentence
		for (let wordIndex = phraseStartWordOffset; wordIndex < sentenceEndWordOffset; wordIndex++) {
			// Get the text of the current word
			const currentWord = wordSequence.getWordAt(wordIndex)

			// Determine whether this word is a phrase separator:
			// it must match the phrase-separator pattern, and if the option is set, ':' or ';' only count when followed by whitespace
			const isCurrentWordPhraseSeparator =
				phraseSeparatorRegExp.test(currentWord) &&
				(!options.requireSpaceAfterColonsOrSemicolons ||
					(currentWord !== ':' && currentWord !== ';') ||
					whitespacePatternRegExp.test(wordSequence.getWordAt(wordIndex + 1)))

			// If the current word is a phrase separator, close the current phrase at this word
			if (isCurrentWordPhraseSeparator) {
				// Tracks whether at least one whitespace word was seen while scanning trailing characters
				let whitespaceSeenOnce = false

				// Include any trailing punctuation (closing quotes, brackets, etc.) in the current phrase
				while (wordIndex < sentenceEndWordOffset - 1) {
					// Get the text of the next word
					const nextWord = wordSequence.getWordAt(wordIndex + 1)

					// Stop consuming if the next word is not trailing punctuation
					if (!sentenceSeparatorTrailingPunctuationCharacterRegExp.test(nextWord)) {
						break
					}

					// Remember if this trailing word is whitespace
					if (whitespacePatternRegExp.test(nextWord)) {
						whitespaceSeenOnce = true
					}

					// A closing double quote that follows whitespace starts the next phrase, so stop before it
					if (nextWord === '"' && whitespaceSeenOnce) {
						break
					}

					// Otherwise consume this trailing-punctuation word
					wordIndex += 1
				}

				// Record the phrase, from its start offset through the separator (and any trailing punctuation)
				phraseWordRanges.push({
					start: phraseStartWordOffset,
					end: wordIndex + 1
				})

				// The next phrase starts right after the current separator
				phraseStartWordOffset = wordIndex + 1
			}
		}

		// If the sentence ends with words that weren't part of any phrase, close the final phrase at the sentence end
		if (phraseStartWordOffset < sentenceEndWordOffset) {
			phraseWordRanges.push({ start: phraseStartWordOffset, end: sentenceEndWordOffset })
		}

		// Build a Phrase object for every phrase range and attach it to the sentence
		for (const wordRange of phraseWordRanges) {
			// Extract the slice of the word sequence that belongs to this phrase
			const phraseWordSequence = wordSequence.slice(wordRange.start, wordRange.end)

			// Create the Phrase for this range and add it to the sentence's phrase list
			sentence.phrases.push(new Phrase(wordRange, phraseWordSequence))
		}
	}

	// Assemble the final result: the full word sequence, the list of sentences, and the segment-to-sentence mapping
	const result: SegmentationResult = {
		words: wordSequence,
		sentences,
		segmentSentenceRanges,
	}

	// Return the assembled segmentation result
	return result
}

// Cache of compiled word-splitter regular expressions, keyed by the JSON-serialized options
// (compiling a regex is expensive, so the same options reuse the same compiled expression)
const cachedWordSplitterRegExps = new Map<string, RegExp>()

// Splits a string of text into a WordSequence of individual words and punctuation marks
// The returned sequence contains both the matched words and the punctuation found between/beside them
export async function splitToWords(text: string, options?: SegmentationOptions) {
	// If no options were given, start from an empty options object
	if (!options) {
		options = {}
	}

	// Fill in any option values the caller didn't provide with their defaults
	options = { ...defaultSegmentationOptions, ...options }

	// If a language was specified, normalize it to its short base code (e.g. "en-US" becomes "en")
	// so that language-specific data is looked up consistently
	if (options.language) {
		options.language = getShortLanguageCode(options.language)
	}

	// Serialize the (now normalized) options to JSON so they can be used as a cache key
	const optionsAsJson = JSON.stringify(options)

	// Look up a previously compiled splitter expression for these exact options
	let wordSplitterRegExp = cachedWordSplitterRegExps.get(optionsAsJson)

	// If no cached expression exists, compile a new one and store it in the cache
	if (!wordSplitterRegExp) {
		wordSplitterRegExp = buildWordSplitterRegExpForOptions(options)

		cachedWordSplitterRegExps.set(optionsAsJson, wordSplitterRegExp)
	}

	// Start with an empty word sequence that will accumulate the results
	let wordSequence = new WordSequence()

	// Adds the words found in a run of text between two matched words (typically punctuation and whitespace)
	// to the word sequence. Spaces are skipped; every other character becomes its own punctuation word,
	// except that consecutive punctuation characters are grouped into a single word.
	function addPunctuationWordsBetween(startOffset: number, endOffset: number) {
		// Extract the substring that lies between the two given character offsets
		const punctuationWordSubstring = text.substring(startOffset, endOffset)

		// The current character offset within the source text
		let charOffset = startOffset
		// The character offset at which the current punctuation word starts
		let punctuationWordStartOffset = startOffset

		// If any characters have been accumulated since the last flush, add them as a single punctuation word
		function addPunctuationWordIfNeeded() {
			// Check whether there are pending characters to add
			if (charOffset > punctuationWordStartOffset) {
				// Extract the pending word text from the source
				const wordText = text.substring(punctuationWordStartOffset, charOffset)
				// Add it to the sequence, marking it as punctuation
				wordSequence.addWord(wordText, punctuationWordStartOffset, true)

				// The next pending word (if any) starts at the current character offset
				punctuationWordStartOffset = charOffset
			}
		}

		// Iterate over every character (codepoint) in the gap between words
		for (const char of punctuationWordSubstring) {
			// Spaces are skipped: they just advance the character offset and are not added as words
			if (char === ' ') {
				charOffset += 1

				continue
			}

			// Flush any pending punctuation word before the current character
			addPunctuationWordIfNeeded()

			// Advance the character offset past the current character (char.length handles astral codepoints)
			charOffset += char.length

			// Flush again so this single character becomes its own punctuation word
			addPunctuationWordIfNeeded()
		}

		// Flush any remaining pending punctuation word at the end of the gap
		addPunctuationWordIfNeeded()
	}

	// Find all word matches (as defined by the splitter expression) throughout the text
	const wordMatches = text.matchAll(wordSplitterRegExp)

	// Track the character offset at which the last matched word ended
	let lastMatchEndOffset = 0

	// Process each matched word in order of appearance
	if (wordMatches) {
		for (const match of wordMatches) {
			// Read the match's start and end character offsets (available because the regex has the 'd' flag)
			const offsets = match.indices![0]!
			const matchStartOffset = offsets[0]
			const matchEndOffset = offsets[1]

			// If there is a gap between the previous match and this one, add the gap's punctuation as words
			if (matchStartOffset > lastMatchEndOffset) {
				addPunctuationWordsBetween(lastMatchEndOffset, matchStartOffset)
			}

			// Extract the matched word's text from the source
			const wordText = text.substring(matchStartOffset, matchEndOffset)
			// Add the matched word to the sequence (it is a real word, not punctuation)
			wordSequence.addWord(wordText, matchStartOffset, false)

			// Remember where this match ended so gaps after it can be found
			lastMatchEndOffset = matchEndOffset
		}

		// If the text extends beyond the last match, add the trailing punctuation as words
		addPunctuationWordsBetween(lastMatchEndOffset, text.length)
	}

	// If East Asian postprocessing is enabled, further split runs of East Asian characters into subwords using ICU word breaking
	if (options.enableEastAsianPostprocessing) {
		wordSequence = await postprocessEastAsianWords(text, wordSequence)
	}

	// Return the resulting word sequence
	return wordSequence
}

// Add any missing punctuation words to a word sequence
// Given a word sequence that was built from the source text (with its character offsets) and the source text itself,
// produce a new word sequence that also includes every punctuation character found between, before, and after the words.
// Also returns a mapping from each new entry's index back to the original word index it came from
// (punctuation-only entries are not present as keys in this mapping).
export function addMissingPunctuationWordsToWordSequence(wordSequence: WordSequence, sourceText: string) {
	// Maps each index in the new (punctuation-included) sequence to the index of the original word it was copied from
	const originalWordsReverseMapping = new Map<number, number>()

	// The new word sequence that will contain both the original words and the missing punctuation words
	const wordSequenceWithPunctuation = new WordSequence()

	// Adds entries for every character of a text slice, grouping runs of non-space characters into single punctuation words
	// (spaces are merged into the preceding punctuation word, preserving them for offset correctness)
	function addWordEntriesForTextSlice(textSlice: string, initialCharOffset: number) {
		// The current character offset within the source text
		let charOffset = initialCharOffset

		// Add entry for every codepoint (this will correctly treat characters beyond BMP)
		for (const char of textSlice) {
			// Compute the source-text offset just past this character
			const charEndOffset = charOffset + char.length

			// Get the last entry added so far (if any)
			const lastEntry = wordSequenceWithPunctuation.lastEntry

			// If this is a space and the last entry is a punctuation word that already starts with a space, extend that entry
			if (char === ' ' && lastEntry && lastEntry.isPunctuation && lastEntry.text[0] === ' ') {
				wordSequenceWithPunctuation.lastEntry.text += ' '
				wordSequenceWithPunctuation.lastEntry.endOffset = charEndOffset
			} else {
				// Otherwise, extract this single character from the source and add it as a new punctuation word
				const wordText = sourceText.substring(charOffset, charEndOffset)

				wordSequenceWithPunctuation.addWord(wordText, charOffset, true)
			}

			// Advance the character offset past the current character
			charOffset = charEndOffset
		}
	}

	// Process each original word in order
	for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
		// Get the current word entry and the offset where it starts in the source text
		const wordEntry = wordSequence.getEntryAt(wordIndex)
		const wordStartOffset = wordEntry.startOffset

		// Get the end offset of the previous word (or 0 if this is the first word)
		const previousWordEndOffset = wordIndex > 0 ? wordSequence.entries[wordIndex - 1].endOffset : 0

		// Add entries for any punctuation characters between the current and previous word (or the start of the text)
		if (previousWordEndOffset !== wordStartOffset) {
			// Extract the slice of source text lying between the two words
			const textSlice = sourceText.substring(previousWordEndOffset, wordStartOffset)

			// Add entries for the punctuation characters in that slice
			addWordEntriesForTextSlice(textSlice, previousWordEndOffset)
		}

		// Copy the original word entry into the new sequence
		wordSequenceWithPunctuation.entries.push(wordEntry)
		// Record that this new entry corresponds to the current original word index
		originalWordsReverseMapping.set(wordSequenceWithPunctuation.length - 1, wordIndex)

		// If last word, add entries for any trailing punctuation characters
		if (wordIndex === wordSequence.length - 1) {
			// If the source text extends beyond the last word, the remainder is trailing punctuation
			if (sourceText.length !== wordEntry.endOffset) {
				// Extract the slice of source text after the last word
				const textSlice = sourceText.substring(wordEntry.endOffset, sourceText.length)

				// Add entries for the trailing punctuation characters
				addWordEntriesForTextSlice(textSlice, wordEntry.endOffset)
			}
		}
	}

	// Return the enriched word sequence along with the reverse mapping to the original words
	return { wordSequenceWithPunctuation, originalWordsReverseMapping }
}

////////////////////////////////////////////////////////////////////////////////////////////////
// Helper methods
////////////////////////////////////////////////////////////////////////////////////////////////
// Builds a global regular expression that matches words in the given language, using the configured options
function buildWordSplitterRegExpForOptions(options: SegmentationOptions) {
	// Get the CLDR suppression list (standard abbreviations like "Dr.") for this language, or an empty list if none exists
	const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? []
	// Get the extended, project-specific suppression list for this language, or an empty list
	const extendedSuppressionsForLang = additionalSuppressions[options.language ?? ''] ?? []
	// Get the leading-apostrophe contraction suppressions for this language, or an empty list
	const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? []
	// Build a variant of those contractions that uses the typographic apostrophe (') instead of the straight one (')
	const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`))
	// Get any custom suppressions the caller provided, or an empty list
	const customSuppressions = options.customSuppressions ?? []

	// Combine all suppression sources into one flat list
	let suppressions = [
		...customSuppressions,
		...cldrSuppressionsForLang,
		...extendedSuppressionsForLang,
		...contractionSuppressionsForLang,
		...contractionSuppressionsForLangWithSingleQuote,
		...nounSuppressions,
		...tldSuppressions,
	]

	// Build the word pattern: match any suppression (in original, lowercase, and uppercase forms) or a regular word
	const wordPattern = buildWordSplitterPattern([
		...suppressions,
		...suppressions.map(word => word.toLocaleLowerCase()),
		...suppressions.map(word => word.toLocaleUpperCase()),
	])

	// Compile the pattern into a global regular expression (the 'd' flag, added by regexp-composer, gives match indices)
	const wordSplitterRegExp = buildRegExp(wordPattern, { global: true })

	// Return the compiled regular expression
	return wordSplitterRegExp
}

// Tries to load the ICU segmentation WebAssembly module; returns undefined if it is not installed
async function getIcuSegmentation() {
	// Attempt to dynamically import the ICU segmentation module
	try {
		const icuSegmentation = await import('@echogarden/icu-segmentation-wasm')

		// If the import succeeded, return the module
		return icuSegmentation
	} catch {
		// If the import failed (module not available), signal that ICU segmentation is unavailable
		return undefined
	}
}

// Further splits words that contain East Asian characters (Chinese, Japanese, Thai, Khmer) into smaller subwords
// using ICU's word-break rules, which are much better suited to these scripts than the generic splitter pattern
async function postprocessEastAsianWords(containingText: string, wordSequence: WordSequence) {
	// Try to load the ICU segmentation module
	const icuSegmentation = await getIcuSegmentation()

	// If ICU segmentation is unavailable, return the original word sequence unchanged
	if (icuSegmentation === undefined) {
		return wordSequence
	}

	// Tracks whether the ICU module has been initialized (initialization only needs to happen once)
	let icuInitialized = false

	// A new word sequence that will hold the postprocessed words
	const newWordSequence = new WordSequence()

	// Process each word in the original sequence
	for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
		// Get the current word entry and the offset where it starts in the source text
		const wordEntry = wordSequence.entries[wordIndex]
		const wordStartOffset = wordEntry.startOffset

		// Get the text of the current word
		const word = wordSequence.getWordAt(wordIndex)

		// If the word contains East Asian characters, split it into subwords
		if (eastAsianCharRangesRegExp.test(word)) {
			// Initialize the ICU module on first use
			if (!icuInitialized) {
				await icuSegmentation.initialize()

				icuInitialized = true
			}

			// Get the word-break positions within the word (offsets into the word string)
			const wordBreaks = [...icuSegmentation.createWordBreakIterator(word)]

			// Create a subword for every pair of consecutive break positions
			for (let i = 0; i < wordBreaks.length - 1; i++) {
				// Compute the subword's start and end offsets within the source text
				const subwordStartOffset = wordStartOffset + wordBreaks[i]
				const subwordEndOffset = wordStartOffset + wordBreaks[i + 1]

				// Extract the subword text from the containing text
				const subwordText = containingText.substring(subwordStartOffset, subwordEndOffset)

				// Add the subword to the new sequence (it is a real word, not punctuation)
				newWordSequence.addWord(
					subwordText,
					subwordStartOffset,
					false,
				)
			}
		} else {
			// Otherwise, extract the word text from the source and copy it to the new sequence unchanged
			const wordText = containingText.substring(wordEntry.startOffset, wordEntry.endOffset)

			newWordSequence.addWord(wordText, wordEntry.startOffset, wordEntry.isPunctuation)
		}
	}

	// Return the postprocessed word sequence
	return newWordSequence
}

// Computes the character ranges of all punctuation in the text.
// This includes gaps between words, entries that are themselves punctuation, and any leading/trailing text
// that is not covered by a word entry.
function getPunctuationRanges(wordSequence: WordSequence, text: string) {
	// This will hold the character ranges of the punctuation found
	const punctuationRanges: Range[] = []
	// Get the list of word entries
	const wordEntries = wordSequence.entries

	// If the text starts before the first word, the leading part is punctuation
	if (wordEntries[0].startOffset > 0) {
		punctuationRanges.push({ start: 0, end: wordEntries[0].startOffset })
	}

	// Examine every word entry
	for (let i = 0; i < wordEntries.length; i++) {
		// Get the current entry
		const entry = wordEntries[i]

		// Get the end offset of the previous entry (or 0 for the first entry)
		const previousEndOffset = wordEntries[i - 1]?.endOffset ?? 0

		// If there is a gap between the previous entry and this one, that gap is punctuation
		if (entry.startOffset > previousEndOffset) {
			punctuationRanges.push({ start: previousEndOffset, end: entry.startOffset })
		}

		// If the current entry is itself a punctuation word, its whole range is punctuation
		if (entry.isPunctuation) {
			punctuationRanges.push({ start: entry.startOffset, end: entry.endOffset })
		}
	}

	{
		// Get the end offset of the last entry (if any)
		const lastEndOffset = wordEntries[wordEntries.length - 1]?.endOffset

		// If the text extends beyond the last entry, the trailing part is punctuation
		if (lastEndOffset && lastEndOffset < text.length) {
			punctuationRanges.push({ start: lastEndOffset, end: text.length })
		}
	}

	// Return the collected punctuation ranges
	return punctuationRanges
}


////////////////////////////////////////////////////////////////////////////////////////////////
// Types
////////////////////////////////////////////////////////////////////////////////////////////////

// Describes the result of segmenting a text: the full word sequence, the detected sentences,
// and how the sentences map back to the segments (paragraphs) they belong to
// (segmentSentenceRanges[i] is the range of sentence indices within segment i)
export interface SegmentationResult {
	words: WordSequence
	sentences: Sentence[]
	segmentSentenceRanges: Range[]
}

// A fragment of text (either a sentence or a phrase): it holds the word range it occupies and its words
// (this is the shared base class of Sentence and Phrase)
export class TextFragment {
	// The range of word indices this fragment covers (in the overall word sequence)
	wordRange: Range
	// The words that make up this fragment
	words: WordSequence

	// Creates a fragment from a word range and the corresponding slice of words
	constructor(wordRange: Range, words: WordSequence) {
		this.wordRange = wordRange
		this.words = words
	}

	// Returns the fragment's text by concatenating all of its words
	get text() {
		return this.words.text
	}

	// Returns the character range of this fragment within the source text,
	// from the start of its first word to the end of its last word
	get charRange(): Range {
		return {
			start: this.words.firstEntry.startOffset,
			end: this.words.lastEntry.endOffset
		}
	}
}

// A sentence: a text fragment that additionally contains the phrases it was split into
// (phrases are filled in during segmentation)
export class Sentence extends TextFragment {
	// The phrases that make up this sentence
	phrases: Phrase[] = []
}

// A phrase: a text fragment representing one part of a sentence
// (it has no additional members; it exists to give phrases their own type)
export class Phrase extends TextFragment {
}

// Represents a range of indices (either word indices or character offsets)
// with an inclusive start and an exclusive end
export interface Range {
	start: number
	end: number
}

// Options that control how text is split into words
export interface SegmentationOptions {
	language?: string
	customSuppressions?: string[]
	enableEastAsianPostprocessing?: boolean
}

// Default options for word splitting: no language, no custom suppressions, and East Asian postprocessing enabled
export const defaultSegmentationOptions: SegmentationOptions = {
	language: '',
	customSuppressions: [],
	enableEastAsianPostprocessing: true,
}

// Options that control how a word sequence is segmented into sentences and phrases
export interface WordSequenceSegmentationOptions {
	requireSpaceAfterColonsOrSemicolons: boolean
}

// Default options for word-sequence segmentation: ':' and ';' only count as phrase separators when followed by a space
export const defaultWordSequenceSegmentationOptions: WordSequenceSegmentationOptions = {
	requireSpaceAfterColonsOrSemicolons: true
}