export type WhiteSpaceMode = 'normal' | 'pre-wrap' export type WordBreakMode = 'normal' | 'keep-all' export type SegmentBreakKind = | 'text' | 'space' | 'preserved-space' | 'tab' | 'glue' | 'zero-width-break' | 'soft-hyphen' | 'hard-break' type SegmentationPiece = { text: string isWordLike: boolean kind: SegmentBreakKind start: number } export type MergedSegmentation = { len: number texts: string[] isWordLike: boolean[] kinds: SegmentBreakKind[] starts: number[] } export type TextAnalysis = { normalized: string } & MergedSegmentation export type AnalysisProfile = { geckoAsciiLineBreaks: boolean carryCJKAfterClosingQuote: boolean breakKeepAllAfterPunctuation: boolean } const collapsibleWhitespaceRunRe = /[ \t\n\r\f]+/g const needsWhitespaceNormalizationRe = /[\t\n\r\f]| {2,}|^ | $/ export function normalizeWhitespaceNormal(text: string): string { if (!needsWhitespaceNormalizationRe.test(text)) return text let normalized = text.replace(collapsibleWhitespaceRunRe, ' ') if (normalized.charCodeAt(0) === 0x20) { normalized = normalized.slice(1) } if (normalized.length > 0 && normalized.charCodeAt(normalized.length - 1) === 0x20) { normalized = normalized.slice(0, -1) } return normalized } function normalizeWhitespacePreWrap(text: string): string { if (!/[\r\f]/.test(text)) return text return text .replace(/\r\n/g, '\n') .replace(/[\r\f]/g, '\n') } let sharedGraphemeSegmenter: Intl.Segmenter | null = null export function getSharedGraphemeSegmenter(): Intl.Segmenter { if (sharedGraphemeSegmenter === null) { sharedGraphemeSegmenter = new Intl.Segmenter(undefined, { granularity: 'grapheme' }) } return sharedGraphemeSegmenter } let sharedWordSegmenter: Intl.Segmenter | null = null let segmenterLocale: string | undefined function getSharedWordSegmenter(): Intl.Segmenter { if (sharedWordSegmenter === null) { sharedWordSegmenter = new Intl.Segmenter(segmenterLocale, { granularity: 'word' }) } return sharedWordSegmenter } export function clearAnalysisCaches(): void { sharedGraphemeSegmenter = null sharedWordSegmenter = null } export function setAnalysisLocale(locale?: string): void { const nextLocale = locale && locale.length > 0 ? locale : undefined if (segmenterLocale === nextLocale) return segmenterLocale = nextLocale sharedWordSegmenter = null } const arabicScriptRe = /\p{Script=Arabic}/u const combiningMarkRe = /\p{M}/u const decimalDigitRe = /\p{Nd}/u function containsArabicScript(text: string): boolean { return arabicScriptRe.test(text) } // CJK ranges used by the wrapping policy, including supplementary ideographs. const cjkRe = /[\u3000-\u30FF\u3130-\u318F\u3400-\u4DBF\u4E00-\u9FFF\uAC00-\uD7AF\uF900-\uFAFF\uFF00-\uFFEF\u{20000}-\u{2A6DF}\u{2A700}-\u{2EE5D}\u{2F800}-\u{2FA1F}\u{30000}-\u{33479}]/u export function isCJK(s: string): boolean { return cjkRe.test(s) } function endsWithLineStartProhibitedText(text: string): boolean { const last = getLastCodePoint(text) return last !== null && (kinsokuStart.has(last) || leftStickyPunctuation.has(last)) } const keepAllGlueChars = new Set([ '\u00A0', '\u202F', '\u2060', '\uFEFF', ]) const keepAllDashBreakChars = new Set([ '-', '\u2010', '\u2013', '\u2014', ]) function endsWithKeepAllGlueText(text: string): boolean { const last = getLastCodePoint(text) return last !== null && keepAllGlueChars.has(last) } function endsWithKeepAllDashBreakText(text: string): boolean { const last = getLastCodePoint(text) return last !== null && keepAllDashBreakChars.has(last) } export function canContinueKeepAllTextRun(previousText: string, breakAfterPunctuation: boolean): boolean { if (endsWithKeepAllGlueText(previousText)) return false if (!breakAfterPunctuation) return true if (endsWithLineStartProhibitedText(previousText)) return false if (endsWithKeepAllDashBreakText(previousText)) return false return true } export const kinsokuStart = new Set([ '\uFF0C', '\uFF0E', '\uFF01', '\uFF1A', '\uFF1B', '\uFF1F', '\u3001', '\u3002', '\u30FB', '\uFF09', '\u3015', '\u3009', '\u300B', '\u300D', '\u300F', '\u3011', '\u3017', '\u3019', '\u301B', '\u30FC', '\u3005', '\u303B', '\u309D', '\u309E', '\u30FD', '\u30FE', ]) export const kinsokuEnd = new Set([ '"', '(', '[', '{', '¡', '¿', '“', '‘', '‚', '„', '«', '‹', '\u2E18', '\uFF08', '\u3014', '\u3008', '\u300A', '\u300C', '\u300E', '\u3010', '\u3016', '\u3018', '\u301A', ]) const forwardStickyGlue = new Set([ "'", '’', ]) export const leftStickyPunctuation = new Set([ '.', ',', '!', '?', ':', ';', '\u060C', '\u061B', '\u061F', '\u0964', '\u0965', '\u104A', '\u104B', '\u104C', '\u104D', '\u104F', ')', ']', '}', '%', '"', '”', '’', '»', '›', '…', ]) const arabicNoSpaceTrailingPunctuation = new Set([ ':', '.', '\u060C', '\u061B', ]) const myanmarMedialGlue = new Set([ '\u104F', ]) const closingQuoteChars = new Set([ '”', '’', '»', '›', '\u300D', '\u300F', '\u3011', '\u300B', '\u3009', '\u3015', '\uFF09', ]) function isLeftStickyPunctuationSegment(segment: string): boolean { if (isPunctuationGlueCluster(segment)) return true let sawPunctuation = false for (const ch of segment) { if (leftStickyPunctuation.has(ch) || isLineBreakNumericAffix(ch)) { sawPunctuation = true continue } if (sawPunctuation && combiningMarkRe.test(ch)) continue return false } return sawPunctuation } function isCJKLineStartProhibitedSegment(segment: string): boolean { for (const ch of segment) { if (!kinsokuStart.has(ch) && !leftStickyPunctuation.has(ch)) return false } return segment.length > 0 } function isForwardStickyClusterSegment(segment: string): boolean { if (isPunctuationGlueCluster(segment)) return true for (const ch of segment) { if ( !kinsokuEnd.has(ch) && !forwardStickyGlue.has(ch) && !combiningMarkRe.test(ch) && !isLineBreakNumericAffix(ch) ) { return false } } return segment.length > 0 } function isPunctuationGlueCluster(segment: string): boolean { let sawPunctuation = false for (const ch of segment) { if (ch === '\\' || combiningMarkRe.test(ch)) continue if (kinsokuEnd.has(ch) || leftStickyPunctuation.has(ch) || forwardStickyGlue.has(ch)) { sawPunctuation = true continue } return false } return sawPunctuation } function previousCodePointStart(text: string, end: number): number { const last = end - 1 if (last <= 0) return Math.max(last, 0) const lastCodeUnit = text.charCodeAt(last) if (lastCodeUnit < 0xDC00 || lastCodeUnit > 0xDFFF) return last const maybeHigh = last - 1 if (maybeHigh < 0) return last const highCodeUnit = text.charCodeAt(maybeHigh) return highCodeUnit >= 0xD800 && highCodeUnit <= 0xDBFF ? maybeHigh : last } function getLastCodePoint(text: string): string | null { if (text.length === 0) return null const start = previousCodePointStart(text, text.length) return text.slice(start) } function getFirstSignificantCodePoint(text: string): string | null { for (const ch of text) { if (!combiningMarkRe.test(ch)) return ch } return null } function getLastSignificantCodePoint(text: string, end = text.length): string | null { for (; end > 0;) { const start = previousCodePointStart(text, end) const ch = text.slice(start, end) if (!combiningMarkRe.test(ch)) return ch end = start } return null } // Unicode line-break PR/PO classes from UAX #14, stored as start/end pairs. const lineBreakNumericAffixRanges = [ 0x0024, 0x0025, 0x002B, 0x002B, 0x005C, 0x005C, 0x00A2, 0x00A5, 0x00B0, 0x00B1, 0x058F, 0x058F, 0x0609, 0x060B, 0x066A, 0x066A, 0x07FE, 0x07FF, 0x09F2, 0x09F3, 0x09F9, 0x09FB, 0x0AF1, 0x0AF1, 0x0BF9, 0x0BF9, 0x0D79, 0x0D79, 0x0E3F, 0x0E3F, 0x17DB, 0x17DB, 0x2030, 0x2037, 0x2057, 0x2057, 0x20A0, 0x20CF, 0x2103, 0x2103, 0x2109, 0x2109, 0x2116, 0x2116, 0x2212, 0x2213, 0xA838, 0xA838, 0xFDFC, 0xFDFC, 0xFE69, 0xFE6A, 0xFF04, 0xFF05, 0xFFE0, 0xFFE1, 0xFFE5, 0xFFE6, 0x11FDD, 0x11FE0, 0x1E2FF, 0x1E2FF, 0x1ECAC, 0x1ECAC, 0x1ECB0, 0x1ECB0, ] as const function isCodePointInRanges(codePoint: number, ranges: readonly number[]): boolean { for (let i = 0; i < ranges.length; i += 2) { if (codePoint >= ranges[i]! && codePoint <= ranges[i + 1]!) return true } return false } function isLineBreakNumericAffix(ch: string): boolean { const codePoint = ch.codePointAt(0) return codePoint !== undefined && isCodePointInRanges(codePoint, lineBreakNumericAffixRanges) } function endsWithLineBreakNumericAffix(text: string): boolean { const last = getLastSignificantCodePoint(text) return last !== null && isLineBreakNumericAffix(last) } function startsWithDecimalDigit(text: string): boolean { const first = getFirstSignificantCodePoint(text) return first !== null && decimalDigitRe.test(first) } function splitTrailingForwardStickyCluster(text: string): { head: string, tail: string } | null { let splitIndex = text.length while (splitIndex > 0) { const start = previousCodePointStart(text, splitIndex) const ch = text.slice(start, splitIndex) if (combiningMarkRe.test(ch) || kinsokuEnd.has(ch) || forwardStickyGlue.has(ch)) { splitIndex = start continue } break } if (splitIndex <= 0 || splitIndex === text.length) return null return { head: text.slice(0, splitIndex), tail: text.slice(splitIndex), } } function getRepeatableSingleCharRunChar( text: string, isWordLike: boolean, kind: SegmentBreakKind, ): string | null { return kind === 'text' && !isWordLike && text.length === 1 && text !== '-' && text !== '—' ? text : null } function hasArabicNoSpacePunctuation( containsArabic: boolean, lastCodePoint: string | null, ): boolean { return containsArabic && lastCodePoint !== null && arabicNoSpaceTrailingPunctuation.has(lastCodePoint) } function endsWithMyanmarMedialGlue(segment: string): boolean { const lastCodePoint = getLastCodePoint(segment) return lastCodePoint !== null && myanmarMedialGlue.has(lastCodePoint) } function splitLeadingSpaceAndMarks(segment: string): { space: string, marks: string } | null { if (segment.length < 2 || segment[0] !== ' ') return null const marks = segment.slice(1) if (/^\p{M}+$/u.test(marks)) { return { space: ' ', marks } } return null } export function endsWithClosingQuote(text: string): boolean { let end = text.length while (end > 0) { const start = previousCodePointStart(text, end) const ch = text.slice(start, end) if (closingQuoteChars.has(ch)) return true if (!leftStickyPunctuation.has(ch)) return false end = start } return false } function classifySegmentBreakChar(ch: string, whiteSpace: WhiteSpaceMode): SegmentBreakKind { if (whiteSpace === 'pre-wrap') { if (ch === ' ') return 'preserved-space' if (ch === '\t') return 'tab' if (ch === '\n') return 'hard-break' } if (ch === ' ') return 'space' if (ch === '\u00A0' || ch === '\u202F' || ch === '\u2060' || ch === '\uFEFF') { return 'glue' } if (ch === '\u200B') return 'zero-width-break' if (ch === '\u00AD') return 'soft-hyphen' return 'text' } // All characters that classifySegmentBreakChar maps to a non-'text' kind. const breakCharRe = /[\x20\t\n\xA0\xAD\u200B\u202F\u2060\uFEFF]/ function joinTextParts(parts: string[]): string { return parts.length === 1 ? parts[0]! : parts.join('') } function joinReversedPrefixParts(prefixParts: string[], tail: string): string { const parts: string[] = [] for (let i = prefixParts.length - 1; i >= 0; i--) { parts.push(prefixParts[i]!) } parts.push(tail) return joinTextParts(parts) } function splitSegmentByBreakKind( segment: string, isWordLike: boolean, start: number, whiteSpace: WhiteSpaceMode, ): SegmentationPiece[] { if (!breakCharRe.test(segment)) { return [{ text: segment, isWordLike, kind: 'text', start }] } const pieces: SegmentationPiece[] = [] let currentKind: SegmentBreakKind | null = null let currentStart = 0 let currentWordLike = false let offset = 0 for (const ch of segment) { const kind = classifySegmentBreakChar(ch, whiteSpace) const wordLike = kind === 'text' && isWordLike if (currentKind !== null && kind === currentKind && wordLike === currentWordLike) { offset += ch.length continue } if (currentKind !== null) { pieces.push({ text: segment.slice(currentStart, offset), isWordLike: currentWordLike, kind: currentKind, start: start + currentStart, }) } currentKind = kind currentStart = offset currentWordLike = wordLike offset += ch.length } if (currentKind !== null) { pieces.push({ text: segment.slice(currentStart), isWordLike: currentWordLike, kind: currentKind, start: start + currentStart, }) } return pieces } function isTextRunBoundary(kind: SegmentBreakKind): boolean { return ( kind === 'space' || kind === 'preserved-space' || kind === 'zero-width-break' || kind === 'hard-break' ) } const urlSchemeSegmentRe = /^[A-Za-z][A-Za-z0-9+.-]*:$/ function isUrlLikeRunStart(segmentation: MergedSegmentation, index: number): boolean { const text = segmentation.texts[index]! if (text.startsWith('www.')) return true return ( urlSchemeSegmentRe.test(text) && index + 1 < segmentation.len && segmentation.kinds[index + 1] === 'text' && segmentation.texts[index + 1] === '//' ) } function isUrlQueryBoundarySegment(text: string): boolean { return text.includes('?') && (text.includes('://') || text.startsWith('www.')) } function mergeUrlRuns(segmentation: MergedSegmentation, normalized: string, profile: AnalysisProfile): MergedSegmentation { const texts: string[] = [] const isWordLike: boolean[] = [] const kinds: SegmentBreakKind[] = [] const starts: number[] = [] for (let i = 0; i < segmentation.len; i++) { const start = segmentation.starts[i]! let text = segmentation.texts[i]! let wordLike = segmentation.isWordLike[i]! const kind = segmentation.kinds[i]! let queryStartOverride = -1 if (kind === 'text' && isUrlLikeRunStart(segmentation, i)) { const urlParts = [text] let j = i + 1 while ( j < segmentation.len && !isTextRunBoundary(segmentation.kinds[j]!) && numericAffixBoundary(normalized, segmentation.starts[j]!, profile) !== false ) { if (queryStartOverride < 0 && isUrlLikeRunStart(segmentation, j)) { queryStartOverride = segmentation.starts[j]! } const nextText = segmentation.texts[j]! urlParts.push(nextText) wordLike = true j++ if (nextText.includes('?')) break } text = joinTextParts(urlParts) i = j - 1 } texts.push(text) isWordLike.push(wordLike) kinds.push(kind) starts.push(start) if (!isUrlQueryBoundarySegment(text)) continue const nextIndex = i + 1 if ( nextIndex >= segmentation.len || isTextRunBoundary(segmentation.kinds[nextIndex]!) ) { continue } const queryParts: string[] = [] const queryStart = queryStartOverride < 0 ? segmentation.starts[nextIndex]! : queryStartOverride let j = nextIndex while ( j < segmentation.len && !isTextRunBoundary(segmentation.kinds[j]!) && numericAffixBoundary(normalized, segmentation.starts[j]!, profile) !== false ) { queryParts.push(segmentation.texts[j]!) j++ } if (queryParts.length > 0) { texts.push(joinTextParts(queryParts)) isWordLike.push(true) kinds.push('text') starts.push(queryStart) i = j - 1 } } return { len: texts.length, texts, isWordLike, kinds, starts, } } const numericJoinerChars = new Set([ ':', '-', '/', '×', ',', '.', '+', '\u2013', '\u2014', ]) const wordInternalSymbolRe = /[\p{P}\p{S}\p{Co}]/u const emojiPresentationRe = /\p{Emoji_Presentation}/u const noSpaceWordBreakAfterChars = new Set([ '?', '\u058A', '-', '\u2010', '\u2012', '\u2013', '\u2014', '\u2026', '\u203C', '\u203D', '\u2049', ]) function isAsciiWordInternalSymbolCode(code: number): boolean { return ( (code >= 0x21 && code <= 0x2F && code !== 0x2D) || (code >= 0x3A && code <= 0x40 && code !== 0x3F) || (code >= 0x5B && code <= 0x60) || (code >= 0x7B && code <= 0x7E) ) } function isNoSpaceWordInternalSymbol(ch: string): boolean { const code = ch.charCodeAt(0) if (code < 0x80) return isAsciiWordInternalSymbolCode(code) return ( !noSpaceWordBreakAfterChars.has(ch) && !emojiPresentationRe.test(ch) && wordInternalSymbolRe.test(ch) ) } function isNoSpaceWordInternalSymbolSegment(text: string): boolean { let sawSymbol = false for (const ch of text) { if (combiningMarkRe.test(ch)) continue if (!isNoSpaceWordInternalSymbol(ch)) return false sawSymbol = true } return sawSymbol } function endsWithNoSpaceWordJoiner(text: string): boolean { for (let end = text.length; end > 0;) { const start = previousCodePointStart(text, end) const ch = text.slice(start, end) if (combiningMarkRe.test(ch)) { end = start continue } return isNoSpaceWordInternalSymbol(ch) || isLineBreakNumericAffix(ch) } return false } const asciiAlphabeticBoundaryRe = /[A-Za-z#&*<=>@^_`~]/ function isAsciiBoundary(left: string, right: string): boolean { return left.charCodeAt(0) < 0x80 && right.charCodeAt(0) < 0x80 } // The observed Gecko ASCII model owns directional PR/PO seams and ordinary // opener attachment. Unicode neighbors retain the existing compatibility tier. function numericAffixBoundary(source: string, boundary: number, profile: AnalysisProfile): boolean | null { if (!profile.geckoAsciiLineBreaks || boundary <= 0 || boundary >= source.length) return null const left = getLastSignificantCodePoint(source, boundary) const right = String.fromCodePoint(source.codePointAt(boundary)!) if (left === null || !isAsciiBoundary(left, right)) return null const leftAffix = isLineBreakNumericAffix(left) const rightAffix = isLineBreakNumericAffix(right) if (!leftAffix && !rightAffix) return null if (right === ')' || right === ']' || right === '}' || '!,.:;?/'.includes(right)) return true if ('([{'.includes(left) || left === '"' || left === "'" || right === '"' || right === "'") return true if (right === '|' || right === '-') return true if (leftAffix && asciiAlphabeticBoundaryRe.test(right) || asciiAlphabeticBoundaryRe.test(left) && rightAffix) return true if (rightAffix && (')]}'.includes(left) || /[0-9]/.test(left))) return true if (leftAffix && ('([{'.includes(right) || /[0-9]/.test(right))) return true return false } // Browser line breakers tailor ASCII opener boundaries beyond Unicode classes. // Preserve that boundary before symbol-chain compaction can erase it. function openingPunctuationJoinsPrevious(left: string, right: string, profile: AnalysisProfile, leftEnd = left.length): boolean | null { const first = right[0] if (first === undefined || !'([{'.includes(first)) return null const last = getLastSignificantCodePoint(left, leftEnd) if (last === null) return null if (profile.geckoAsciiLineBreaks && isAsciiBoundary(last, first)) { return asciiAlphabeticBoundaryRe.test(last) || decimalDigitRe.test(last) || '"\'([{'.includes(last) || isLineBreakNumericAffix(last) } if (last.charCodeAt(0) >= 0x80) { // Non-CJK letters and numbers retain alphabetic opener attachment (LB30). return !isCJK(last) && /[\p{L}\p{N}]/u.test(last) ? true : null } return /[A-Za-z0-9]/.test(last) || "$'(/<@[^_`{".includes(last) } function canJoinNoSpaceWordBoundary( leftText: string, leftWordLike: boolean, rightText: string, rightWordLike: boolean, profile: AnalysisProfile, ): boolean { // The forward-sticky pass joins a sign to its numeric suffix. Preserve its // CJK left edge so final unit construction can place the ordinary boundary. if (rightText[0] === '-' && isCJK(leftText)) return true // CJK-leading mixed runs retain their own annotation/kinsoku boundaries, // even when the last scalar before an opener is an ASCII letter. if (isCJK(leftText) || isCJK(rightText)) return false const openingJoin = openingPunctuationJoinsPrevious(leftText, rightText, profile) if (openingJoin !== null) return openingJoin const leftSymbol = !leftWordLike && isNoSpaceWordInternalSymbolSegment(leftText) const rightSymbol = !rightWordLike && isNoSpaceWordInternalSymbolSegment(rightText) const leftAffix = endsWithLineBreakNumericAffix(leftText) const leftEndsJoiner = (leftWordLike || leftAffix) && endsWithNoSpaceWordJoiner(leftText) if (!leftSymbol && !rightSymbol && !leftEndsJoiner) return false return (leftWordLike || leftSymbol || leftAffix) && (rightWordLike || rightSymbol) } function segmentContainsDecimalDigit(text: string): boolean { for (const ch of text) { if (decimalDigitRe.test(ch)) return true } return false } export function isNumericRunSegment(text: string): boolean { if (text.length === 0) return false for (const ch of text) { if (decimalDigitRe.test(ch) || numericJoinerChars.has(ch)) continue return false } return true } function mergeNumericRuns(segmentation: MergedSegmentation, normalized: string, profile: AnalysisProfile): MergedSegmentation { const texts: string[] = [] const isWordLike: boolean[] = [] const kinds: SegmentBreakKind[] = [] const starts: number[] = [] function pushNumericRun(text: string, start: number): void { if (text.includes('-')) { const parts = text.split('-') let shouldSplit = parts.length > 1 for (let i = 0; i < parts.length; i++) { const part = parts[i]! if (!shouldSplit) break if ( part.length === 0 || !segmentContainsDecimalDigit(part) || !isNumericRunSegment(part) ) { shouldSplit = false } } if (shouldSplit) { let offset = 0 for (let i = 0; i < parts.length; i++) { const part = parts[i]! const splitText = i < parts.length - 1 ? `${part}-` : part texts.push(splitText) isWordLike.push(true) kinds.push('text') starts.push(start + offset) offset += splitText.length } return } } texts.push(text) isWordLike.push(true) kinds.push('text') starts.push(start) } for (let i = 0; i < segmentation.len; i++) { const text = segmentation.texts[i]! const kind = segmentation.kinds[i]! if (kind === 'text' && isNumericRunSegment(text) && segmentContainsDecimalDigit(text)) { const mergedParts = [text] let j = i + 1 while ( j < segmentation.len && segmentation.kinds[j] === 'text' && isNumericRunSegment(segmentation.texts[j]!) && numericAffixBoundary(normalized, segmentation.starts[j]!, profile) !== false ) { mergedParts.push(segmentation.texts[j]!) j++ } pushNumericRun(joinTextParts(mergedParts), segmentation.starts[i]!) i = j - 1 continue } texts.push(text) isWordLike.push(segmentation.isWordLike[i]!) kinds.push(kind) starts.push(segmentation.starts[i]!) } return { len: texts.length, texts, isWordLike, kinds, starts, } } function mergeNoSpaceWordChains(segmentation: MergedSegmentation, normalized: string, profile: AnalysisProfile): MergedSegmentation { const texts: string[] = [] const isWordLike: boolean[] = [] const kinds: SegmentBreakKind[] = [] const starts: number[] = [] let i = 0 while (i < segmentation.len) { const text = segmentation.texts[i]! const kind = segmentation.kinds[i]! const wordLike = segmentation.isWordLike[i]! if (kind === 'text') { const mergedParts = [text] let j = i + 1 let mergedWordLike = wordLike while ( j < segmentation.len && segmentation.kinds[j] === 'text' && (numericAffixBoundary(normalized, segmentation.starts[j]!, profile) ?? canJoinNoSpaceWordBoundary( segmentation.texts[j - 1]!, segmentation.isWordLike[j - 1]!, segmentation.texts[j]!, segmentation.isWordLike[j]!, profile, )) ) { const nextText = segmentation.texts[j]! mergedParts.push(nextText) mergedWordLike = mergedWordLike || segmentation.isWordLike[j]! j++ } if (j > i + 1) { texts.push(joinTextParts(mergedParts)) isWordLike.push(mergedWordLike) kinds.push('text') starts.push(segmentation.starts[i]!) i = j continue } } texts.push(text) isWordLike.push(wordLike) kinds.push(kind) starts.push(segmentation.starts[i]!) i++ } return { len: texts.length, texts, isWordLike, kinds, starts, } } function mergeGlueConnectedTextRuns(segmentation: MergedSegmentation): MergedSegmentation { const texts: string[] = [] const isWordLike: boolean[] = [] const kinds: SegmentBreakKind[] = [] const starts: number[] = [] let read = 0 while (read < segmentation.len) { const textParts = [segmentation.texts[read]!] let wordLike = segmentation.isWordLike[read]! let kind = segmentation.kinds[read]! let start = segmentation.starts[read]! if (kind === 'glue') { const glueParts = [textParts[0]!] const glueStart = start read++ while (read < segmentation.len && segmentation.kinds[read] === 'glue') { glueParts.push(segmentation.texts[read]!) read++ } const glueText = joinTextParts(glueParts) if (read < segmentation.len && segmentation.kinds[read] === 'text') { textParts[0] = glueText textParts.push(segmentation.texts[read]!) wordLike = segmentation.isWordLike[read]! kind = 'text' start = glueStart read++ } else { texts.push(glueText) isWordLike.push(false) kinds.push('glue') starts.push(glueStart) continue } } else { read++ } if (kind === 'text') { while (read < segmentation.len && segmentation.kinds[read] === 'glue') { const glueParts: string[] = [] while (read < segmentation.len && segmentation.kinds[read] === 'glue') { glueParts.push(segmentation.texts[read]!) read++ } const glueText = joinTextParts(glueParts) if (read < segmentation.len && segmentation.kinds[read] === 'text') { textParts.push(glueText, segmentation.texts[read]!) wordLike = wordLike || segmentation.isWordLike[read]! read++ continue } textParts.push(glueText) } } texts.push(joinTextParts(textParts)) isWordLike.push(wordLike) kinds.push(kind) starts.push(start) } return { len: texts.length, texts, isWordLike, kinds, starts, } } function carryTrailingForwardStickyAcrossCJKBoundary(segmentation: MergedSegmentation): void { const { texts, kinds, starts } = segmentation for (let i = 0; i < texts.length - 1; i++) { if (kinds[i] !== 'text' || kinds[i + 1] !== 'text') continue if (!isCJK(texts[i]!) || !isCJK(texts[i + 1]!)) continue const split = splitTrailingForwardStickyCluster(texts[i]!) if (split === null) continue texts[i] = split.head texts[i + 1] = split.tail + texts[i + 1]! starts[i + 1] = starts[i]! + split.head.length } } function buildMergedSegmentation( normalized: string, profile: AnalysisProfile, whiteSpace: WhiteSpaceMode, ): MergedSegmentation { const wordSegmenter = getSharedWordSegmenter() let mergedLen = 0 const mergedTexts: string[] = [] const mergedWordLike: boolean[] = [] const mergedKinds: SegmentBreakKind[] = [] const mergedStarts: number[] = [] // First-pass merges only extend the immediately adjacent text run. Keep that // live tail as a source range, then materialize it once at the next boundary. let hasTail = false let tailStart = 0 let tailEnd = 0 let tailWordLike = false let tailKind: SegmentBreakKind = 'text' let tailSingleCharRunChar: string | null = null let tailContainsCJK = false let tailContainsArabicScript = false let tailEndsWithClosingQuote = false let tailEndsWithMyanmarMedialGlue = false let tailHasArabicNoSpacePunctuation = false for (const s of wordSegmenter.segment(normalized)) { for (const piece of splitSegmentByBreakKind(s.segment, s.isWordLike ?? false, s.index, whiteSpace)) { const isText = piece.kind === 'text' const repeatableSingleCharRunChar = getRepeatableSingleCharRunChar(piece.text, piece.isWordLike, piece.kind) const pieceContainsCJK = isCJK(piece.text) const pieceContainsArabicScript = containsArabicScript(piece.text) const pieceLastCodePoint = getLastCodePoint(piece.text) const pieceEndsWithClosingQuote = endsWithClosingQuote(piece.text) const pieceEndsWithMyanmarMedialGlue = endsWithMyanmarMedialGlue(piece.text) const pieceEnd = piece.start + piece.text.length const boundaryJoin = numericAffixBoundary(normalized, piece.start, profile) ?? (tailContainsCJK || pieceContainsCJK ? null : openingPunctuationJoinsPrevious(normalized, piece.text, profile, piece.start)) let appendToTail = false // First-pass keeps: no-space script-specific joins and punctuation glue // that depend on the immediately preceding text run. if ( profile.carryCJKAfterClosingQuote && isText && hasTail && tailKind === 'text' && pieceContainsCJK && tailContainsCJK && tailEndsWithClosingQuote ) { appendToTail = true } else if ( isText && hasTail && tailKind === 'text' && isCJKLineStartProhibitedSegment(piece.text) && tailContainsCJK ) { appendToTail = true } else if ( isText && hasTail && tailKind === 'text' && tailEndsWithMyanmarMedialGlue ) { appendToTail = true } else if ( isText && hasTail && tailKind === 'text' && piece.isWordLike && pieceContainsArabicScript && tailHasArabicNoSpacePunctuation ) { appendToTail = true } else if ( repeatableSingleCharRunChar !== null && hasTail && tailKind === 'text' && tailSingleCharRunChar === repeatableSingleCharRunChar && boundaryJoin !== false ) { tailEnd = pieceEnd continue } else if ( isText && !piece.isWordLike && hasTail && tailKind === 'text' && !tailContainsCJK && ( isLeftStickyPunctuationSegment(piece.text) || (piece.text === '-' && tailWordLike) ) ) { appendToTail = true } if (isText && hasTail && tailKind === 'text' && boundaryJoin !== null) appendToTail = boundaryJoin if (appendToTail) { tailEnd = pieceEnd tailWordLike = tailWordLike || piece.isWordLike tailSingleCharRunChar = null tailContainsCJK = tailContainsCJK || pieceContainsCJK tailContainsArabicScript = tailContainsArabicScript || pieceContainsArabicScript tailEndsWithClosingQuote = pieceEndsWithClosingQuote tailEndsWithMyanmarMedialGlue = pieceEndsWithMyanmarMedialGlue tailHasArabicNoSpacePunctuation = hasArabicNoSpacePunctuation( tailContainsArabicScript, pieceLastCodePoint, ) } else { if (hasTail) { mergedTexts[mergedLen] = normalized.slice(tailStart, tailEnd) mergedWordLike[mergedLen] = tailWordLike mergedKinds[mergedLen] = tailKind mergedStarts[mergedLen] = tailStart mergedLen++ } hasTail = true tailStart = piece.start tailEnd = pieceEnd tailWordLike = piece.isWordLike tailKind = piece.kind tailSingleCharRunChar = repeatableSingleCharRunChar tailContainsCJK = pieceContainsCJK tailContainsArabicScript = pieceContainsArabicScript tailEndsWithClosingQuote = pieceEndsWithClosingQuote tailEndsWithMyanmarMedialGlue = pieceEndsWithMyanmarMedialGlue tailHasArabicNoSpacePunctuation = hasArabicNoSpacePunctuation( pieceContainsArabicScript, pieceLastCodePoint, ) } } } if (hasTail) { mergedTexts[mergedLen] = normalized.slice(tailStart, tailEnd) mergedWordLike[mergedLen] = tailWordLike mergedKinds[mergedLen] = tailKind mergedStarts[mergedLen] = tailStart mergedLen++ } // Later passes operate on the merged text stream itself: contextual escaped // quote glue, forward-sticky carry, compaction, then the broader URL/numeric // and Arabic-leading-mark fixes. for (let i = 1; i < mergedLen; i++) { if ( mergedKinds[i] === 'text' && !mergedWordLike[i]! && isPunctuationGlueCluster(mergedTexts[i]!) && mergedKinds[i - 1] === 'text' && !isCJK(mergedTexts[i - 1]!) && (numericAffixBoundary(normalized, mergedStarts[i]!, profile) ?? openingPunctuationJoinsPrevious(normalized, mergedTexts[i]!, profile, mergedStarts[i]!)) !== false ) { mergedTexts[i - 1] += mergedTexts[i]! mergedWordLike[i - 1] = mergedWordLike[i - 1]! || mergedWordLike[i]! mergedTexts[i] = '' } } let nextLiveIndex = -1 let forwardStickyPrefixParts: string[] | null = null for (let i = mergedLen - 1; i >= 0; i--) { const text = mergedTexts[i]! if (text.length === 0) continue const nextText = forwardStickyPrefixParts?.at(-1) ?? (nextLiveIndex >= 0 ? mergedTexts[nextLiveIndex]! : null) if ( mergedKinds[i] === 'text' && !mergedWordLike[i]! && nextText !== null && mergedKinds[nextLiveIndex] === 'text' && ( (numericAffixBoundary(normalized, mergedStarts[i]! + text.length, profile) ?? (isForwardStickyClusterSegment(text) && openingPunctuationJoinsPrevious(text, nextText, profile) !== false)) || (text === '-' && startsWithDecimalDigit(nextText)) ) ) { if (forwardStickyPrefixParts === null) forwardStickyPrefixParts = [] forwardStickyPrefixParts.push(text) mergedStarts[nextLiveIndex] = mergedStarts[i]! mergedTexts[i] = '' continue } if (forwardStickyPrefixParts !== null) { mergedTexts[nextLiveIndex] = joinReversedPrefixParts( forwardStickyPrefixParts, mergedTexts[nextLiveIndex]!, ) forwardStickyPrefixParts = null } nextLiveIndex = i } if (forwardStickyPrefixParts !== null) { mergedTexts[nextLiveIndex] = joinReversedPrefixParts( forwardStickyPrefixParts, mergedTexts[nextLiveIndex]!, ) } let compactLen = 0 for (let read = 0; read < mergedLen; read++) { const text = mergedTexts[read]! if (text.length === 0) continue if (compactLen !== read) { mergedTexts[compactLen] = text mergedWordLike[compactLen] = mergedWordLike[read]! mergedKinds[compactLen] = mergedKinds[read]! mergedStarts[compactLen] = mergedStarts[read]! } compactLen++ } mergedTexts.length = compactLen mergedWordLike.length = compactLen mergedKinds.length = compactLen mergedStarts.length = compactLen const compacted = mergeGlueConnectedTextRuns({ len: compactLen, texts: mergedTexts, isWordLike: mergedWordLike, kinds: mergedKinds, starts: mergedStarts, }) const mergedRuns = mergeNoSpaceWordChains(mergeNumericRuns(mergeUrlRuns(compacted, normalized, profile), normalized, profile), normalized, profile) carryTrailingForwardStickyAcrossCJKBoundary(mergedRuns) for (let i = 0; i < mergedRuns.len - 1; i++) { const split = splitLeadingSpaceAndMarks(mergedRuns.texts[i]!) if (split === null) continue if ( (mergedRuns.kinds[i] !== 'space' && mergedRuns.kinds[i] !== 'preserved-space') || mergedRuns.kinds[i + 1] !== 'text' || !containsArabicScript(mergedRuns.texts[i + 1]!) ) { continue } mergedRuns.texts[i] = split.space mergedRuns.isWordLike[i] = false mergedRuns.kinds[i] = mergedRuns.kinds[i] === 'preserved-space' ? 'preserved-space' : 'space' mergedRuns.texts[i + 1] = split.marks + mergedRuns.texts[i + 1]! mergedRuns.starts[i + 1] = mergedRuns.starts[i]! + split.space.length } return mergedRuns } function mergeKeepAllTextSegments( normalized: string, segmentation: MergedSegmentation, profile: AnalysisProfile, ): MergedSegmentation { if (segmentation.len <= 1) return segmentation const texts: string[] = [] const isWordLike: boolean[] = [] const kinds: SegmentBreakKind[] = [] const starts: number[] = [] let groupStart = -1 let groupContainsCJK = false function pushOriginalText(index: number): void { texts.push(segmentation.texts[index]!) isWordLike.push(segmentation.isWordLike[index]!) kinds.push('text') starts.push(segmentation.starts[index]!) } function pushMergedText(start: number, end: number): void { let wordLike = false for (let i = start; i < end; i++) { wordLike = wordLike || segmentation.isWordLike[i]! } const sourceStart = segmentation.starts[start]! const sourceEnd = end < segmentation.len ? segmentation.starts[end]! : normalized.length texts.push(normalized.slice(sourceStart, sourceEnd)) isWordLike.push(wordLike) kinds.push('text') starts.push(sourceStart) } function flushGroup(end: number): void { if (groupStart < 0) return if (groupContainsCJK) { if (groupStart + 1 === end) { pushOriginalText(groupStart) } else { pushMergedText(groupStart, end) } } else { for (let i = groupStart; i < end; i++) pushOriginalText(i) } groupStart = -1 groupContainsCJK = false } for (let i = 0; i < segmentation.len; i++) { const text = segmentation.texts[i]! const kind = segmentation.kinds[i]! if (kind === 'text') { if ( groupStart >= 0 && (!canContinueKeepAllTextRun(segmentation.texts[i - 1]!, profile.breakKeepAllAfterPunctuation) || numericAffixBoundary(normalized, segmentation.starts[i]!, profile) === false) ) { flushGroup(i) } if (groupStart < 0) groupStart = i groupContainsCJK = groupContainsCJK || isCJK(text) continue } flushGroup(i) texts.push(text) isWordLike.push(segmentation.isWordLike[i]!) kinds.push(kind) starts.push(segmentation.starts[i]!) } flushGroup(segmentation.len) return { len: texts.length, texts, isWordLike, kinds, starts, } } // A numeric sign stays with its number. Latin letter/number hyphens remain // preferred boundaries; CJK-adjacent signs also stay attached on their left. function isNumericHyphen(text: string, index: number): boolean { const next = text.charCodeAt(index + 1) if (next < 0x80) { if (next < 0x30 || next > 0x39) return false } else { const codePoint = text.codePointAt(index + 1) if (codePoint === undefined || !decimalDigitRe.test(String.fromCodePoint(codePoint))) return false } for (let end = index; end > 0;) { const start = previousCodePointStart(text, end) const previous = text.slice(start, end) if (!combiningMarkRe.test(previous)) return isCJK(previous) || !/[\p{L}\p{N}]/u.test(previous) end = start } return true } type TextBreakUnit = { text: string start: number overflow: 'none' | 'word-like' | 'grapheme' } function buildBaseCjkUnits( segText: string, profile: AnalysisProfile, ): TextBreakUnit[] { const units: TextBreakUnit[] = [] let unitStart = 0 let unitEnd = 0 let unitContainsCJK = false let unitEndsWithClosingQuote = false let unitIsSingleKinsokuEnd = false let unitHasHyphen = false let unitHasNumericHyphen = false function pushUnit(): void { if (unitEnd === unitStart) return units.push({ text: segText.slice(unitStart, unitEnd), start: unitStart, overflow: unitContainsCJK ? (unitHasHyphen ? 'grapheme' : 'none') : 'word-like', }) unitStart = unitEnd unitContainsCJK = false unitEndsWithClosingQuote = false unitIsSingleKinsokuEnd = false unitHasHyphen = false unitHasNumericHyphen = false } function startUnit(grapheme: string, start: number, graphemeContainsCJK: boolean): void { unitStart = start unitEnd = start + grapheme.length unitContainsCJK = graphemeContainsCJK unitHasHyphen = grapheme === '-' unitEndsWithClosingQuote = endsWithClosingQuote(grapheme) unitIsSingleKinsokuEnd = kinsokuEnd.has(grapheme) } function appendToUnit(grapheme: string, graphemeContainsCJK: boolean): void { unitEnd += grapheme.length unitContainsCJK = unitContainsCJK || graphemeContainsCJK unitHasHyphen = unitHasHyphen || grapheme === '-' const graphemeEndsWithClosingQuote = endsWithClosingQuote(grapheme) if (grapheme.length === 1 && leftStickyPunctuation.has(grapheme)) { unitEndsWithClosingQuote = unitEndsWithClosingQuote || graphemeEndsWithClosingQuote } else { unitEndsWithClosingQuote = graphemeEndsWithClosingQuote } unitIsSingleKinsokuEnd = false } for (const gs of getSharedGraphemeSegmenter().segment(segText)) { const grapheme = gs.segment const graphemeContainsCJK = isCJK(grapheme) if (unitEnd === unitStart) { startUnit(grapheme, gs.index, graphemeContainsCJK) continue } const attachHyphen = grapheme === '-' && unitContainsCJK if (attachHyphen && isNumericHyphen(segText, gs.index)) unitHasNumericHyphen = true if ( unitIsSingleKinsokuEnd || kinsokuStart.has(grapheme) || leftStickyPunctuation.has(grapheme) || attachHyphen || (unitHasNumericHyphen && !graphemeContainsCJK) || (profile.carryCJKAfterClosingQuote && graphemeContainsCJK && unitEndsWithClosingQuote) ) { appendToUnit(grapheme, graphemeContainsCJK) continue } if (!unitContainsCJK && !graphemeContainsCJK) { appendToUnit(grapheme, graphemeContainsCJK) continue } pushUnit() startUnit(grapheme, gs.index, graphemeContainsCJK) } pushUnit() return units } function mergeKeepAllTextUnits( segText: string, units: TextBreakUnit[], profile: AnalysisProfile, ): TextBreakUnit[] { if (units.length <= 1) return units const merged: TextBreakUnit[] = [] let groupStart = -1 let groupContainsCJK = false function pushMergedUnit(start: number, end: number): void { const sourceStart = units[start]!.start const sourceEnd = end < units.length ? units[end]!.start : segText.length merged.push({ text: segText.slice(sourceStart, sourceEnd), start: sourceStart, overflow: 'word-like', }) } function flushGroup(end: number): void { if (groupStart < 0) return if (groupContainsCJK) { if (groupStart + 1 === end) { merged.push(units[groupStart]!) } else { pushMergedUnit(groupStart, end) } } else { for (let i = groupStart; i < end; i++) merged.push(units[i]!) } groupStart = -1 groupContainsCJK = false } for (let i = 0; i < units.length; i++) { const unit = units[i]! if ( groupStart >= 0 && (!canContinueKeepAllTextRun(units[i - 1]!.text, profile.breakKeepAllAfterPunctuation) || numericAffixBoundary(segText, unit.start, profile) === false) ) { flushGroup(i) } if (groupStart < 0) groupStart = i groupContainsCJK = groupContainsCJK || isCJK(unit.text) } flushGroup(units.length) return merged } // Ordinary CJK boundaries and emergency overflow permission are separate facts. // Keep these decisions in preprocessing; measurement only observes their units. export function getCjkTextUnits(text: string, profile: AnalysisProfile, wordBreak: WordBreakMode): TextBreakUnit[] { const units = buildBaseCjkUnits(text, profile) return wordBreak === 'keep-all' ? mergeKeepAllTextUnits(text, units, profile) : units } function isPreferredBreakGrapheme(grapheme: string): boolean { return ( grapheme === '-' || grapheme === '\u058A' || grapheme === '\u2010' || grapheme === '\u2012' || grapheme === '\u2013' || grapheme === '\u2014' ) } // Intl word-likeness is not overflow permission: independent punctuation and // symbol graphemes can also break. Emoji retain their separate ordinary // boundary policy, like no-space compaction above. Control-bearing fragments // and standalone extenders also retain their existing source-shaping policy. export function isIndependentSymbolRun(text: string): boolean { if (text.length === 0 || /\p{Cf}/u.test(text)) return false for (const { segment } of getSharedGraphemeSegmenter().segment(text)) { const base = String.fromCodePoint(segment.codePointAt(0)!) if (!/[\p{P}\p{S}]/u.test(base) || emojiPresentationRe.test(base) || segment.includes('\uFE0F') || /\p{Emoji_Modifier}/u.test(base)) return false } return true } export function getBreakablePreferredBreaks(text: string): number[] | null { if (!/[-\u058A\u2010\u2012\u2013\u2014]/u.test(text)) return null const breaks: number[] = [] let graphemeIndex = 0 for (const gs of getSharedGraphemeSegmenter().segment(text)) { graphemeIndex++ const numericSign = gs.segment === '-' && isNumericHyphen(text, gs.index) if (isPreferredBreakGrapheme(gs.segment) && !numericSign) breaks.push(graphemeIndex) } return breaks.length === 0 ? null : breaks } export function analyzeText( text: string, profile: AnalysisProfile, whiteSpace: WhiteSpaceMode = 'normal', wordBreak: WordBreakMode = 'normal', ): TextAnalysis { const normalized = whiteSpace === 'pre-wrap' ? normalizeWhitespacePreWrap(text) : normalizeWhitespaceNormal(text) if (normalized.length === 0) { return { normalized, len: 0, texts: [], isWordLike: [], kinds: [], starts: [], } } const mergedSegmentation = buildMergedSegmentation(normalized, profile, whiteSpace) const segmentation = wordBreak === 'keep-all' ? mergeKeepAllTextSegments(normalized, mergedSegmentation, profile) : mergedSegmentation return { normalized, ...segmentation, } }