/** Unicode and identifier tokenization without aliases or translation. @module dsh-tool-retriever/retrieval/tokenizer */

const segmenter = new Intl.Segmenter(undefined, { granularity: 'word' })
const CJK_SEQUENCE = new RegExp(String.raw`[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+`, 'u')
const NON_IDENTIFIER = new RegExp(String.raw`[^\p{L}\p{N}]+`, 'gu')
const MULTI_WHITESPACE = new RegExp(String.raw`\s+`, 'g')
const CAMEL_BOUNDARY = new RegExp(String.raw`([\p{Ll}\p{N}])(\p{Lu})`, 'gu')
const ACRONYM_BOUNDARY = new RegExp(String.raw`([\p{Lu}]+)(\p{Lu}\p{Ll})`, 'gu')
const IDENTIFIER_SEPARATOR = new RegExp('[_-]+', 'g')

/**
 * Normalize a complete identifier for strict name matching.
 * @param value - Raw query or tool name.
 * @returns Lowercase identifier components separated by one space.
 */
export function normalizeIdentifier(value: string): string {
  return expandIdentifier(value)
    .toLocaleLowerCase()
    .replace(NON_IDENTIFIER, ' ')
    .trim()
    .replace(MULTI_WHITESPACE, ' ')
}

/**
 * Tokenize natural language and programming identifiers with deterministic CJK fragments.
 * @param value - Text from a query or tool schema.
 * @returns Tokens in source order, including repeated terms for BM25 term frequency.
 */
export function tokenize(value: string): string[] {
  const normalized = expandIdentifier(value).normalize('NFKC').toLocaleLowerCase()
  const tokens: string[] = []
  for (const part of segmenter.segment(normalized)) {
    if (!part.isWordLike) continue
    const token = part.segment.trim()
    if (!token) continue
    tokens.push(token)
    if (CJK_SEQUENCE.test(token) && [...token].length > 1) {
      const characters = [...token]
      for (let index = 0; index < characters.length - 1; index += 1) {
        const bigram = `${characters[index]}${characters[index + 1]}`
        if (bigram !== token) tokens.push(bigram)
      }
    }
  }
  return tokens
}

/** Split camel case and separator-delimited identifiers before word segmentation. */
function expandIdentifier(value: string): string {
  return value
    .normalize('NFKC')
    .replace(CAMEL_BOUNDARY, '$1 $2')
    .replace(ACRONYM_BOUNDARY, '$1 $2')
    .replace(IDENTIFIER_SEPARATOR, ' ')
}