// Port of the pi-fabric word-diff tokenizer (src/ui/word-diff/tokens.ts).
// The port keeps the tokenizer surface the change-block pipeline needs —
// tokens with text ranges and weights for exact-match alignment — and drops
// the full engine's identifier-part splitting and similarity features.

export interface WordToken {
  readonly end: number;
  readonly start: number;
  readonly value: string;
}

const WORD_TOKEN_PATTERN =
  /[$_\p{L}][$_\p{L}\p{N}\p{Mark}]*|\p{N}+(?:\.\p{N}+)?|===|!==|=>|==|!=|<=|>=|&&|\|\||[^\s]/gu;
const IDENTIFIER_TOKEN_PATTERN = /^[$_\p{L}][$_\p{L}\p{N}\p{Mark}]*$/u;
const NUMBER_TOKEN_PATTERN = /^\p{N}+(?:\.\p{N}+)?$/u;
const MEANINGFUL_OPERATOR_TOKEN_PATTERN =
  /^(?:===|!==|=>|==|!=|<=|>=|&&|\|\||[+\-*/%<>=!?:~&|^]+)$/;
const DOMAIN_SEPARATOR_TOKEN_PATTERN = /^[-/:@#]$/;
const STRUCTURAL_PUNCTUATION_TOKEN_PATTERN = /^[{}()[\].,;]$/;

export function wordTokens(text: string): WordToken[] {
  const tokens: WordToken[] = [];
  for (const match of text.matchAll(WORD_TOKEN_PATTERN)) {
    const [value] = match;
    const start = match.index;
    tokens.push({ value, start, end: start + value.length });
  }
  return tokens;
}

export function wordTokenWeight(value: string): number {
  if (IDENTIFIER_TOKEN_PATTERN.test(value)) {
    return 2;
  }
  if (NUMBER_TOKEN_PATTERN.test(value)) {
    return 1.5;
  }
  if (DOMAIN_SEPARATOR_TOKEN_PATTERN.test(value)) {
    return 0.25;
  }
  if (MEANINGFUL_OPERATOR_TOKEN_PATTERN.test(value)) {
    return 1;
  }
  if (STRUCTURAL_PUNCTUATION_TOKEN_PATTERN.test(value)) {
    return 0.05;
  }
  return 1;
}