/** * Subword features for fastText-style embeddings. * * Korean is the weak spot of plain word2vec here: inflected/rare forms * (후킹, 후킹을, 후킹하면) are distinct tokens, most below the frequency floor, so * they get no vector. The fix is to represent every word as the sum of its * character n-grams — and for Hangul, to first DECOMPOSE each syllable into its * jamo so morphologically related words actually share substrings * (후 → ㅎㅜ, 킹 → ㅋㅣㅇ). A rare or even unseen Korean word then still gets a * composed vector from subwords it shares with common words. * * Tokenisation is untouched — subwords are an internal representation only — so * the embedding vocabulary stays aligned with the search vocabulary. */ /** Decompose Hangul syllables to jamo; pass everything else through unchanged. */ export declare function decomposeHangul(text: string): string; /** * The subword set for a token: character n-grams (minN..maxN) over the * jamo-decomposed, boundary-marked word, plus the whole word as a distinguished * feature. Returns deduped feature strings. */ export declare function wordSubwords(word: string, minN?: number, maxN?: number): string[]; /** * Derive a subword vector layer from already-trained word vectors: each subword * vector is the centroid of the (semantic) vectors of in-vocab words that * contain it. This adds coverage for rare / unseen words — their composed vector * is the average of the centroids of the substrings they share with known * words — WITHOUT perturbing the trained word vectors. Subwords occurring in * fewer than `subwordMinCount` words are dropped. */ export declare function buildSubwordCentroids(vocab: string[], vectors: Float32Array, dim: number, minN: number, maxN: number, subwordMinCount: number): { subwordVocab: string[]; subwordVectors: Float32Array; subwordCounts: number[]; };