{"version":3,"file":"perplexity-C16KMToi.cjs","sources":["../src/metrics/perplexity.ts"],"sourcesContent":["import {\n\tAutoModelForCausalLM,\n\tAutoTokenizer,\n\ttype PreTrainedModel,\n\ttype PreTrainedTokenizer,\n\tTensor,\n} from \"@huggingface/transformers\";\nimport { withSpan } from \"../telemetry.js\";\n\nexport interface PerplexityOptions {\n\tmodel?: string;\n\tstride?: number;\n}\n\nexport interface PerplexityResult {\n\tperplexity: number;\n\tscore: number;\n\ttokenCount: number;\n\taverageLogProb: number;\n\tmodelUsed: string;\n\tfeedback: string;\n}\n\nlet cachedTokenizer: PreTrainedTokenizer | null = null;\nlet cachedModel: PreTrainedModel | null = null;\nlet cachedModelName: string | null = null;\n\nconst getModelAndTokenizer = async (\n\tmodelName: string,\n): Promise<{ tokenizer: PreTrainedTokenizer; model: PreTrainedModel }> => {\n\tif (cachedTokenizer && cachedModel && cachedModelName === modelName) {\n\t\treturn { tokenizer: cachedTokenizer, model: cachedModel };\n\t}\n\n\tconst [tokenizer, model] = await Promise.all([\n\t\tAutoTokenizer.from_pretrained(modelName),\n\t\tAutoModelForCausalLM.from_pretrained(modelName, {\n\t\t\tdtype: \"fp32\",\n\t\t}),\n\t]);\n\n\tcachedTokenizer = tokenizer;\n\tcachedModel = model;\n\tcachedModelName = modelName;\n\n\treturn { tokenizer, model };\n};\n\nconst softmax = (logits: number[]): number[] => {\n\tconst maxLogit = Math.max(...logits);\n\tconst exps = logits.map((x) => Math.exp(x - maxLogit));\n\tconst sumExps = exps.reduce((a, b) => a + b, 0);\n\treturn exps.map((x) => x / sumExps);\n};\n\nconst normalizePerplexityToScore = (perplexity: number): number => {\n\tif (perplexity < 20) {\n\t\treturn 90 + ((20 - perplexity) / 20) * 10;\n\t}\n\tif (perplexity < 50) {\n\t\treturn 70 + ((50 - perplexity) / 30) * 20;\n\t}\n\tif (perplexity < 100) {\n\t\treturn 40 + ((100 - perplexity) / 50) * 30;\n\t}\n\tif (perplexity < 300) {\n\t\treturn 10 + ((300 - perplexity) / 200) * 30;\n\t}\n\treturn Math.max(0, 10 - (perplexity - 300) / 100);\n};\n\nconst generateFeedback = (perplexity: number, _score: number): string => {\n\tif (perplexity < 20) {\n\t\treturn `Excellent text quality with very natural, human-like language (perplexity: ${perplexity.toFixed(1)})`;\n\t}\n\tif (perplexity < 50) {\n\t\treturn `Good text quality with mostly natural phrasing (perplexity: ${perplexity.toFixed(1)})`;\n\t}\n\tif (perplexity < 100) {\n\t\treturn `Fair text quality with somewhat unnatural phrasing (perplexity: ${perplexity.toFixed(1)})`;\n\t}\n\tif (perplexity < 300) {\n\t\treturn `Poor text quality with many awkward sequences (perplexity: ${perplexity.toFixed(1)})`;\n\t}\n\treturn `Very poor text quality with nonsensical or extremely unnatural language (perplexity: ${perplexity.toFixed(1)})`;\n};\n\nexport const calculatePerplexity = async (\n\ttext: string,\n\toptions: PerplexityOptions = {},\n): Promise<PerplexityResult> => {\n\tconst { model: modelName = \"Xenova/gpt2\", stride = 512 } = options;\n\n\treturn withSpan(\n\t\t\"eval-kit.metric.perplexity\",\n\t\t{\n\t\t\tattributes: {\n\t\t\t\t\"eval_kit.metric.name\": \"perplexity\",\n\t\t\t\t\"eval_kit.metric.model\": modelName,\n\t\t\t},\n\t\t},\n\t\tasync (span) => {\n\t\t\tconst wasCached = cachedModel !== null && cachedModelName === modelName;\n\t\t\tconst { tokenizer, model } = await getModelAndTokenizer(modelName);\n\t\t\tif (!wasCached) {\n\t\t\t\tspan.addEvent(\"model_loaded\", {\n\t\t\t\t\t\"eval_kit.metric.model\": modelName,\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tconst encoded = await tokenizer(text, {\n\t\t\t\treturn_tensor: true,\n\t\t\t\ttruncation: false,\n\t\t\t\tadd_special_tokens: true,\n\t\t\t});\n\n\t\t\tconst inputIds = Array.from(encoded.input_ids.data as BigInt64Array).map(\n\t\t\t\t(x) => Number(x),\n\t\t\t);\n\n\t\t\tif (inputIds.length <= 1) {\n\t\t\t\tspan.setAttribute(\"eval_kit.metric.token_count\", inputIds.length);\n\t\t\t\tspan.setAttribute(\"eval_kit.result.score\", 100);\n\t\t\t\treturn {\n\t\t\t\t\tperplexity: 1.0,\n\t\t\t\t\tscore: 100,\n\t\t\t\t\ttokenCount: inputIds.length,\n\t\t\t\t\taverageLogProb: 0,\n\t\t\t\t\tmodelUsed: modelName,\n\t\t\t\t\tfeedback: \"Text too short to calculate perplexity meaningfully\",\n\t\t\t\t};\n\t\t\t}\n\n\t\t\tlet totalLogProb = 0;\n\t\t\tlet totalTokens = 0;\n\n\t\t\tfor (let i = 0; i < inputIds.length; i += stride) {\n\t\t\t\tconst end = Math.min(i + stride + 1, inputIds.length);\n\t\t\t\tconst batch = inputIds.slice(i, end);\n\n\t\t\t\tif (batch.length <= 1) continue;\n\n\t\t\t\tconst batchTensor = {\n\t\t\t\t\tinput_ids: new Tensor(\n\t\t\t\t\t\t\"int64\",\n\t\t\t\t\t\tnew BigInt64Array(batch.map((x) => BigInt(x))),\n\t\t\t\t\t\t[1, batch.length],\n\t\t\t\t\t),\n\t\t\t\t};\n\n\t\t\t\tconst outputs = await model(batchTensor);\n\t\t\t\tconst logits = outputs.logits;\n\n\t\t\t\tif (!logits || !logits.data) continue;\n\n\t\t\t\t// Get vocab size from logits shape: [batch_size, seq_len, vocab_size]\n\t\t\t\tconst vocabSize = logits.dims?.[2] || 50257;\n\t\t\t\tconst logitsArray = Array.from(logits.data as Float32Array);\n\n\t\t\t\tfor (let j = 1; j < batch.length; j++) {\n\t\t\t\t\tconst startIdx = (j - 1) * vocabSize;\n\t\t\t\t\tconst endIdx = startIdx + vocabSize;\n\t\t\t\t\tconst prevLogits = logitsArray.slice(startIdx, endIdx);\n\n\t\t\t\t\tconst probs = softmax(prevLogits);\n\t\t\t\t\tconst targetTokenId = batch[j];\n\t\t\t\t\tconst prob = probs[targetTokenId] || 1e-10;\n\n\t\t\t\t\ttotalLogProb += Math.log(prob);\n\t\t\t\t\ttotalTokens++;\n\t\t\t\t}\n\n\t\t\t\tif (end >= inputIds.length) break;\n\t\t\t}\n\n\t\t\tconst averageLogProb = totalTokens > 0 ? totalLogProb / totalTokens : 0;\n\t\t\tconst perplexity = Math.exp(-averageLogProb);\n\t\t\tconst score = normalizePerplexityToScore(perplexity);\n\n\t\t\tspan.setAttribute(\"eval_kit.metric.token_count\", totalTokens);\n\t\t\tspan.setAttribute(\n\t\t\t\t\"eval_kit.result.perplexity\",\n\t\t\t\tMath.round(perplexity * 100) / 100,\n\t\t\t);\n\t\t\tspan.setAttribute(\"eval_kit.result.score\", Math.round(score * 100) / 100);\n\n\t\t\treturn {\n\t\t\t\tperplexity: Math.round(perplexity * 100) / 100,\n\t\t\t\tscore: Math.round(score * 100) / 100,\n\t\t\t\ttokenCount: totalTokens,\n\t\t\t\taverageLogProb: Math.round(averageLogProb * 10000) / 10000,\n\t\t\t\tmodelUsed: modelName,\n\t\t\t\tfeedback: generateFeedback(perplexity, score),\n\t\t\t};\n\t\t},\n\t);\n};\n\nexport const clearPerplexityCache = (): void => {\n\tcachedTokenizer = null;\n\tcachedModel = null;\n\tcachedModelName = null;\n};\n"],"names":["AutoTokenizer","AutoModelForCausalLM","withSpan","Tensor"],"mappings":";;;;AAuBA,IAAI,kBAA8C;AAClD,IAAI,cAAsC;AAC1C,IAAI,kBAAiC;AAErC,MAAM,uBAAuB,OAC5B,cACyE;AACzE,MAAI,mBAAmB,eAAe,oBAAoB,WAAW;AACpE,WAAO,EAAE,WAAW,iBAAiB,OAAO,YAAA;AAAA,EAC7C;AAEA,QAAM,CAAC,WAAW,KAAK,IAAI,MAAM,QAAQ,IAAI;AAAA,IAC5CA,aAAAA,cAAc,gBAAgB,SAAS;AAAA,IACvCC,aAAAA,qBAAqB,gBAAgB,WAAW;AAAA,MAC/C,OAAO;AAAA,IAAA,CACP;AAAA,EAAA,CACD;AAED,oBAAkB;AAClB,gBAAc;AACd,oBAAkB;AAElB,SAAO,EAAE,WAAW,MAAA;AACrB;AAEA,MAAM,UAAU,CAAC,WAA+B;AAC/C,QAAM,WAAW,KAAK,IAAI,GAAG,MAAM;AACnC,QAAM,OAAO,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,QAAQ,CAAC;AACrD,QAAM,UAAU,KAAK,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAC9C,SAAO,KAAK,IAAI,CAAC,MAAM,IAAI,OAAO;AACnC;AAEA,MAAM,6BAA6B,CAAC,eAA+B;AAClE,MAAI,aAAa,IAAI;AACpB,WAAO,MAAO,KAAK,cAAc,KAAM;AAAA,EACxC;AACA,MAAI,aAAa,IAAI;AACpB,WAAO,MAAO,KAAK,cAAc,KAAM;AAAA,EACxC;AACA,MAAI,aAAa,KAAK;AACrB,WAAO,MAAO,MAAM,cAAc,KAAM;AAAA,EACzC;AACA,MAAI,aAAa,KAAK;AACrB,WAAO,MAAO,MAAM,cAAc,MAAO;AAAA,EAC1C;AACA,SAAO,KAAK,IAAI,GAAG,MAAM,aAAa,OAAO,GAAG;AACjD;AAEA,MAAM,mBAAmB,CAAC,YAAoB,WAA2B;AACxE,MAAI,aAAa,IAAI;AACpB,WAAO,8EAA8E,WAAW,QAAQ,CAAC,CAAC;AAAA,EAC3G;AACA,MAAI,aAAa,IAAI;AACpB,WAAO,+DAA+D,WAAW,QAAQ,CAAC,CAAC;AAAA,EAC5F;AACA,MAAI,aAAa,KAAK;AACrB,WAAO,mEAAmE,WAAW,QAAQ,CAAC,CAAC;AAAA,EAChG;AACA,MAAI,aAAa,KAAK;AACrB,WAAO,8DAA8D,WAAW,QAAQ,CAAC,CAAC;AAAA,EAC3F;AACA,SAAO,wFAAwF,WAAW,QAAQ,CAAC,CAAC;AACrH;AAEO,MAAM,sBAAsB,OAClC,MACA,UAA6B,OACE;AAC/B,QAAM,EAAE,OAAO,YAAY,eAAe,SAAS,QAAQ;AAE3D,SAAOC,MAAAA;AAAAA,IACN;AAAA,IACA;AAAA,MACC,YAAY;AAAA,QACX,wBAAwB;AAAA,QACxB,yBAAyB;AAAA,MAAA;AAAA,IAC1B;AAAA,IAED,OAAO,SAAS;AACf,YAAM,YAAY,gBAAgB,QAAQ,oBAAoB;AAC9D,YAAM,EAAE,WAAW,MAAA,IAAU,MAAM,qBAAqB,SAAS;AACjE,UAAI,CAAC,WAAW;AACf,aAAK,SAAS,gBAAgB;AAAA,UAC7B,yBAAyB;AAAA,QAAA,CACzB;AAAA,MACF;AAEA,YAAM,UAAU,MAAM,UAAU,MAAM;AAAA,QACrC,eAAe;AAAA,QACf,YAAY;AAAA,QACZ,oBAAoB;AAAA,MAAA,CACpB;AAED,YAAM,WAAW,MAAM,KAAK,QAAQ,UAAU,IAAqB,EAAE;AAAA,QACpE,CAAC,MAAM,OAAO,CAAC;AAAA,MAAA;AAGhB,UAAI,SAAS,UAAU,GAAG;AACzB,aAAK,aAAa,+BAA+B,SAAS,MAAM;AAChE,aAAK,aAAa,yBAAyB,GAAG;AAC9C,eAAO;AAAA,UACN,YAAY;AAAA,UACZ,OAAO;AAAA,UACP,YAAY,SAAS;AAAA,UACrB,gBAAgB;AAAA,UAChB,WAAW;AAAA,UACX,UAAU;AAAA,QAAA;AAAA,MAEZ;AAEA,UAAI,eAAe;AACnB,UAAI,cAAc;AAElB,eAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,QAAQ;AACjD,cAAM,MAAM,KAAK,IAAI,IAAI,SAAS,GAAG,SAAS,MAAM;AACpD,cAAM,QAAQ,SAAS,MAAM,GAAG,GAAG;AAEnC,YAAI,MAAM,UAAU,EAAG;AAEvB,cAAM,cAAc;AAAA,UACnB,WAAW,IAAIC,aAAAA;AAAAA,YACd;AAAA,YACA,IAAI,cAAc,MAAM,IAAI,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC;AAAA,YAC7C,CAAC,GAAG,MAAM,MAAM;AAAA,UAAA;AAAA,QACjB;AAGD,cAAM,UAAU,MAAM,MAAM,WAAW;AACvC,cAAM,SAAS,QAAQ;AAEvB,YAAI,CAAC,UAAU,CAAC,OAAO,KAAM;AAG7B,cAAM,YAAY,OAAO,OAAO,CAAC,KAAK;AACtC,cAAM,cAAc,MAAM,KAAK,OAAO,IAAoB;AAE1D,iBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACtC,gBAAM,YAAY,IAAI,KAAK;AAC3B,gBAAM,SAAS,WAAW;AAC1B,gBAAM,aAAa,YAAY,MAAM,UAAU,MAAM;AAErD,gBAAM,QAAQ,QAAQ,UAAU;AAChC,gBAAM,gBAAgB,MAAM,CAAC;AAC7B,gBAAM,OAAO,MAAM,aAAa,KAAK;AAErC,0BAAgB,KAAK,IAAI,IAAI;AAC7B;AAAA,QACD;AAEA,YAAI,OAAO,SAAS,OAAQ;AAAA,MAC7B;AAEA,YAAM,iBAAiB,cAAc,IAAI,eAAe,cAAc;AACtE,YAAM,aAAa,KAAK,IAAI,CAAC,cAAc;AAC3C,YAAM,QAAQ,2BAA2B,UAAU;AAEnD,WAAK,aAAa,+BAA+B,WAAW;AAC5D,WAAK;AAAA,QACJ;AAAA,QACA,KAAK,MAAM,aAAa,GAAG,IAAI;AAAA,MAAA;AAEhC,WAAK,aAAa,yBAAyB,KAAK,MAAM,QAAQ,GAAG,IAAI,GAAG;AAExE,aAAO;AAAA,QACN,YAAY,KAAK,MAAM,aAAa,GAAG,IAAI;AAAA,QAC3C,OAAO,KAAK,MAAM,QAAQ,GAAG,IAAI;AAAA,QACjC,YAAY;AAAA,QACZ,gBAAgB,KAAK,MAAM,iBAAiB,GAAK,IAAI;AAAA,QACrD,WAAW;AAAA,QACX,UAAU,iBAAiB,YAAY,KAAK;AAAA,MAAA;AAAA,IAE9C;AAAA,EAAA;AAEF;AAEO,MAAM,uBAAuB,MAAY;AAC/C,oBAAkB;AAClB,gBAAc;AACd,oBAAkB;AACnB;;;"}