{"version":3,"file":"bert-score-g9sM6-6x.cjs","sources":["../src/metrics/bert-score.ts"],"sourcesContent":["import type { FeatureExtractionPipeline } from \"@huggingface/transformers\";\nimport { cos_sim, pipeline } from \"@huggingface/transformers\";\nimport { withSpan } from \"../telemetry.js\";\nimport { tokenizeWords } from \"../utils/tokenization.js\";\n\nexport interface BertScoreOptions {\n\tmodel?: string;\n\tscoreType?: \"f1\" | \"precision\" | \"recall\";\n}\n\nexport interface BertScoreResult {\n\tscore: number;\n\tprecision: number;\n\trecall: number;\n\tf1: number;\n\tmodelUsed: string;\n}\n\nlet cachedPipeline: FeatureExtractionPipeline | null = null;\nlet cachedModelName: string | null = null;\n\nconst getPipeline = async (\n\tmodelName: string,\n): Promise<FeatureExtractionPipeline> => {\n\tif (cachedPipeline && cachedModelName === modelName) {\n\t\treturn cachedPipeline;\n\t}\n\n\tcachedPipeline = await pipeline(\"feature-extraction\", modelName, {\n\t\tdtype: \"fp32\",\n\t});\n\tcachedModelName = modelName;\n\n\treturn cachedPipeline;\n};\n\nconst getTokenEmbeddings = async (\n\ttokens: string[],\n\textractor: FeatureExtractionPipeline,\n): Promise<number[][]> => {\n\tconst embeddings: number[][] = [];\n\n\tfor (const token of tokens) {\n\t\tconst embedding = await extractor(token, {\n\t\t\tpooling: \"mean\",\n\t\t\tnormalize: true,\n\t\t});\n\t\tembeddings.push(Array.from(embedding.data as Float32Array));\n\t}\n\n\treturn embeddings;\n};\n\nconst computeMaxSimilarities = (\n\tsourceEmbeddings: number[][],\n\ttargetEmbeddings: number[][],\n): number[] => {\n\tconst maxSimilarities: number[] = [];\n\n\tfor (const sourceEmb of sourceEmbeddings) {\n\t\tlet maxSim = -1;\n\t\tfor (const targetEmb of targetEmbeddings) {\n\t\t\tconst sim = cos_sim(sourceEmb, targetEmb);\n\t\t\tif (sim > maxSim) {\n\t\t\t\tmaxSim = sim;\n\t\t\t}\n\t\t}\n\t\tmaxSimilarities.push(maxSim);\n\t}\n\n\treturn maxSimilarities;\n};\n\nexport const calculateBertScore = async (\n\tcandidate: string,\n\treference: string,\n\toptions: BertScoreOptions = {},\n): Promise<BertScoreResult> => {\n\tconst { model = \"Xenova/all-MiniLM-L6-v2\", scoreType = \"f1\" } = options;\n\n\treturn withSpan(\n\t\t\"eval-kit.metric.bert_score\",\n\t\t{\n\t\t\tattributes: {\n\t\t\t\t\"eval_kit.metric.name\": \"bert_score\",\n\t\t\t\t\"eval_kit.metric.model\": model,\n\t\t\t\t\"eval_kit.metric.score_type\": scoreType,\n\t\t\t},\n\t\t},\n\t\tasync (span) => {\n\t\t\tconst wasCached = cachedPipeline !== null && cachedModelName === model;\n\t\t\tconst extractor = await getPipeline(model);\n\t\t\tif (!wasCached) {\n\t\t\t\tspan.addEvent(\"model_loaded\", {\n\t\t\t\t\t\"eval_kit.metric.model\": model,\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tconst candidateTokens = tokenizeWords(candidate);\n\t\t\tconst referenceTokens = tokenizeWords(reference);\n\n\t\t\tconst candidateEmbeddings = await getTokenEmbeddings(\n\t\t\t\tcandidateTokens,\n\t\t\t\textractor,\n\t\t\t);\n\t\t\tconst referenceEmbeddings = await getTokenEmbeddings(\n\t\t\t\treferenceTokens,\n\t\t\t\textractor,\n\t\t\t);\n\n\t\t\tconst precisionSimilarities = computeMaxSimilarities(\n\t\t\t\tcandidateEmbeddings,\n\t\t\t\treferenceEmbeddings,\n\t\t\t);\n\t\t\tconst recallSimilarities = computeMaxSimilarities(\n\t\t\t\treferenceEmbeddings,\n\t\t\t\tcandidateEmbeddings,\n\t\t\t);\n\n\t\t\tconst precision =\n\t\t\t\tprecisionSimilarities.length > 0\n\t\t\t\t\t? precisionSimilarities.reduce((sum, sim) => sum + sim, 0) /\n\t\t\t\t\t\tprecisionSimilarities.length\n\t\t\t\t\t: 0;\n\n\t\t\tconst recall =\n\t\t\t\trecallSimilarities.length > 0\n\t\t\t\t\t? recallSimilarities.reduce((sum, sim) => sum + sim, 0) /\n\t\t\t\t\t\trecallSimilarities.length\n\t\t\t\t\t: 0;\n\n\t\t\tconst f1 =\n\t\t\t\tprecision + recall > 0\n\t\t\t\t\t? (2 * precision * recall) / (precision + recall)\n\t\t\t\t\t: 0;\n\n\t\t\tlet finalScore = f1;\n\t\t\tif (scoreType === \"precision\") {\n\t\t\t\tfinalScore = precision;\n\t\t\t} else if (scoreType === \"recall\") {\n\t\t\t\tfinalScore = recall;\n\t\t\t}\n\n\t\t\tconst result = {\n\t\t\t\tscore: Math.round(finalScore * 10000) / 100,\n\t\t\t\tprecision: Math.round(precision * 10000) / 100,\n\t\t\t\trecall: Math.round(recall * 10000) / 100,\n\t\t\t\tf1: Math.round(f1 * 10000) / 100,\n\t\t\t\tmodelUsed: model,\n\t\t\t};\n\n\t\t\tspan.setAttribute(\"eval_kit.result.score\", result.score);\n\t\t\treturn result;\n\t\t},\n\t);\n};\n\nexport const clearBertCache = (): void => {\n\tcachedPipeline = null;\n\tcachedModelName = null;\n};\n"],"names":["pipeline","cos_sim","withSpan","tokenizeWords"],"mappings":";;;;AAkBA,IAAI,iBAAmD;AACvD,IAAI,kBAAiC;AAErC,MAAM,cAAc,OACnB,cACwC;AACxC,MAAI,kBAAkB,oBAAoB,WAAW;AACpD,WAAO;AAAA,EACR;AAEA,mBAAiB,MAAMA,aAAAA,SAAS,sBAAsB,WAAW;AAAA,IAChE,OAAO;AAAA,EAAA,CACP;AACD,oBAAkB;AAElB,SAAO;AACR;AAEA,MAAM,qBAAqB,OAC1B,QACA,cACyB;AACzB,QAAM,aAAyB,CAAA;AAE/B,aAAW,SAAS,QAAQ;AAC3B,UAAM,YAAY,MAAM,UAAU,OAAO;AAAA,MACxC,SAAS;AAAA,MACT,WAAW;AAAA,IAAA,CACX;AACD,eAAW,KAAK,MAAM,KAAK,UAAU,IAAoB,CAAC;AAAA,EAC3D;AAEA,SAAO;AACR;AAEA,MAAM,yBAAyB,CAC9B,kBACA,qBACc;AACd,QAAM,kBAA4B,CAAA;AAElC,aAAW,aAAa,kBAAkB;AACzC,QAAI,SAAS;AACb,eAAW,aAAa,kBAAkB;AACzC,YAAM,MAAMC,aAAAA,QAAQ,WAAW,SAAS;AACxC,UAAI,MAAM,QAAQ;AACjB,iBAAS;AAAA,MACV;AAAA,IACD;AACA,oBAAgB,KAAK,MAAM;AAAA,EAC5B;AAEA,SAAO;AACR;AAEO,MAAM,qBAAqB,OACjC,WACA,WACA,UAA4B,CAAA,MACE;AAC9B,QAAM,EAAE,QAAQ,2BAA2B,YAAY,SAAS;AAEhE,SAAOC,MAAAA;AAAAA,IACN;AAAA,IACA;AAAA,MACC,YAAY;AAAA,QACX,wBAAwB;AAAA,QACxB,yBAAyB;AAAA,QACzB,8BAA8B;AAAA,MAAA;AAAA,IAC/B;AAAA,IAED,OAAO,SAAS;AACf,YAAM,YAAY,mBAAmB,QAAQ,oBAAoB;AACjE,YAAM,YAAY,MAAM,YAAY,KAAK;AACzC,UAAI,CAAC,WAAW;AACf,aAAK,SAAS,gBAAgB;AAAA,UAC7B,yBAAyB;AAAA,QAAA,CACzB;AAAA,MACF;AAEA,YAAM,kBAAkBC,MAAAA,cAAc,SAAS;AAC/C,YAAM,kBAAkBA,MAAAA,cAAc,SAAS;AAE/C,YAAM,sBAAsB,MAAM;AAAA,QACjC;AAAA,QACA;AAAA,MAAA;AAED,YAAM,sBAAsB,MAAM;AAAA,QACjC;AAAA,QACA;AAAA,MAAA;AAGD,YAAM,wBAAwB;AAAA,QAC7B;AAAA,QACA;AAAA,MAAA;AAED,YAAM,qBAAqB;AAAA,QAC1B;AAAA,QACA;AAAA,MAAA;AAGD,YAAM,YACL,sBAAsB,SAAS,IAC5B,sBAAsB,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,IACxD,sBAAsB,SACrB;AAEJ,YAAM,SACL,mBAAmB,SAAS,IACzB,mBAAmB,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,IACrD,mBAAmB,SAClB;AAEJ,YAAM,KACL,YAAY,SAAS,IACjB,IAAI,YAAY,UAAW,YAAY,UACxC;AAEJ,UAAI,aAAa;AACjB,UAAI,cAAc,aAAa;AAC9B,qBAAa;AAAA,MACd,WAAW,cAAc,UAAU;AAClC,qBAAa;AAAA,MACd;AAEA,YAAM,SAAS;AAAA,QACd,OAAO,KAAK,MAAM,aAAa,GAAK,IAAI;AAAA,QACxC,WAAW,KAAK,MAAM,YAAY,GAAK,IAAI;AAAA,QAC3C,QAAQ,KAAK,MAAM,SAAS,GAAK,IAAI;AAAA,QACrC,IAAI,KAAK,MAAM,KAAK,GAAK,IAAI;AAAA,QAC7B,WAAW;AAAA,MAAA;AAGZ,WAAK,aAAa,yBAAyB,OAAO,KAAK;AACvD,aAAO;AAAA,IACR;AAAA,EAAA;AAEF;AAEO,MAAM,iBAAiB,MAAY;AACzC,mBAAiB;AACjB,oBAAkB;AACnB;;;"}