{"version":3,"sources":["../../src/metrics/answer_relevancy.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport similarity from 'compute-cosine-similarity';\nimport { z, type Genkit, type ModelArgument } from 'genkit';\nimport type { EmbedderArgument } from 'genkit/embedder';\nimport {\n EvalStatusEnum,\n type BaseEvalDataPoint,\n type Score,\n} from 'genkit/evaluator';\nimport path from 'path';\nimport { getDirName, loadPromptFile, renderText } from './helper.mjs';\n\nconst AnswerRelevancyResponseSchema = z.object({\n question: z.string(),\n answered: z.boolean(),\n noncommittal: z.boolean(),\n});\n\nexport async function answerRelevancyScore<\n CustomModelOptions extends z.ZodTypeAny,\n CustomEmbedderOptions extends z.ZodTypeAny,\n>(\n ai: Genkit,\n judgeLlm: ModelArgument,\n dataPoint: BaseEvalDataPoint,\n embedder: EmbedderArgument,\n judgeConfig?: CustomModelOptions,\n embedderOptions?: z.infer\n): Promise {\n try {\n if (!dataPoint.input) {\n throw new Error('Input was not provided');\n }\n if (!dataPoint.output) {\n throw new Error('Output was not provided');\n }\n if (!dataPoint.context?.length) {\n throw new Error('Context was not provided');\n }\n\n const input =\n typeof dataPoint.input === 'string'\n ? dataPoint.input\n : JSON.stringify(dataPoint.input);\n const output =\n typeof dataPoint.output === 'string'\n ? dataPoint.output\n : JSON.stringify(dataPoint.output);\n const context = dataPoint.context.map((i) => JSON.stringify(i));\n\n const prompt = await loadPromptFile(\n path.resolve(getDirName(), '../../prompts/answer_relevancy.prompt')\n );\n const response = await ai.generate({\n model: judgeLlm,\n config: judgeConfig,\n prompt: await renderText(prompt, {\n question: input,\n answer: output,\n context: context.join(' '),\n }),\n output: {\n schema: AnswerRelevancyResponseSchema,\n },\n });\n const genQuestion = response.output?.question;\n if (!genQuestion)\n throw new Error('Error generating question for answer relevancy');\n\n const questionEmbed = (\n await ai.embed({\n embedder,\n content: input,\n options: embedderOptions,\n })\n )[0].embedding; // Single embedding for text\n const genQuestionEmbed = (\n await ai.embed({\n embedder,\n content: genQuestion,\n options: embedderOptions,\n })\n )[0].embedding; // Single embedding for text\n const score = cosineSimilarity(questionEmbed, genQuestionEmbed);\n const answered = response.output?.answered ?? false;\n const isNonCommittal = response.output?.noncommittal ?? false;\n const answeredPenalty = !answered ? 0.5 : 0;\n const adjustedScore =\n score - answeredPenalty < 0 ? 0 : score - answeredPenalty;\n const reasoning = isNonCommittal\n ? 'Noncommittal'\n : answered\n ? 'Cosine similarity'\n : 'Cosine similarity with penalty for insufficient answer';\n const finalScore = adjustedScore * (isNonCommittal ? 0 : 1);\n return {\n score: finalScore,\n details: {\n reasoning,\n },\n status: finalScore > 0.5 ? EvalStatusEnum.PASS : EvalStatusEnum.FAIL,\n };\n } catch (err) {\n console.debug(\n `Genkit answer_relevancy evaluation failed with error ${err} for sample ${JSON.stringify(\n dataPoint\n )}`\n );\n throw err;\n }\n}\n\nfunction cosineSimilarity(v1: Array, v2: Array) {\n const maybeScore = similarity(v1, v2);\n if (!maybeScore) {\n throw new Error('Unable to compute cosine similarity');\n }\n return Math.abs(maybeScore);\n}\n"],"mappings":"AAgBA,OAAO,gBAAgB;AACvB,SAAS,SAA0C;AAEnD;AAAA,EACE;AAAA,OAGK;AACP,OAAO,UAAU;AACjB,SAAS,YAAY,gBAAgB,kBAAkB;AAEvD,MAAM,gCAAgC,EAAE,OAAO;AAAA,EAC7C,UAAU,EAAE,OAAO;AAAA,EACnB,UAAU,EAAE,QAAQ;AAAA,EACpB,cAAc,EAAE,QAAQ;AAC1B,CAAC;AAED,eAAsB,qBAIpB,IACA,UACA,WACA,UACA,aACA,iBACgB;AAChB,MAAI;AACF,QAAI,CAAC,UAAU,OAAO;AACpB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AACA,QAAI,CAAC,UAAU,QAAQ;AACrB,YAAM,IAAI,MAAM,yBAAyB;AAAA,IAC3C;AACA,QAAI,CAAC,UAAU,SAAS,QAAQ;AAC9B,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AAEA,UAAM,QACJ,OAAO,UAAU,UAAU,WACvB,UAAU,QACV,KAAK,UAAU,UAAU,KAAK;AACpC,UAAM,SACJ,OAAO,UAAU,WAAW,WACxB,UAAU,SACV,KAAK,UAAU,UAAU,MAAM;AACrC,UAAM,UAAU,UAAU,QAAQ,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;AAE9D,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,QAAQ,WAAW,GAAG,uCAAuC;AAAA,IACpE;AACA,UAAM,WAAW,MAAM,GAAG,SAAS;AAAA,MACjC,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,QAAQ,MAAM,WAAW,QAAQ;AAAA,QAC/B,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,SAAS,QAAQ,KAAK,GAAG;AAAA,MAC3B,CAAC;AAAA,MACD,QAAQ;AAAA,QACN,QAAQ;AAAA,MACV;AAAA,IACF,CAAC;AACD,UAAM,cAAc,SAAS,QAAQ;AACrC,QAAI,CAAC;AACH,YAAM,IAAI,MAAM,gDAAgD;AAElE,UAAM,iBACJ,MAAM,GAAG,MAAM;AAAA,MACb;AAAA,MACA,SAAS;AAAA,MACT,SAAS;AAAA,IACX,CAAC,GACD,CAAC,EAAE;AACL,UAAM,oBACJ,MAAM,GAAG,MAAM;AAAA,MACb;AAAA,MACA,SAAS;AAAA,MACT,SAAS;AAAA,IACX,CAAC,GACD,CAAC,EAAE;AACL,UAAM,QAAQ,iBAAiB,eAAe,gBAAgB;AAC9D,UAAM,WAAW,SAAS,QAAQ,YAAY;AAC9C,UAAM,iBAAiB,SAAS,QAAQ,gBAAgB;AACxD,UAAM,kBAAkB,CAAC,WAAW,MAAM;AAC1C,UAAM,gBACJ,QAAQ,kBAAkB,IAAI,IAAI,QAAQ;AAC5C,UAAM,YAAY,iBACd,iBACA,WACE,sBACA;AACN,UAAM,aAAa,iBAAiB,iBAAiB,IAAI;AACzD,WAAO;AAAA,MACL,OAAO;AAAA,MACP,SAAS;AAAA,QACP;AAAA,MACF;AAAA,MACA,QAAQ,aAAa,MAAM,eAAe,OAAO,eAAe;AAAA,IAClE;AAAA,EACF,SAAS,KAAK;AACZ,YAAQ;AAAA,MACN,wDAAwD,GAAG,eAAe,KAAK;AAAA,QAC7E;AAAA,MACF,CAAC;AAAA,IACH;AACA,UAAM;AAAA,EACR;AACF;AAEA,SAAS,iBAAiB,IAAmB,IAAmB;AAC9D,QAAM,aAAa,WAAW,IAAI,EAAE;AACpC,MAAI,CAAC,YAAY;AACf,UAAM,IAAI,MAAM,qCAAqC;AAAA,EACvD;AACA,SAAO,KAAK,IAAI,UAAU;AAC5B;","names":[]}