{"version":3,"sources":["../../src/metrics/faithfulness.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { z, type Genkit, type ModelArgument } from 'genkit';\nimport {\n EvalStatusEnum,\n type BaseEvalDataPoint,\n type Score,\n} from 'genkit/evaluator';\nimport path from 'path';\nimport { getDirName, loadPromptFile, renderText } from './helper.mjs';\n\nconst LongFormResponseSchema = z.object({ statements: z.array(z.string()) });\n\nconst NliResponseBaseSchema = z.object({\n statement: z.string(),\n reason: z.string(),\n verdict: z.boolean(),\n});\n\ntype NliResponseBase = z.infer;\nconst NliResponseSchema = z.object({\n responses: z.array(NliResponseBaseSchema),\n});\n\n/**\n *\n */\nexport async function faithfulnessScore<\n CustomModelOptions extends z.ZodTypeAny,\n>(\n ai: Genkit,\n judgeLlm: ModelArgument,\n dataPoint: BaseEvalDataPoint,\n judgeConfig?: CustomModelOptions\n): Promise {\n try {\n if (!dataPoint.input) {\n throw new Error('Input was not provided');\n }\n if (!dataPoint.output) {\n throw new Error('Output was not provided');\n }\n if (!dataPoint.context?.length) {\n throw new Error('Context was not provided');\n }\n\n const input =\n typeof dataPoint.input === 'string'\n ? dataPoint.input\n : JSON.stringify(dataPoint.input);\n const output =\n typeof dataPoint.output === 'string'\n ? dataPoint.output\n : JSON.stringify(dataPoint.output);\n const context = dataPoint.context.map((i) => JSON.stringify(i));\n\n const longFormPrompt = await loadPromptFile(\n path.resolve(getDirName(), '../../prompts/faithfulness_long_form.prompt')\n );\n const longFormResponse = await ai.generate({\n model: judgeLlm,\n config: judgeConfig,\n prompt: await renderText(longFormPrompt, {\n question: input,\n answer: output,\n }),\n output: {\n schema: LongFormResponseSchema,\n },\n });\n const parsedLongFormResponse = longFormResponse.output;\n const statements = parsedLongFormResponse?.statements ?? [];\n if (statements.length === 0) {\n throw new Error('No statements returned');\n }\n const allStatements = statements.map((s) => `statement: ${s}`).join('\\n');\n const allContext = context.join('\\n');\n const nliPrompt = await loadPromptFile(\n path.resolve(getDirName(), '../../prompts/faithfulness_nli.prompt')\n );\n const response = await ai.generate({\n model: judgeLlm,\n prompt: await renderText(nliPrompt, {\n context: allContext,\n statements: allStatements,\n }),\n output: {\n schema: NliResponseSchema,\n },\n });\n const parsedResponse = response.output;\n return nliResponseToScore(parsedResponse?.responses ?? []);\n } catch (err) {\n console.debug(\n `Genkit faithfulness evaluation failed with error ${err} for sample ${JSON.stringify(\n dataPoint\n )}`\n );\n throw err;\n }\n}\n\nfunction nliResponseToScore(input: NliResponseBase[] | null): Score {\n if (!input) {\n throw new Error(`Evaluator response empty`);\n }\n const faithfulStatements = input.reduce((total, resp) => {\n return total + (resp.verdict ? 1 : 0);\n }, 0);\n const score = faithfulStatements / input.length;\n return {\n score,\n details: { reasoning: input.map((r) => r.reason).join('; ') },\n status: score > 0.5 ? EvalStatusEnum.PASS : EvalStatusEnum.FAIL,\n };\n}\n"],"mappings":"AAgBA,SAAS,SAA0C;AACnD;AAAA,EACE;AAAA,OAGK;AACP,OAAO,UAAU;AACjB,SAAS,YAAY,gBAAgB,kBAAkB;AAEvD,MAAM,yBAAyB,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;AAE3E,MAAM,wBAAwB,EAAE,OAAO;AAAA,EACrC,WAAW,EAAE,OAAO;AAAA,EACpB,QAAQ,EAAE,OAAO;AAAA,EACjB,SAAS,EAAE,QAAQ;AACrB,CAAC;AAGD,MAAM,oBAAoB,EAAE,OAAO;AAAA,EACjC,WAAW,EAAE,MAAM,qBAAqB;AAC1C,CAAC;AAKD,eAAsB,kBAGpB,IACA,UACA,WACA,aACgB;AAChB,MAAI;AACF,QAAI,CAAC,UAAU,OAAO;AACpB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AACA,QAAI,CAAC,UAAU,QAAQ;AACrB,YAAM,IAAI,MAAM,yBAAyB;AAAA,IAC3C;AACA,QAAI,CAAC,UAAU,SAAS,QAAQ;AAC9B,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AAEA,UAAM,QACJ,OAAO,UAAU,UAAU,WACvB,UAAU,QACV,KAAK,UAAU,UAAU,KAAK;AACpC,UAAM,SACJ,OAAO,UAAU,WAAW,WACxB,UAAU,SACV,KAAK,UAAU,UAAU,MAAM;AACrC,UAAM,UAAU,UAAU,QAAQ,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;AAE9D,UAAM,iBAAiB,MAAM;AAAA,MAC3B,KAAK,QAAQ,WAAW,GAAG,6CAA6C;AAAA,IAC1E;AACA,UAAM,mBAAmB,MAAM,GAAG,SAAS;AAAA,MACzC,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,QAAQ,MAAM,WAAW,gBAAgB;AAAA,QACvC,UAAU;AAAA,QACV,QAAQ;AAAA,MACV,CAAC;AAAA,MACD,QAAQ;AAAA,QACN,QAAQ;AAAA,MACV;AAAA,IACF,CAAC;AACD,UAAM,yBAAyB,iBAAiB;AAChD,UAAM,aAAa,wBAAwB,cAAc,CAAC;AAC1D,QAAI,WAAW,WAAW,GAAG;AAC3B,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AACA,UAAM,gBAAgB,WAAW,IAAI,CAAC,MAAM,cAAc,CAAC,EAAE,EAAE,KAAK,IAAI;AACxE,UAAM,aAAa,QAAQ,KAAK,IAAI;AACpC,UAAM,YAAY,MAAM;AAAA,MACtB,KAAK,QAAQ,WAAW,GAAG,uCAAuC;AAAA,IACpE;AACA,UAAM,WAAW,MAAM,GAAG,SAAS;AAAA,MACjC,OAAO;AAAA,MACP,QAAQ,MAAM,WAAW,WAAW;AAAA,QAClC,SAAS;AAAA,QACT,YAAY;AAAA,MACd,CAAC;AAAA,MACD,QAAQ;AAAA,QACN,QAAQ;AAAA,MACV;AAAA,IACF,CAAC;AACD,UAAM,iBAAiB,SAAS;AAChC,WAAO,mBAAmB,gBAAgB,aAAa,CAAC,CAAC;AAAA,EAC3D,SAAS,KAAK;AACZ,YAAQ;AAAA,MACN,oDAAoD,GAAG,eAAe,KAAK;AAAA,QACzE;AAAA,MACF,CAAC;AAAA,IACH;AACA,UAAM;AAAA,EACR;AACF;AAEA,SAAS,mBAAmB,OAAwC;AAClE,MAAI,CAAC,OAAO;AACV,UAAM,IAAI,MAAM,0BAA0B;AAAA,EAC5C;AACA,QAAM,qBAAqB,MAAM,OAAO,CAAC,OAAO,SAAS;AACvD,WAAO,SAAS,KAAK,UAAU,IAAI;AAAA,EACrC,GAAG,CAAC;AACJ,QAAM,QAAQ,qBAAqB,MAAM;AACzC,SAAO;AAAA,IACL;AAAA,IACA,SAAS,EAAE,WAAW,MAAM,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;AAAA,IAC5D,QAAQ,QAAQ,MAAM,eAAe,OAAO,eAAe;AAAA,EAC7D;AACF;","names":[]}