{"version":3,"sources":["../../src/metrics/answer_accuracy.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { Genkit, ModelArgument, z } from 'genkit';\nimport {\n  EvalStatusEnum,\n  type BaseEvalDataPoint,\n  type Score,\n} from 'genkit/evaluator';\nimport path from 'path';\nimport { getDirName, loadPromptFile, renderText } from './helper.mjs';\n\nexport async function answerAccuracyScore<\n  CustomModelOptions extends z.ZodTypeAny,\n>(\n  ai: Genkit,\n  judgeLlm: ModelArgument<CustomModelOptions>,\n  dataPoint: BaseEvalDataPoint,\n  judgeConfig?: CustomModelOptions\n): Promise<Score> {\n  if (!dataPoint.output) {\n    throw new Error('Output was not provided');\n  }\n  if (!dataPoint.reference) {\n    throw new Error('Reference was not provided');\n  }\n  const input =\n    typeof dataPoint.input === 'string'\n      ? dataPoint.input\n      : JSON.stringify(dataPoint.input);\n  const output =\n    typeof dataPoint.output === 'string'\n      ? dataPoint.output\n      : JSON.stringify(dataPoint.output);\n  const reference =\n    typeof dataPoint.reference === 'string'\n      ? dataPoint.reference\n      : JSON.stringify(dataPoint.reference);\n\n  const prompt = await loadPromptFile(\n    path.resolve(getDirName(), '../../prompts/answer_accuracy.prompt')\n  );\n  const origResp = await ai.generate({\n    model: judgeLlm,\n    config: judgeConfig,\n    prompt: await renderText(prompt, {\n      query: input,\n      output,\n      reference,\n    }),\n  });\n  const origScore = Number.parseInt(origResp.text);\n  if (Number.isNaN(origScore)) {\n    throw new Error('Error generating original response for answer accuracy');\n  }\n\n  const invResp = await ai.generate({\n    model: judgeLlm,\n    config: judgeConfig,\n    prompt: await renderText(prompt, {\n      query: input,\n      output: reference,\n      reference: output,\n    }),\n  });\n  const invScore = Number.parseInt(invResp.text);\n  if (Number.isNaN(invScore)) {\n    throw new Error('Error generating inverted response for answer accuracy');\n  }\n  const score = (origScore + invScore) / 8;\n\n  return {\n    score,\n    status: score >= 0.5 ? EvalStatusEnum.PASS : EvalStatusEnum.FAIL,\n  };\n}\n"],"mappings":"AAiBA;AAAA,EACE;AAAA,OAGK;AACP,OAAO,UAAU;AACjB,SAAS,YAAY,gBAAgB,kBAAkB;AAEvD,eAAsB,oBAGpB,IACA,UACA,WACA,aACgB;AAChB,MAAI,CAAC,UAAU,QAAQ;AACrB,UAAM,IAAI,MAAM,yBAAyB;AAAA,EAC3C;AACA,MAAI,CAAC,UAAU,WAAW;AACxB,UAAM,IAAI,MAAM,4BAA4B;AAAA,EAC9C;AACA,QAAM,QACJ,OAAO,UAAU,UAAU,WACvB,UAAU,QACV,KAAK,UAAU,UAAU,KAAK;AACpC,QAAM,SACJ,OAAO,UAAU,WAAW,WACxB,UAAU,SACV,KAAK,UAAU,UAAU,MAAM;AACrC,QAAM,YACJ,OAAO,UAAU,cAAc,WAC3B,UAAU,YACV,KAAK,UAAU,UAAU,SAAS;AAExC,QAAM,SAAS,MAAM;AAAA,IACnB,KAAK,QAAQ,WAAW,GAAG,sCAAsC;AAAA,EACnE;AACA,QAAM,WAAW,MAAM,GAAG,SAAS;AAAA,IACjC,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,QAAQ,MAAM,WAAW,QAAQ;AAAA,MAC/B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,QAAM,YAAY,OAAO,SAAS,SAAS,IAAI;AAC/C,MAAI,OAAO,MAAM,SAAS,GAAG;AAC3B,UAAM,IAAI,MAAM,wDAAwD;AAAA,EAC1E;AAEA,QAAM,UAAU,MAAM,GAAG,SAAS;AAAA,IAChC,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,QAAQ,MAAM,WAAW,QAAQ;AAAA,MAC/B,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,WAAW;AAAA,IACb,CAAC;AAAA,EACH,CAAC;AACD,QAAM,WAAW,OAAO,SAAS,QAAQ,IAAI;AAC7C,MAAI,OAAO,MAAM,QAAQ,GAAG;AAC1B,UAAM,IAAI,MAAM,wDAAwD;AAAA,EAC1E;AACA,QAAM,SAAS,YAAY,YAAY;AAEvC,SAAO;AAAA,IACL;AAAA,IACA,QAAQ,SAAS,MAAM,eAAe,OAAO,eAAe;AAAA,EAC9D;AACF;","names":[]}