{"version":3,"sources":["../src/index.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { z, type Genkit } from 'genkit';\nimport {\n  evaluatorRef,\n  type BaseEvalDataPoint,\n  type EvalResponse,\n  type EvalStatusEnum,\n  type Score,\n} from 'genkit/evaluator';\nimport { genkitPlugin, type GenkitPlugin } from 'genkit/plugin';\nimport { answerAccuracyScore } from './metrics/answer_accuracy.mjs';\nimport {\n  answerRelevancyScore,\n  deepEqual,\n  faithfulnessScore,\n  jsonata,\n  maliciousnessScore,\n  regexp,\n} from './metrics/index.mjs';\nimport {\n  GenkitMetric,\n  isGenkitMetricConfig,\n  type AnswerRelevancyGenkitMetricConfig,\n  type GenkitMetricConfig,\n  type PluginOptions,\n  type ResolvedConfig,\n} from './types.mjs';\nexport { GenkitMetric, type GenkitMetricConfig, type PluginOptions };\n\nconst PLUGIN_NAME = 'genkitEval';\n\n/**\n * Reference to the Genkit evaluator for a specified metric\n */\nexport const genkitEvalRef = (metric: GenkitMetric) =>\n  evaluatorRef({\n    name: `${PLUGIN_NAME}/${metric.toLocaleLowerCase()}`,\n    configSchema: z.undefined(),\n    info: {\n      label: `Genkit RAG Evaluator for ${metric}`,\n      metrics: [metric],\n    },\n  });\n\n/**\n * Genkit evaluation plugin that provides the RAG evaluators\n */\nexport function genkitEval<\n  ModelCustomOptions extends z.ZodTypeAny,\n  EmbedderCustomOptions extends z.ZodTypeAny,\n>(\n  params: PluginOptions<ModelCustomOptions, EmbedderCustomOptions>\n): GenkitPlugin {\n  return genkitPlugin(`${PLUGIN_NAME}`, async (ai: Genkit) => {\n    genkitEvaluators(ai, params);\n  });\n}\n\nexport default genkitEval;\n\nfunction fillScores(\n  dataPoint: BaseEvalDataPoint,\n  score: Score,\n  statusOverrideFn?: (args: { score: Score }) => EvalStatusEnum\n): EvalResponse {\n  let status = score.status;\n  if (statusOverrideFn) {\n    status = statusOverrideFn({ score });\n  }\n  return { testCaseId: dataPoint.testCaseId, evaluation: { ...score, status } };\n}\n\n/**\n * Configures a Genkit evaluator\n */\nexport function genkitEvaluators<\n  ModelCustomOptions extends z.ZodTypeAny,\n  EmbedderCustomOptions extends z.ZodTypeAny,\n>(\n  ai: Genkit,\n  params: PluginOptions<ModelCustomOptions, EmbedderCustomOptions>\n) {\n  const { metrics } = params;\n  if (metrics.length === 0) {\n    throw new Error('No metrics configured in genkitEval plugin');\n  }\n  return metrics.map((metric) => {\n    const {\n      type,\n      judge,\n      judgeConfig,\n      embedder,\n      embedderOptions,\n      statusOverrideFn,\n    } = resolveConfig(metric, params);\n    const evaluator = `${PLUGIN_NAME}/${type.toLocaleLowerCase()}`;\n    switch (type) {\n      case GenkitMetric.ANSWER_RELEVANCY: {\n        if (!judge) {\n          throw new Error(\n            'Judge llms must be specified if computing answer relvancy'\n          );\n        }\n        if (!embedder) {\n          throw new Error(\n            'Embedder must be specified if computing answer relvancy'\n          );\n        }\n        return ai.defineEvaluator(\n          {\n            name: evaluator,\n            displayName: 'Answer Relevancy',\n            definition:\n              'Assesses how pertinent the generated answer is to the given prompt',\n          },\n          async (datapoint: BaseEvalDataPoint) => {\n            const answerRelevancy = await answerRelevancyScore(\n              ai,\n              judge!,\n              datapoint,\n              embedder!,\n              judgeConfig,\n              embedderOptions\n            );\n            return fillScores(datapoint, answerRelevancy, statusOverrideFn);\n          }\n        );\n      }\n      case GenkitMetric.FAITHFULNESS: {\n        if (!judge) {\n          throw new Error(\n            'Judge llms must be specified if computing faithfulness'\n          );\n        }\n        return ai.defineEvaluator(\n          {\n            name: evaluator,\n            displayName: 'Faithfulness',\n            definition:\n              'Measures the factual consistency of the generated answer against the given context',\n          },\n          async (datapoint: BaseEvalDataPoint) => {\n            const faithfulness = await faithfulnessScore(\n              ai,\n              judge!,\n              datapoint,\n              judgeConfig\n            );\n            return fillScores(datapoint, faithfulness, statusOverrideFn);\n          }\n        );\n      }\n      case GenkitMetric.MALICIOUSNESS: {\n        if (!judge) {\n          throw new Error(\n            'Judge llms must be specified if computing maliciousness'\n          );\n        }\n        return ai.defineEvaluator(\n          {\n            name: evaluator,\n            displayName: 'Maliciousness',\n            definition:\n              'Measures whether the generated output intends to deceive, harm, or exploit',\n          },\n          async (datapoint: BaseEvalDataPoint) => {\n            const maliciousness = await maliciousnessScore(\n              ai,\n              judge!,\n              datapoint,\n              judgeConfig\n            );\n            return fillScores(datapoint, maliciousness, statusOverrideFn);\n          }\n        );\n      }\n      case GenkitMetric.ANSWER_ACCURACY: {\n        if (!judge) {\n          throw new Error(\n            'Judge llms must be specified if computing answer accuracy'\n          );\n        }\n        return ai.defineEvaluator(\n          {\n            name: evaluator,\n            displayName: 'Answer Accuracy',\n            definition:\n              'Measures how accurately the generated output matches against the reference output',\n          },\n          async (datapoint: BaseEvalDataPoint) => {\n            const answerAccuracy = await answerAccuracyScore(\n              ai,\n              judge!,\n              datapoint,\n              judgeConfig\n            );\n            return fillScores(datapoint, answerAccuracy, statusOverrideFn);\n          }\n        );\n      }\n      case GenkitMetric.REGEX: {\n        return ai.defineEvaluator(\n          {\n            name: evaluator,\n            displayName: 'RegExp',\n            definition: 'Tests output against the regexp provided as reference',\n          },\n          async (datapoint: BaseEvalDataPoint) => {\n            return fillScores(datapoint, await regexp(datapoint));\n          }\n        );\n      }\n      case GenkitMetric.DEEP_EQUAL: {\n        return ai.defineEvaluator(\n          {\n            name: evaluator,\n            displayName: 'Deep Equals',\n            definition:\n              'Tests equality of output against the provided reference',\n          },\n          async (datapoint: BaseEvalDataPoint) => {\n            return fillScores(\n              datapoint,\n              await deepEqual(datapoint),\n              statusOverrideFn\n            );\n          }\n        );\n      }\n      case GenkitMetric.JSONATA: {\n        return ai.defineEvaluator(\n          {\n            name: evaluator,\n            displayName: 'JSONata',\n            definition:\n              'Tests JSONata expression (provided in reference) against output',\n          },\n          async (datapoint: BaseEvalDataPoint) => {\n            return fillScores(\n              datapoint,\n              await jsonata(datapoint),\n              statusOverrideFn\n            );\n          }\n        );\n      }\n    }\n  });\n}\n\nfunction resolveConfig<M extends z.ZodTypeAny, E extends z.ZodTypeAny>(\n  metric: GenkitMetricConfig<M, E>,\n  params: PluginOptions<M, E>\n): ResolvedConfig<M, E> {\n  if (isGenkitMetricConfig(metric)) {\n    return {\n      type: metric.type,\n      statusOverrideFn: metric.statusOverrideFn,\n      judge: metric.judge ?? params.judge,\n      judgeConfig: metric.judgeConfig ?? params.judgeConfig,\n      embedder:\n        metric.type === GenkitMetric.ANSWER_RELEVANCY\n          ? (metric as AnswerRelevancyGenkitMetricConfig<M, E>).embedder\n          : undefined,\n      embedderOptions:\n        metric.type === GenkitMetric.ANSWER_RELEVANCY\n          ? (metric as AnswerRelevancyGenkitMetricConfig<M, E>).embedderOptions\n          : undefined,\n    } as ResolvedConfig<M, E>;\n  }\n  return { type: metric, ...params };\n}\n"],"mappings":"AAgBA,SAAS,SAAsB;AAC/B;AAAA,EACE;AAAA,OAKK;AACP,SAAS,oBAAuC;AAChD,SAAS,2BAA2B;AACpC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,OAKK;AAGP,MAAM,cAAc;AAKb,MAAM,gBAAgB,CAAC,WAC5B,aAAa;AAAA,EACX,MAAM,GAAG,WAAW,IAAI,OAAO,kBAAkB,CAAC;AAAA,EAClD,cAAc,EAAE,UAAU;AAAA,EAC1B,MAAM;AAAA,IACJ,OAAO,4BAA4B,MAAM;AAAA,IACzC,SAAS,CAAC,MAAM;AAAA,EAClB;AACF,CAAC;AAKI,SAAS,WAId,QACc;AACd,SAAO,aAAa,GAAG,WAAW,IAAI,OAAO,OAAe;AAC1D,qBAAiB,IAAI,MAAM;AAAA,EAC7B,CAAC;AACH;AAEA,IAAO,gBAAQ;AAEf,SAAS,WACP,WACA,OACA,kBACc;AACd,MAAI,SAAS,MAAM;AACnB,MAAI,kBAAkB;AACpB,aAAS,iBAAiB,EAAE,MAAM,CAAC;AAAA,EACrC;AACA,SAAO,EAAE,YAAY,UAAU,YAAY,YAAY,EAAE,GAAG,OAAO,OAAO,EAAE;AAC9E;AAKO,SAAS,iBAId,IACA,QACA;AACA,QAAM,EAAE,QAAQ,IAAI;AACpB,MAAI,QAAQ,WAAW,GAAG;AACxB,UAAM,IAAI,MAAM,4CAA4C;AAAA,EAC9D;AACA,SAAO,QAAQ,IAAI,CAAC,WAAW;AAC7B,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,IAAI,cAAc,QAAQ,MAAM;AAChC,UAAM,YAAY,GAAG,WAAW,IAAI,KAAK,kBAAkB,CAAC;AAC5D,YAAQ,MAAM;AAAA,MACZ,KAAK,aAAa,kBAAkB;AAClC,YAAI,CAAC,OAAO;AACV,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA,YAAI,CAAC,UAAU;AACb,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA,eAAO,GAAG;AAAA,UACR;AAAA,YACE,MAAM;AAAA,YACN,aAAa;AAAA,YACb,YACE;AAAA,UACJ;AAAA,UACA,OAAO,cAAiC;AACtC,kBAAM,kBAAkB,MAAM;AAAA,cAC5B;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,YACF;AACA,mBAAO,WAAW,WAAW,iBAAiB,gBAAgB;AAAA,UAChE;AAAA,QACF;AAAA,MACF;AAAA,MACA,KAAK,aAAa,cAAc;AAC9B,YAAI,CAAC,OAAO;AACV,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA,eAAO,GAAG;AAAA,UACR;AAAA,YACE,MAAM;AAAA,YACN,aAAa;AAAA,YACb,YACE;AAAA,UACJ;AAAA,UACA,OAAO,cAAiC;AACtC,kBAAM,eAAe,MAAM;AAAA,cACzB;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,YACF;AACA,mBAAO,WAAW,WAAW,cAAc,gBAAgB;AAAA,UAC7D;AAAA,QACF;AAAA,MACF;AAAA,MACA,KAAK,aAAa,eAAe;AAC/B,YAAI,CAAC,OAAO;AACV,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA,eAAO,GAAG;AAAA,UACR;AAAA,YACE,MAAM;AAAA,YACN,aAAa;AAAA,YACb,YACE;AAAA,UACJ;AAAA,UACA,OAAO,cAAiC;AACtC,kBAAM,gBAAgB,MAAM;AAAA,cAC1B;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,YACF;AACA,mBAAO,WAAW,WAAW,eAAe,gBAAgB;AAAA,UAC9D;AAAA,QACF;AAAA,MACF;AAAA,MACA,KAAK,aAAa,iBAAiB;AACjC,YAAI,CAAC,OAAO;AACV,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA,eAAO,GAAG;AAAA,UACR;AAAA,YACE,MAAM;AAAA,YACN,aAAa;AAAA,YACb,YACE;AAAA,UACJ;AAAA,UACA,OAAO,cAAiC;AACtC,kBAAM,iBAAiB,MAAM;AAAA,cAC3B;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,YACF;AACA,mBAAO,WAAW,WAAW,gBAAgB,gBAAgB;AAAA,UAC/D;AAAA,QACF;AAAA,MACF;AAAA,MACA,KAAK,aAAa,OAAO;AACvB,eAAO,GAAG;AAAA,UACR;AAAA,YACE,MAAM;AAAA,YACN,aAAa;AAAA,YACb,YAAY;AAAA,UACd;AAAA,UACA,OAAO,cAAiC;AACtC,mBAAO,WAAW,WAAW,MAAM,OAAO,SAAS,CAAC;AAAA,UACtD;AAAA,QACF;AAAA,MACF;AAAA,MACA,KAAK,aAAa,YAAY;AAC5B,eAAO,GAAG;AAAA,UACR;AAAA,YACE,MAAM;AAAA,YACN,aAAa;AAAA,YACb,YACE;AAAA,UACJ;AAAA,UACA,OAAO,cAAiC;AACtC,mBAAO;AAAA,cACL;AAAA,cACA,MAAM,UAAU,SAAS;AAAA,cACzB;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,MACA,KAAK,aAAa,SAAS;AACzB,eAAO,GAAG;AAAA,UACR;AAAA,YACE,MAAM;AAAA,YACN,aAAa;AAAA,YACb,YACE;AAAA,UACJ;AAAA,UACA,OAAO,cAAiC;AACtC,mBAAO;AAAA,cACL;AAAA,cACA,MAAM,QAAQ,SAAS;AAAA,cACvB;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,SAAS,cACP,QACA,QACsB;AACtB,MAAI,qBAAqB,MAAM,GAAG;AAChC,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,kBAAkB,OAAO;AAAA,MACzB,OAAO,OAAO,SAAS,OAAO;AAAA,MAC9B,aAAa,OAAO,eAAe,OAAO;AAAA,MAC1C,UACE,OAAO,SAAS,aAAa,mBACxB,OAAmD,WACpD;AAAA,MACN,iBACE,OAAO,SAAS,aAAa,mBACxB,OAAmD,kBACpD;AAAA,IACR;AAAA,EACF;AACA,SAAO,EAAE,MAAM,QAAQ,GAAG,OAAO;AACnC;","names":[]}