{"version":3,"sources":["../src/evaluators.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { Genkit, ModelArgument } from 'genkit';\nimport type { BaseEvalDataPoint } from 'genkit/evaluator';\nimport { loadEvaluator, type Criteria } from 'langchain/evaluation';\nimport { genkitModel } from './model.mjs';\nimport { GenkitTracer } from './tracing.mjs';\n\nexport function langchainEvaluator(\n  ai: Genkit,\n  type: 'labeled_criteria' | 'criteria',\n  criteria: Criteria,\n  judgeLlm: ModelArgument,\n  judgeConfig?: any\n) {\n  return ai.defineEvaluator(\n    {\n      name: `langchain/${criteria}`,\n      displayName: `${criteria}`,\n      definition: `${criteria}: refer to https://js.langchain.com/docs/guides/evaluation`,\n    },\n    async (datapoint: BaseEvalDataPoint) => {\n      try {\n        switch (type) {\n          case 'labeled_criteria':\n          case 'criteria':\n            const evaluator = await loadEvaluator(\n              type as 'labeled_criteria' | 'criteria',\n              {\n                criteria,\n                // TODO: Figure out why this breaks with TypeScript 5.3\n                llm: genkitModel(ai, judgeLlm, judgeConfig),\n                chainOptions: {\n                  callbacks: [new GenkitTracer()],\n                },\n              }\n            );\n            const lcData = {\n              input: datapoint.input as string,\n              prediction: datapoint.output as string,\n            };\n            if (datapoint.reference) {\n              lcData['reference'] = datapoint.reference as string;\n            }\n            const res = await evaluator.evaluateStrings({ ...lcData });\n            return {\n              testCaseId: datapoint.testCaseId,\n              evaluation: {\n                score: res.score,\n                details: {\n                  reasoning: res.reasoning,\n                  value: res.value,\n                },\n              },\n            };\n          default:\n            throw new Error(`unsupported evaluator type ${type}`);\n        }\n      } catch (e) {\n        return {\n          testCaseId: datapoint.testCaseId,\n          evaluation: {\n            error: `${e}`,\n          },\n        };\n      }\n    }\n  );\n}\n"],"mappings":"AAkBA,SAAS,qBAAoC;AAC7C,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;AAEtB,SAAS,mBACd,IACA,MACA,UACA,UACA,aACA;AACA,SAAO,GAAG;AAAA,IACR;AAAA,MACE,MAAM,aAAa,QAAQ;AAAA,MAC3B,aAAa,GAAG,QAAQ;AAAA,MACxB,YAAY,GAAG,QAAQ;AAAA,IACzB;AAAA,IACA,OAAO,cAAiC;AACtC,UAAI;AACF,gBAAQ,MAAM;AAAA,UACZ,KAAK;AAAA,UACL,KAAK;AACH,kBAAM,YAAY,MAAM;AAAA,cACtB;AAAA,cACA;AAAA,gBACE;AAAA;AAAA,gBAEA,KAAK,YAAY,IAAI,UAAU,WAAW;AAAA,gBAC1C,cAAc;AAAA,kBACZ,WAAW,CAAC,IAAI,aAAa,CAAC;AAAA,gBAChC;AAAA,cACF;AAAA,YACF;AACA,kBAAM,SAAS;AAAA,cACb,OAAO,UAAU;AAAA,cACjB,YAAY,UAAU;AAAA,YACxB;AACA,gBAAI,UAAU,WAAW;AACvB,qBAAO,WAAW,IAAI,UAAU;AAAA,YAClC;AACA,kBAAM,MAAM,MAAM,UAAU,gBAAgB,EAAE,GAAG,OAAO,CAAC;AACzD,mBAAO;AAAA,cACL,YAAY,UAAU;AAAA,cACtB,YAAY;AAAA,gBACV,OAAO,IAAI;AAAA,gBACX,SAAS;AAAA,kBACP,WAAW,IAAI;AAAA,kBACf,OAAO,IAAI;AAAA,gBACb;AAAA,cACF;AAAA,YACF;AAAA,UACF;AACE,kBAAM,IAAI,MAAM,8BAA8B,IAAI,EAAE;AAAA,QACxD;AAAA,MACF,SAAS,GAAG;AACV,eAAO;AAAA,UACL,YAAY,UAAU;AAAA,UACtB,YAAY;AAAA,YACV,OAAO,GAAG,CAAC;AAAA,UACb;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;","names":[]}