{"version":3,"sources":["../../src/evaluation/evaluator_factory.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { type Action, type Genkit, type z } from 'genkit';\nimport type { BaseEvalDataPoint, Score } from 'genkit/evaluator';\nimport { runInNewSpan } from 'genkit/tracing';\nimport type { GoogleAuth } from 'google-auth-library';\nimport { getGenkitClientHeader } from '../common/index.js';\nimport type { VertexAIEvaluationMetricType } from './evaluation.js';\n\nexport class EvaluatorFactory {\n  constructor(\n    private readonly auth: GoogleAuth,\n    private readonly location: string,\n    private readonly projectId: string\n  ) {}\n\n  create<ResponseType extends z.ZodTypeAny>(\n    ai: Genkit,\n    config: {\n      metric: VertexAIEvaluationMetricType;\n      displayName: string;\n      definition: string;\n      responseSchema: ResponseType;\n    },\n    toRequest: (datapoint: BaseEvalDataPoint) => any,\n    responseHandler: (response: z.infer<ResponseType>) => Score\n  ): Action {\n    return ai.defineEvaluator(\n      {\n        name: `vertexai/${config.metric.toLocaleLowerCase()}`,\n        displayName: config.displayName,\n        definition: config.definition,\n      },\n      async (datapoint: BaseEvalDataPoint) => {\n        const responseSchema = config.responseSchema;\n        const response = await this.evaluateInstances(\n          ai,\n          toRequest(datapoint),\n          responseSchema\n        );\n\n        return {\n          evaluation: responseHandler(response),\n          testCaseId: datapoint.testCaseId,\n        };\n      }\n    );\n  }\n\n  async evaluateInstances<ResponseType extends z.ZodTypeAny>(\n    ai: Genkit,\n    partialRequest: any,\n    responseSchema: ResponseType\n  ): Promise<z.infer<ResponseType>> {\n    const locationName = `projects/${this.projectId}/locations/${this.location}`;\n    return await runInNewSpan(\n      ai,\n      {\n        metadata: {\n          name: 'EvaluationService#evaluateInstances',\n        },\n      },\n      async (metadata, _otSpan) => {\n        const request = {\n          location: locationName,\n          ...partialRequest,\n        };\n\n        metadata.input = request;\n        const client = await this.auth.getClient();\n        const url = `https://${this.location}-aiplatform.googleapis.com/v1beta1/${locationName}:evaluateInstances`;\n        const response = await client.request({\n          url,\n          method: 'POST',\n          body: JSON.stringify(request),\n          headers: {\n            'X-Goog-Api-Client': getGenkitClientHeader(),\n          },\n        });\n        metadata.output = response.data;\n\n        try {\n          return responseSchema.parse(response.data);\n        } catch (e) {\n          throw new Error(`Error parsing ${url} API response: ${e}`);\n        }\n      }\n    );\n  }\n}\n"],"mappings":"AAkBA,SAAS,oBAAoB;AAE7B,SAAS,6BAA6B;AAG/B,MAAM,iBAAiB;AAAA,EAC5B,YACmB,MACA,UACA,WACjB;AAHiB;AACA;AACA;AAAA,EAChB;AAAA,EAEH,OACE,IACA,QAMA,WACA,iBACQ;AACR,WAAO,GAAG;AAAA,MACR;AAAA,QACE,MAAM,YAAY,OAAO,OAAO,kBAAkB,CAAC;AAAA,QACnD,aAAa,OAAO;AAAA,QACpB,YAAY,OAAO;AAAA,MACrB;AAAA,MACA,OAAO,cAAiC;AACtC,cAAM,iBAAiB,OAAO;AAC9B,cAAM,WAAW,MAAM,KAAK;AAAA,UAC1B;AAAA,UACA,UAAU,SAAS;AAAA,UACnB;AAAA,QACF;AAEA,eAAO;AAAA,UACL,YAAY,gBAAgB,QAAQ;AAAA,UACpC,YAAY,UAAU;AAAA,QACxB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,kBACJ,IACA,gBACA,gBACgC;AAChC,UAAM,eAAe,YAAY,KAAK,SAAS,cAAc,KAAK,QAAQ;AAC1E,WAAO,MAAM;AAAA,MACX;AAAA,MACA;AAAA,QACE,UAAU;AAAA,UACR,MAAM;AAAA,QACR;AAAA,MACF;AAAA,MACA,OAAO,UAAU,YAAY;AAC3B,cAAM,UAAU;AAAA,UACd,UAAU;AAAA,UACV,GAAG;AAAA,QACL;AAEA,iBAAS,QAAQ;AACjB,cAAM,SAAS,MAAM,KAAK,KAAK,UAAU;AACzC,cAAM,MAAM,WAAW,KAAK,QAAQ,sCAAsC,YAAY;AACtF,cAAM,WAAW,MAAM,OAAO,QAAQ;AAAA,UACpC;AAAA,UACA,QAAQ;AAAA,UACR,MAAM,KAAK,UAAU,OAAO;AAAA,UAC5B,SAAS;AAAA,YACP,qBAAqB,sBAAsB;AAAA,UAC7C;AAAA,QACF,CAAC;AACD,iBAAS,SAAS,SAAS;AAE3B,YAAI;AACF,iBAAO,eAAe,MAAM,SAAS,IAAI;AAAA,QAC3C,SAAS,GAAG;AACV,gBAAM,IAAI,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,EAAE;AAAA,QAC3D;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;","names":[]}