{"version":3,"sources":["../../src/testing/model-tester.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { z } from '@genkit-ai/core';\nimport type { Registry } from '@genkit-ai/core/registry';\nimport { runInNewSpan } from '@genkit-ai/core/tracing';\nimport * as assert from 'assert';\nimport { generate } from '../generate';\nimport type { ModelAction } from '../model';\nimport { defineTool } from '../tool';\n\nconst tests: Record = {\n 'basic hi': async (registry: Registry, model: string) => {\n const response = await generate(registry, {\n model,\n prompt: 'just say \"Hi\", literally',\n });\n\n const got = response.text.trim();\n assert.match(got, /Hi/i);\n },\n multimodal: async (registry: Registry, model: string) => {\n const resolvedModel = (await registry.lookupAction(\n `/model/${model}`\n )) as ModelAction;\n if (!resolvedModel.__action.metadata?.model.supports?.media) {\n skip();\n }\n const response = await generate(registry, {\n model,\n prompt: [\n {\n media: {\n url: 'data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAABhGlDQ1BJQ0MgcHJvZmlsZQAAKJF9kT1Iw0AcxV9TpSoVETOIOGSoulgQFXHUKhShQqgVWnUwufRDaNKQtLg4Cq4FBz8Wqw4uzro6uAqC4AeIs4OToouU+L+k0CLGg+N+vLv3uHsHCLUi0+22MUA3ylYyHpPSmRUp9IpOhCCiFyMKs81ZWU7Ad3zdI8DXuyjP8j/35+jWsjYDAhLxDDOtMvE68dRm2eS8TyyygqIRnxOPWnRB4keuqx6/cc67LPBM0Uol54hFYinfwmoLs4KlE08SRzTdoHwh7bHGeYuzXqywxj35C8NZY3mJ6zQHEccCFiFDgooKNlBEGVFaDVJsJGk/5uMfcP0yuVRybYCRYx4l6FBcP/gf/O7Wzk2Me0nhGND+4jgfQ0BoF6hXHef72HHqJ0DwGbgymv5SDZj+JL3a1CJHQM82cHHd1NQ94HIH6H8yFUtxpSBNIZcD3s/omzJA3y3Qter11tjH6QOQoq4SN8DBITCcp+w1n3d3tPb275lGfz9aC3Kd0jYiSQAAAAlwSFlzAAAuIwAALiMBeKU/dgAAAAd0SU1FB+gJBxQRO1/5qB8AAAAZdEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIEdJTVBXgQ4XAAAAsUlEQVQoz61SMQqEMBDcO5SYToUE/IBPyRMCftAH+INUviApUwYjNkKCVcTiQK7IHSw45czODrMswCOQUkopEQZjzDiOWemdZfu+b5oGYYgx1nWNMPwB2vACAK01Y4wQ8qGqqirL8jzPlNI9t64r55wQUgBA27be+xDCfaJhGJxzSqnv3UKIn7ne+2VZEB2stZRSRLN93+d5RiRs28Y5RySEEI7jyEpFlp2mqeu6Zx75ApQwPdsIcq0ZAAAAAElFTkSuQmCC',\n },\n },\n {\n text: 'what math operation is this? plus, minus, multiply or divide?',\n },\n ],\n });\n\n const want = /plus/i;\n const got = response.text.trim();\n assert.match(got, want);\n },\n history: async (registry: Registry, model: string) => {\n const resolvedModel = (await registry.lookupAction(\n `/model/${model}`\n )) as ModelAction;\n if (!resolvedModel.__action.metadata?.model.supports?.multiturn) {\n skip();\n }\n const response1 = await generate(registry, {\n model,\n prompt: 'My name is Glorb',\n });\n const response = await generate(registry, {\n model,\n prompt: \"What's my name?\",\n messages: response1.messages,\n });\n\n const got = response.text.trim();\n assert.match(got, /Glorb/);\n },\n 'system prompt': async (registry: Registry, model: string) => {\n const { text } = await generate(registry, {\n model,\n prompt: 'Hi',\n messages: [\n {\n role: 'system',\n content: [\n {\n text: 'If the user says \"Hi\", just say \"Bye\" ',\n },\n ],\n },\n ],\n });\n\n const want = 'Bye';\n const got = text.trim();\n assert.equal(got, want);\n },\n 'structured output': async (registry: Registry, model: string) => {\n const response = await generate(registry, {\n model,\n prompt: 'extract data as json from: Jack was a Lumberjack',\n output: {\n format: 'json',\n schema: z.object({\n name: z.string(),\n occupation: z.string(),\n }),\n },\n });\n\n const want = {\n name: 'Jack',\n occupation: 'Lumberjack',\n };\n const got = response.output;\n assert.deepEqual(want, got);\n },\n 'tool calling': async (registry: Registry, model: string) => {\n const resolvedModel = (await registry.lookupAction(\n `/model/${model}`\n )) as ModelAction;\n if (!resolvedModel.__action.metadata?.model.supports?.tools) {\n skip();\n }\n\n const { text } = await generate(registry, {\n model,\n prompt: 'what is a gablorken of 2? use provided tool',\n tools: ['gablorkenTool'],\n });\n\n const got = text.trim();\n assert.match(got, /9.407/);\n },\n};\n\ntype TestReport = {\n description: string;\n models: {\n name: string;\n passed: boolean;\n skipped?: boolean;\n error?: {\n message: string;\n stack?: string;\n };\n }[];\n}[];\n\ntype TestCase = (ai: Registry, model: string) => Promise;\n\nexport async function testModels(\n registry: Registry,\n models: string[]\n): Promise {\n defineTool(\n registry,\n {\n name: 'gablorkenTool',\n description: 'use when need to calculate a gablorken',\n inputSchema: z.object({\n value: z.number(),\n }),\n outputSchema: z.number(),\n },\n async (input) => {\n return Math.pow(input.value, 3) + 1.407;\n }\n );\n\n return await runInNewSpan(\n registry,\n { metadata: { name: 'testModels' } },\n async () => {\n const report: TestReport = [];\n for (const test of Object.keys(tests)) {\n await runInNewSpan(registry, { metadata: { name: test } }, async () => {\n report.push({\n description: test,\n models: [],\n });\n const caseReport = report[report.length - 1];\n for (const model of models) {\n caseReport.models.push({\n name: model,\n passed: true, // optimistically\n });\n const modelReport = caseReport.models[caseReport.models.length - 1];\n try {\n await tests[test](registry, model);\n } catch (e) {\n modelReport.passed = false;\n if (e instanceof SkipTestError) {\n modelReport.skipped = true;\n } else if (e instanceof Error) {\n modelReport.error = {\n message: e.message,\n stack: e.stack,\n };\n } else {\n modelReport.error = {\n message: `${e}`,\n };\n }\n }\n }\n });\n }\n\n return report;\n }\n );\n}\n\nclass SkipTestError extends Error {}\n\nfunction skip() {\n throw new SkipTestError();\n}\n"],"mappings":"AAgBA,SAAS,SAAS;AAElB,SAAS,oBAAoB;AAC7B,YAAY,YAAY;AACxB,SAAS,gBAAgB;AAEzB,SAAS,kBAAkB;AAE3B,MAAM,QAAkC;AAAA,EACtC,YAAY,OAAO,UAAoB,UAAkB;AACvD,UAAM,WAAW,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,IACV,CAAC;AAED,UAAM,MAAM,SAAS,KAAK,KAAK;AAC/B,WAAO,MAAM,KAAK,KAAK;AAAA,EACzB;AAAA,EACA,YAAY,OAAO,UAAoB,UAAkB;AACvD,UAAM,gBAAiB,MAAM,SAAS;AAAA,MACpC,UAAU,KAAK;AAAA,IACjB;AACA,QAAI,CAAC,cAAc,SAAS,UAAU,MAAM,UAAU,OAAO;AAC3D,WAAK;AAAA,IACP;AACA,UAAM,WAAW,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,QACN;AAAA,UACE,OAAO;AAAA,YACL,KAAK;AAAA,UACP;AAAA,QACF;AAAA,QACA;AAAA,UACE,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,OAAO;AACb,UAAM,MAAM,SAAS,KAAK,KAAK;AAC/B,WAAO,MAAM,KAAK,IAAI;AAAA,EACxB;AAAA,EACA,SAAS,OAAO,UAAoB,UAAkB;AACpD,UAAM,gBAAiB,MAAM,SAAS;AAAA,MACpC,UAAU,KAAK;AAAA,IACjB;AACA,QAAI,CAAC,cAAc,SAAS,UAAU,MAAM,UAAU,WAAW;AAC/D,WAAK;AAAA,IACP;AACA,UAAM,YAAY,MAAM,SAAS,UAAU;AAAA,MACzC;AAAA,MACA,QAAQ;AAAA,IACV,CAAC;AACD,UAAM,WAAW,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,MACR,UAAU,UAAU;AAAA,IACtB,CAAC;AAED,UAAM,MAAM,SAAS,KAAK,KAAK;AAC/B,WAAO,MAAM,KAAK,OAAO;AAAA,EAC3B;AAAA,EACA,iBAAiB,OAAO,UAAoB,UAAkB;AAC5D,UAAM,EAAE,KAAK,IAAI,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,MACR,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS;AAAA,YACP;AAAA,cACE,MAAM;AAAA,YACR;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,OAAO;AACb,UAAM,MAAM,KAAK,KAAK;AACtB,WAAO,MAAM,KAAK,IAAI;AAAA,EACxB;AAAA,EACA,qBAAqB,OAAO,UAAoB,UAAkB;AAChE,UAAM,WAAW,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,MACR,QAAQ;AAAA,QACN,QAAQ;AAAA,QACR,QAAQ,EAAE,OAAO;AAAA,UACf,MAAM,EAAE,OAAO;AAAA,UACf,YAAY,EAAE,OAAO;AAAA,QACvB,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,UAAM,OAAO;AAAA,MACX,MAAM;AAAA,MACN,YAAY;AAAA,IACd;AACA,UAAM,MAAM,SAAS;AACrB,WAAO,UAAU,MAAM,GAAG;AAAA,EAC5B;AAAA,EACA,gBAAgB,OAAO,UAAoB,UAAkB;AAC3D,UAAM,gBAAiB,MAAM,SAAS;AAAA,MACpC,UAAU,KAAK;AAAA,IACjB;AACA,QAAI,CAAC,cAAc,SAAS,UAAU,MAAM,UAAU,OAAO;AAC3D,WAAK;AAAA,IACP;AAEA,UAAM,EAAE,KAAK,IAAI,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,MACR,OAAO,CAAC,eAAe;AAAA,IACzB,CAAC;AAED,UAAM,MAAM,KAAK,KAAK;AACtB,WAAO,MAAM,KAAK,OAAO;AAAA,EAC3B;AACF;AAiBA,eAAsB,WACpB,UACA,QACqB;AACrB;AAAA,IACE;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,aAAa;AAAA,MACb,aAAa,EAAE,OAAO;AAAA,QACpB,OAAO,EAAE,OAAO;AAAA,MAClB,CAAC;AAAA,MACD,cAAc,EAAE,OAAO;AAAA,IACzB;AAAA,IACA,OAAO,UAAU;AACf,aAAO,KAAK,IAAI,MAAM,OAAO,CAAC,IAAI;AAAA,IACpC;AAAA,EACF;AAEA,SAAO,MAAM;AAAA,IACX;AAAA,IACA,EAAE,UAAU,EAAE,MAAM,aAAa,EAAE;AAAA,IACnC,YAAY;AACV,YAAM,SAAqB,CAAC;AAC5B,iBAAW,QAAQ,OAAO,KAAK,KAAK,GAAG;AACrC,cAAM,aAAa,UAAU,EAAE,UAAU,EAAE,MAAM,KAAK,EAAE,GAAG,YAAY;AACrE,iBAAO,KAAK;AAAA,YACV,aAAa;AAAA,YACb,QAAQ,CAAC;AAAA,UACX,CAAC;AACD,gBAAM,aAAa,OAAO,OAAO,SAAS,CAAC;AAC3C,qBAAW,SAAS,QAAQ;AAC1B,uBAAW,OAAO,KAAK;AAAA,cACrB,MAAM;AAAA,cACN,QAAQ;AAAA;AAAA,YACV,CAAC;AACD,kBAAM,cAAc,WAAW,OAAO,WAAW,OAAO,SAAS,CAAC;AAClE,gBAAI;AACF,oBAAM,MAAM,IAAI,EAAE,UAAU,KAAK;AAAA,YACnC,SAAS,GAAG;AACV,0BAAY,SAAS;AACrB,kBAAI,aAAa,eAAe;AAC9B,4BAAY,UAAU;AAAA,cACxB,WAAW,aAAa,OAAO;AAC7B,4BAAY,QAAQ;AAAA,kBAClB,SAAS,EAAE;AAAA,kBACX,OAAO,EAAE;AAAA,gBACX;AAAA,cACF,OAAO;AACL,4BAAY,QAAQ;AAAA,kBAClB,SAAS,GAAG,CAAC;AAAA,gBACf;AAAA,cACF;AAAA,YACF;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEA,MAAM,sBAAsB,MAAM;AAAC;AAEnC,SAAS,OAAO;AACd,QAAM,IAAI,cAAc;AAC1B;","names":[]}