{"version":3,"file":"llm.cjs","names":["z","parseArgs","readTestCases","path","fs","generateText","DecisionCode"],"sources":["../../src/presets/llm.ts"],"sourcesContent":["import fs from 'node:fs';\nimport path from 'node:path';\n\nimport { bedrock } from '@ai-sdk/amazon-bedrock';\nimport { google } from '@ai-sdk/google';\nimport { openai } from '@ai-sdk/openai';\nimport { xai } from '@ai-sdk/xai';\nimport type { LanguageModel, ModelMessage } from 'ai';\nimport { generateText } from 'ai';\nimport { z } from 'zod';\n\nimport { parseArgs } from '../helpers/parseArgs.js';\nimport { printTestCaseResult } from '../helpers/printTestCaseResult.js';\nimport { readTestCases } from '../helpers/readTestCases.js';\nimport { DecisionCode } from '../types/decisionCode.js';\nimport type { TestCaseResult } from '../types/testCaseResult.js';\n\nconst PROMPT_FILENAME = 'prompt.txt';\n\nconst providerByName: Record<string, typeof bedrock | typeof google | typeof openai | typeof xai> = {\n  // requires `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`\n  bedrock,\n  // requires `GOOGLE_GENERATIVE_AI_API_KEY`\n  google,\n  // requires `OPENAI_API_KEY`\n  openai,\n  // requires `XAI_API_KEY`\n  xai,\n} as const;\n\nconst judgeParamsSchema = z.object({\n  model: z.string().min(1),\n});\n\ninterface LlmJudgePresetOptions {\n  buildPrompt?: (context: {\n    prompt: string;\n    testCase: { id: string; input?: string; output?: string };\n  }) => string | ModelMessage[];\n  test: (context: {\n    testCase: { id: string; input?: string; output?: string };\n    result: { output: string };\n  }) => Partial<TestCaseResult> | Promise<Partial<TestCaseResult>>;\n}\n\n/**\n * A preset judge function for running and testing a user prompt in LLM.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { llmJudgePreset } from '@exercode/problem-utils/presets/llm';\n * import { DecisionCode } from '@exercode/problem-utils';\n *\n * await llmJudgePreset(import.meta.dirname, {\n *   test: (context) {\n *     return { decisionCode: context.result.output ? DecisionCode.ACCEPTED : DecisionCode.WRONG_ANSWER };\n *   }\n * });\n * ```\n *\n * Run with the required parameters:\n * ```bash\n * bun judge.ts model_answers/java '{ \"model\": \"gemini-2.5-flash-lite\" }'\n * ```\n */\nexport async function llmJudgePreset(problemDir: string, options: LlmJudgePresetOptions): Promise<void> {\n  const args = parseArgs(process.argv);\n  if (!args.cwd) throw new Error('cwd argument required');\n  const params = judgeParamsSchema.parse(args.params);\n\n  const testCases = await readTestCases(path.join(problemDir, 'test_cases'));\n\n  const prompt = await fs.promises.readFile(path.join(args.cwd, PROMPT_FILENAME), 'utf8');\n\n  for (const testCase of testCases) {\n    const startTimeMilliseconds = Date.now();\n    try {\n      const { text } = await generateText({\n        model: toLanguageModel(params.model),\n        prompt: options.buildPrompt?.({ prompt, testCase }) ?? prompt.replaceAll('{input}', testCase.input ?? ''),\n      });\n\n      const stopTimeMilliseconds = Date.now();\n\n      const testCaseResult = {\n        testCaseId: testCase.id,\n        decisionCode: DecisionCode.ACCEPTED,\n        stdin: testCase.input,\n        stdout: text,\n        timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n        ...(await options.test({ testCase, result: { output: text } })),\n      };\n\n      printTestCaseResult(testCaseResult);\n\n      if (testCaseResult.decisionCode !== DecisionCode.ACCEPTED) break;\n    } catch (error) {\n      const stopTimeMilliseconds = Date.now();\n\n      printTestCaseResult({\n        testCaseId: testCase.id,\n        decisionCode: DecisionCode.RUNTIME_ERROR,\n        stdin: testCase.input,\n        stderr: error instanceof Error ? error.message : String(error),\n        timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n      });\n\n      break;\n    }\n  }\n}\n\nfunction toLanguageModel(model: string): LanguageModel {\n  const [providerId, modelId] = model.split('/');\n  if (!providerId || !modelId) throw new Error(`bad model: ${model}`);\n  const languageModel = providerByName[providerId]?.(modelId);\n  if (!languageModel) throw new Error(`model not found: ${model}`);\n  return languageModel;\n}\n"],"mappings":"4hBAiBA,MAEM,EAA8F,CAElG,QAAA,EAAA,QAEA,OAAA,EAAA,OAEA,OAAA,EAAA,OAEA,IAAA,EAAA,GACF,EAEM,EAAoBA,EAAAA,EAAE,OAAO,CACjC,MAAOA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CACzB,CAAC,EAkCD,eAAsB,EAAe,EAAoB,EAA+C,CACtG,IAAM,EAAOC,EAAAA,UAAU,QAAQ,IAAI,EACnC,GAAI,CAAC,EAAK,IAAK,MAAU,MAAM,uBAAuB,EACtD,IAAM,EAAS,EAAkB,MAAM,EAAK,MAAM,EAE5C,EAAY,MAAMC,EAAAA,cAAcC,EAAAA,QAAK,KAAK,EAAY,YAAY,CAAC,EAEnE,EAAS,MAAMC,EAAAA,QAAG,SAAS,SAASD,EAAAA,QAAK,KAAK,EAAK,IAAK,YAAe,EAAG,MAAM,EAEtF,IAAK,IAAM,KAAY,EAAW,CAChC,IAAM,EAAwB,KAAK,IAAI,EACvC,GAAI,CACF,GAAM,CAAE,QAAS,MAAA,EAAME,EAAAA,aAAAA,CAAa,CAClC,MAAO,EAAgB,EAAO,KAAK,EACnC,OAAQ,EAAQ,cAAc,CAAE,SAAQ,UAAS,CAAC,GAAK,EAAO,WAAW,UAAW,EAAS,OAAS,EAAE,CAC1G,CAAC,EAEK,EAAuB,KAAK,IAAI,EAEhC,EAAiB,CACrB,WAAY,EAAS,GACrB,aAAcC,EAAAA,aAAa,SAC3B,MAAO,EAAS,MAChB,OAAQ,EACR,aAAc,EAAuB,GAAyB,IAC9D,GAAI,MAAM,EAAQ,KAAK,CAAE,WAAU,OAAQ,CAAE,OAAQ,CAAK,CAAE,CAAC,CAC/D,EAIA,GAFA,EAAA,oBAAoB,CAAc,EAE9B,EAAe,eAAiBA,EAAAA,aAAa,SAAU,KAC7D,OAAS,EAAO,CACd,IAAM,EAAuB,KAAK,IAAI,EAEtC,EAAA,oBAAoB,CAClB,WAAY,EAAS,GACrB,aAAcA,EAAAA,aAAa,cAC3B,MAAO,EAAS,MAChB,OAAQ,aAAiB,MAAQ,EAAM,QAAU,OAAO,CAAK,EAC7D,aAAc,EAAuB,GAAyB,GAChE,CAAC,EAED,KACF,CACF,CACF,CAEA,SAAS,EAAgB,EAA8B,CACrD,GAAM,CAAC,EAAY,GAAW,EAAM,MAAM,GAAG,EAC7C,GAAI,CAAC,GAAc,CAAC,EAAS,MAAU,MAAM,cAAc,GAAO,EAClE,IAAM,EAAgB,EAAe,EAAW,GAAG,CAAO,EAC1D,GAAI,CAAC,EAAe,MAAU,MAAM,oBAAoB,GAAO,EAC/D,OAAO,CACT"}