{"version":3,"file":"evaluation-f-2QwBid.mjs","names":[],"sources":["../src/core/model-output.ts","../src/core/evaluation.ts"],"sourcesContent":["const MAX_PATCH_LENGTH = 4000\nconst MAX_CONTENT_LENGTH = 20000\nconst MAX_MODEL_TREE_ENTRIES = 200\nconst MAX_MODEL_DIFF_FILES = 80\n\nfunction truncateText(text: string, maxLength: number): string {\n  if (text.length <= maxLength) return text\n  return `${text.slice(0, maxLength)}\\n\\n[truncated: ${text.length - maxLength} more characters]`\n}\n\nfunction truncatePatchFields<T extends { patch?: string }>(files: T[]): T[] {\n  return files.map(file => ({\n    ...file,\n    patch: file.patch ? truncateText(file.patch, MAX_PATCH_LENGTH) : file.patch,\n  }))\n}\n\nfunction capDiffFiles<T extends { patch?: string }>(files: T[] | undefined) {\n  if (!files) return { files, filesOmitted: 0 }\n  const truncated = truncatePatchFields(files)\n  if (truncated.length <= MAX_MODEL_DIFF_FILES) return { files: truncated, filesOmitted: 0 }\n  return {\n    files: truncated.slice(0, MAX_MODEL_DIFF_FILES),\n    filesOmitted: truncated.length - MAX_MODEL_DIFF_FILES,\n  }\n}\n\ntype ToModelOutputOptions = {\n  toolCallId: string\n  input: unknown\n  output: unknown\n}\n\ntype ListPullRequestFile = {\n  filename: string\n  status: string\n  additions: number\n  deletions: number\n  changes: number\n  patch?: string\n}\n\ntype ListPullRequestFilesOutput = {\n  items: ListPullRequestFile[]\n  hasMore: boolean\n  page: number\n  perPage: number\n  nextPage?: number\n}\n\ntype GetCommitOutput = {\n  sha: string\n  message: string\n  author?: string\n  authorLogin?: string\n  date?: string\n  url: string\n  stats: { additions: number, deletions: number, total: number } | null\n  files?: Array<{\n    filename: string\n    status: string\n    additions: number\n    deletions: number\n    patch?: string\n  }>\n}\n\ntype GetFileContentOutput =\n  | { type: 'directory', entries: Array<{ name: string, type: string, path: string }> }\n  | { type: string, path: string }\n  | {\n      type: 'file'\n      path: string\n      sha: string\n      size: number\n      content: string\n      totalLines?: number\n      startLine?: number\n      endLine?: number\n      truncated?: boolean\n    }\n\nexport function listPullRequestFilesToModelOutput({ output }: ToModelOutputOptions) {\n  const result = output as ListPullRequestFilesOutput\n  return {\n    type: 'json' as const,\n    value: {\n      ...result,\n      items: truncatePatchFields(result.items),\n    },\n  }\n}\n\nexport function getCommitToModelOutput({ output }: ToModelOutputOptions) {\n  const commit = output as GetCommitOutput\n  const { files, filesOmitted } = capDiffFiles(commit.files)\n  return {\n    type: 'json' as const,\n    value: {\n      ...commit,\n      files,\n      ...filesOmitted > 0 ? { filesOmitted } : {},\n    },\n  }\n}\n\ntype CompareCommitsOutput = {\n  status: string\n  aheadBy: number\n  behindBy: number\n  totalCommits: number\n  url: string\n  commits: Array<{ sha: string, message: string, author?: string, authorLogin?: string }>\n  files?: Array<{\n    filename: string\n    status: string\n    additions: number\n    deletions: number\n    patch?: string\n  }>\n}\n\nexport function compareCommitsToModelOutput({ output }: ToModelOutputOptions) {\n  const comparison = output as CompareCommitsOutput\n  const { files, filesOmitted } = capDiffFiles(comparison.files)\n  return {\n    type: 'json' as const,\n    value: {\n      ...comparison,\n      files,\n      ...filesOmitted > 0 ? { filesOmitted } : {},\n    },\n  }\n}\n\nexport function getFileContentToModelOutput({ output }: ToModelOutputOptions) {\n  const result = output as GetFileContentOutput\n  if (result.type === 'directory' && 'entries' in result && result.entries.length > MAX_MODEL_TREE_ENTRIES) {\n    return {\n      type: 'json' as const,\n      value: {\n        ...result,\n        truncated: true,\n        entries: result.entries.slice(0, MAX_MODEL_TREE_ENTRIES),\n        entriesOmitted: result.entries.length - MAX_MODEL_TREE_ENTRIES,\n      },\n    }\n  }\n  if ('content' in result && result.content.length > MAX_CONTENT_LENGTH) {\n    return {\n      type: 'json' as const,\n      value: {\n        ...result,\n        content: truncateText(result.content, MAX_CONTENT_LENGTH),\n      },\n    }\n  }\n  return { type: 'json' as const, value: result }\n}\n\ntype GetRepositoryTreeOutput = {\n  sha: string\n  truncated: boolean\n  path?: string\n  entries: Array<{ path?: string, type?: string, size?: number, sha?: string }>\n}\n\nexport function getRepositoryTreeToModelOutput({ output }: ToModelOutputOptions) {\n  const result = output as GetRepositoryTreeOutput\n  if (result.entries.length <= MAX_MODEL_TREE_ENTRIES) {\n    return { type: 'json' as const, value: result }\n  }\n  return {\n    type: 'json' as const,\n    value: {\n      ...result,\n      truncated: true,\n      entries: result.entries.slice(0, MAX_MODEL_TREE_ENTRIES),\n      entriesOmitted: result.entries.length - MAX_MODEL_TREE_ENTRIES,\n    },\n  }\n}\n\ntype GetPullRequestContextOutput = {\n  pullRequest: {\n    number: number\n    title: string\n    body: string | null\n    state: string\n    url: string\n    author?: string\n    branch: string\n    headSha: string\n    base: string\n    draft?: boolean\n    merged: boolean\n    mergeable: boolean | null\n    additions: number\n    deletions: number\n    changedFiles: number\n    createdAt: string\n    updatedAt: string\n    mergedAt: string | null\n  }\n  files?: ListPullRequestFile[]\n  filesHasMore?: boolean\n  reviews?: Array<{\n    id: number\n    state: string\n    body: string\n    author?: string\n    url: string\n    submittedAt?: string | null\n  }>\n  checks?: {\n    checkRuns: {\n      totalCount: number\n      checkRuns: Array<{\n        id: number\n        name: string\n        status: string\n        conclusion: string | null\n        url: string | null\n        startedAt: string | null\n        completedAt: string | null\n      }>\n    }\n    combinedStatus: {\n      state: string\n      totalCount: number\n      statuses: Array<{\n        context: string\n        state: string\n        description: string | null\n        url: string | null\n      }>\n    }\n  }\n}\n\nexport function getPullRequestContextToModelOutput({ output }: ToModelOutputOptions) {\n  const result = output as GetPullRequestContextOutput\n  const { files, filesOmitted } = capDiffFiles(result.files)\n  return {\n    type: 'json' as const,\n    value: {\n      ...result,\n      files,\n      ...filesOmitted > 0 ? { filesOmitted } : {},\n    },\n  }\n}\n","import * as ai from 'ai'\nimport type { Experimental_EvaluationModel, JSONValue, ModelMessage } from 'ai'\nimport { GITHUB_TOOL_CATALOG } from './catalog'\nimport { githubToolsErrors } from './errors'\nimport { PRESET_TOOLS } from './presets'\nimport { DEFAULT_EVALUATION_MODEL, RISK_QUESTION, INTENT_QUESTION, PRESET_PURPOSES, rankPresets, requiresApproval, type RoutablePreset } from './evaluation-policy'\nexport { DEFAULT_EVALUATION_MODEL, AUTO_APPROVAL_TOOLS } from './evaluation-policy'\nimport type { GithubWriteToolName } from './write-tools'\n\n/**\n * Tuning for `requireApproval: 'auto'` and `preset: 'auto'`. Every field is optional.\n */\nexport type GithubEvaluationOptions = {\n  /**\n   * Evaluation model for AI SDK `experimental_evaluate`: an AI Gateway ID or a model instance.\n   * @default 'typesafe-ai/jev'\n   */\n  model?: Experimental_EvaluationModel\n  /**\n   * Highest risk an `'auto'` tool call can have and still run without approval, on a 0–2 scale\n   * (0 negligible, 1 visible but reversible, 2 hard to undo).\n   * @default 1\n   */\n  maxRisk?: number\n  /**\n   * Lowest probability that the latest user message asked for the call, for it to run without approval.\n   * @default 0.6\n   */\n  minIntent?: number\n  /**\n   * Lowest probability for `preset: 'auto'` to select a preset. When no preset reaches it,\n   * the single most likely preset is used.\n   * @default 0.7\n   */\n  minPresetProbability?: number\n  /**\n   * Most presets `preset: 'auto'` combines in one call, keeping the most likely ones.\n   * @default 2\n   */\n  maxPresets?: number\n}\n\nfunction loadEvaluate() {\n  if (!('experimental_evaluate' in ai)) throw githubToolsErrors.EVALUATION_UNAVAILABLE()\n  return ai.experimental_evaluate\n}\n\nfunction warnEvaluationFailed(model: Experimental_EvaluationModel, fallback: string, error: unknown) {\n  const cause = error instanceof Error ? error : new Error(String(error))\n  console.warn('[github-tools]', githubToolsErrors.EVALUATION_FAILED({\n    model: typeof model === 'string' ? model : model.modelId,\n    fallback,\n    detail: cause.message,\n    cause,\n  }))\n}\n\n/**\n * Whether an `'auto'` write tool call needs human approval: `false` only when the model\n * rates it low-risk and asked for by the latest user message. When the evaluation call\n * fails, logs `EVALUATION_FAILED` and asks for approval.\n */\nexport async function needsAutoApproval(\n  toolName: GithubWriteToolName,\n  input: unknown,\n  messages: readonly ModelMessage[],\n  { model = DEFAULT_EVALUATION_MODEL, maxRisk = 1, minIntent = 0.6 }: GithubEvaluationOptions = {},\n): Promise<boolean> {\n  const evaluate = loadEvaluate()\n  const result = await evaluate({\n    model,\n    state: {\n      request: latestUserText(messages),\n      toolCall: {\n        tool: toolName,\n        description: GITHUB_TOOL_CATALOG[toolName].description,\n        input: input as JSONValue,\n      },\n    },\n    questions: {\n      risk: { type: 'score', ...RISK_QUESTION },\n      intent: { type: 'boolean', ...INTENT_QUESTION },\n    },\n  }).catch((error: unknown) => {\n    warnEvaluationFailed(model, `asking approval for ${toolName}`, error)\n    return undefined\n  })\n  if (!result) return true\n  const { answers } = result\n  return requiresApproval(answers.risk.score, answers.intent.probability, { maxRisk, minIntent })\n}\n\n/**\n * Presets the latest user message needs, from one evaluation call with an independent\n * yes/no question per preset. Returns up to `maxPresets` presets at or above\n * `minPresetProbability`, most likely first; when none qualifies, the single most likely preset.\n * `repo-explorer` is only returned when no other preset qualifies.\n * When the evaluation call fails, logs `EVALUATION_FAILED` and returns read-only `repo-explorer`.\n * Never returns `maintainer`, so the full catalog is never exposed.\n */\nexport async function selectPresets(\n  messages: readonly ModelMessage[],\n  { model = DEFAULT_EVALUATION_MODEL, minPresetProbability = 0.7, maxPresets = 2 }: GithubEvaluationOptions = {},\n): Promise<RoutablePreset[]> {\n  const evaluate = loadEvaluate()\n  const presets = Object.keys(PRESET_PURPOSES) as RoutablePreset[]\n  const result = await evaluate({\n    model,\n    state: { request: latestUserText(messages) },\n    questions: Object.fromEntries(presets.map(preset => [preset, {\n      type: 'boolean' as const,\n      instructions: 'Does handling this GitHub request need the tools of this preset?',\n      criteria: { true: { purpose: PRESET_PURPOSES[preset], tools: [...PRESET_TOOLS[preset]] } },\n    }])),\n  }).catch((error: unknown) => {\n    warnEvaluationFailed(model, 'using the repo-explorer preset', error)\n    return undefined\n  })\n  if (!result) return ['repo-explorer']\n  const { answers } = result\n  return rankPresets(Object.fromEntries(presets.map(preset => [preset, answers[preset].probability])) as Record<RoutablePreset, number>, { minPresetProbability, maxPresets })\n}\n\nexport function latestUserText(messages: readonly ModelMessage[]): string {\n  const message = messages.findLast(m => m.role === 'user')\n  if (!message) return ''\n  if (typeof message.content === 'string') return message.content\n  return message.content.flatMap(part => part.type === 'text' ? [part.text] : []).join('\\n')\n}\n"],"mappings":";;;AAAA,MAAM,mBAAmB;AACzB,MAAM,qBAAqB;AAC3B,MAAM,yBAAyB;AAC/B,MAAM,uBAAuB;AAE7B,SAAS,aAAa,MAAc,WAA2B;CAC7D,IAAI,KAAK,UAAU,WAAW,OAAO;CACrC,OAAO,GAAG,KAAK,MAAM,GAAG,SAAS,EAAE,kBAAkB,KAAK,SAAS,UAAU;AAC/E;AAEA,SAAS,oBAAkD,OAAiB;CAC1E,OAAO,MAAM,KAAI,UAAS;EACxB,GAAG;EACH,OAAO,KAAK,QAAQ,aAAa,KAAK,OAAO,gBAAgB,IAAI,KAAK;CACxE,EAAE;AACJ;AAEA,SAAS,aAA2C,OAAwB;CAC1E,IAAI,CAAC,OAAO,OAAO;EAAE;EAAO,cAAc;CAAE;CAC5C,MAAM,YAAY,oBAAoB,KAAK;CAC3C,IAAI,UAAU,UAAU,sBAAsB,OAAO;EAAE,OAAO;EAAW,cAAc;CAAE;CACzF,OAAO;EACL,OAAO,UAAU,MAAM,GAAG,oBAAoB;EAC9C,cAAc,UAAU,SAAS;CACnC;AACF;AAyDA,SAAgB,kCAAkC,EAAE,UAAgC;CAClF,MAAM,SAAS;CACf,OAAO;EACL,MAAM;EACN,OAAO;GACL,GAAG;GACH,OAAO,oBAAoB,OAAO,KAAK;EACzC;CACF;AACF;AAEA,SAAgB,uBAAuB,EAAE,UAAgC;CACvE,MAAM,SAAS;CACf,MAAM,EAAE,OAAO,iBAAiB,aAAa,OAAO,KAAK;CACzD,OAAO;EACL,MAAM;EACN,OAAO;GACL,GAAG;GACH;GACA,GAAG,eAAe,IAAI,EAAE,aAAa,IAAI,CAAC;EAC5C;CACF;AACF;AAkBA,SAAgB,4BAA4B,EAAE,UAAgC;CAC5E,MAAM,aAAa;CACnB,MAAM,EAAE,OAAO,iBAAiB,aAAa,WAAW,KAAK;CAC7D,OAAO;EACL,MAAM;EACN,OAAO;GACL,GAAG;GACH;GACA,GAAG,eAAe,IAAI,EAAE,aAAa,IAAI,CAAC;EAC5C;CACF;AACF;AAEA,SAAgB,4BAA4B,EAAE,UAAgC;CAC5E,MAAM,SAAS;CACf,IAAI,OAAO,SAAS,eAAe,aAAa,UAAU,OAAO,QAAQ,SAAS,wBAChF,OAAO;EACL,MAAM;EACN,OAAO;GACL,GAAG;GACH,WAAW;GACX,SAAS,OAAO,QAAQ,MAAM,GAAG,sBAAsB;GACvD,gBAAgB,OAAO,QAAQ,SAAS;EAC1C;CACF;CAEF,IAAI,aAAa,UAAU,OAAO,QAAQ,SAAS,oBACjD,OAAO;EACL,MAAM;EACN,OAAO;GACL,GAAG;GACH,SAAS,aAAa,OAAO,SAAS,kBAAkB;EAC1D;CACF;CAEF,OAAO;EAAE,MAAM;EAAiB,OAAO;CAAO;AAChD;AASA,SAAgB,+BAA+B,EAAE,UAAgC;CAC/E,MAAM,SAAS;CACf,IAAI,OAAO,QAAQ,UAAU,wBAC3B,OAAO;EAAE,MAAM;EAAiB,OAAO;CAAO;CAEhD,OAAO;EACL,MAAM;EACN,OAAO;GACL,GAAG;GACH,WAAW;GACX,SAAS,OAAO,QAAQ,MAAM,GAAG,sBAAsB;GACvD,gBAAgB,OAAO,QAAQ,SAAS;EAC1C;CACF;AACF;AA2DA,SAAgB,mCAAmC,EAAE,UAAgC;CACnF,MAAM,SAAS;CACf,MAAM,EAAE,OAAO,iBAAiB,aAAa,OAAO,KAAK;CACzD,OAAO;EACL,MAAM;EACN,OAAO;GACL,GAAG;GACH;GACA,GAAG,eAAe,IAAI,EAAE,aAAa,IAAI,CAAC;EAC5C;CACF;AACF;;;ACjNA,SAAS,eAAe;CACtB,IAAI,EAAE,2BAA2B,KAAK,MAAM,kBAAkB,uBAAuB;CACrF,OAAO,GAAG;AACZ;AAEA,SAAS,qBAAqB,OAAqC,UAAkB,OAAgB;CACnG,MAAM,QAAQ,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;CACtE,QAAQ,KAAK,kBAAkB,kBAAkB,kBAAkB;EACjE,OAAO,OAAO,UAAU,WAAW,QAAQ,MAAM;EACjD;EACA,QAAQ,MAAM;EACd;CACF,CAAC,CAAC;AACJ;;;;;;AAOA,eAAsB,kBACpB,UACA,OACA,UACA,EAAE,QAAQ,0BAA0B,UAAU,GAAG,YAAY,OAAiC,CAAC,GAC7E;CAElB,MAAM,SAAS,MADE,aACW,CAAC,CAAC;EAC5B;EACA,OAAO;GACL,SAAS,eAAe,QAAQ;GAChC,UAAU;IACR,MAAM;IACN,aAAa,oBAAoB,SAAS,CAAC;IACpC;GACT;EACF;EACA,WAAW;GACT,MAAM;IAAE,MAAM;IAAS,GAAG;GAAc;GACxC,QAAQ;IAAE,MAAM;IAAW,GAAG;GAAgB;EAChD;CACF,CAAC,CAAC,CAAC,OAAO,UAAmB;EAC3B,qBAAqB,OAAO,uBAAuB,YAAY,KAAK;CAEtE,CAAC;CACD,IAAI,CAAC,QAAQ,OAAO;CACpB,MAAM,EAAE,YAAY;CACpB,OAAO,iBAAiB,QAAQ,KAAK,OAAO,QAAQ,OAAO,aAAa;EAAE;EAAS;CAAU,CAAC;AAChG;;;;;;;;;AAUA,eAAsB,cACpB,UACA,EAAE,QAAQ,0BAA0B,uBAAuB,IAAK,aAAa,MAA+B,CAAC,GAClF;CAC3B,MAAM,WAAW,aAAa;CAC9B,MAAM,UAAU,OAAO,KAAK,eAAe;CAC3C,MAAM,SAAS,MAAM,SAAS;EAC5B;EACA,OAAO,EAAE,SAAS,eAAe,QAAQ,EAAE;EAC3C,WAAW,OAAO,YAAY,QAAQ,KAAI,WAAU,CAAC,QAAQ;GAC3D,MAAM;GACN,cAAc;GACd,UAAU,EAAE,MAAM;IAAE,SAAS,gBAAgB;IAAS,OAAO,CAAC,GAAG,aAAa,OAAO;GAAE,EAAE;EAC3F,CAAC,CAAC,CAAC;CACL,CAAC,CAAC,CAAC,OAAO,UAAmB;EAC3B,qBAAqB,OAAO,kCAAkC,KAAK;CAErE,CAAC;CACD,IAAI,CAAC,QAAQ,OAAO,CAAC,eAAe;CACpC,MAAM,EAAE,YAAY;CACpB,OAAO,YAAY,OAAO,YAAY,QAAQ,KAAI,WAAU,CAAC,QAAQ,QAAQ,OAAO,CAAC,WAAW,CAAC,CAAC,GAAqC;EAAE;EAAsB;CAAW,CAAC;AAC7K;AAEA,SAAgB,eAAe,UAA2C;CACxE,MAAM,UAAU,SAAS,UAAS,MAAK,EAAE,SAAS,MAAM;CACxD,IAAI,CAAC,SAAS,OAAO;CACrB,IAAI,OAAO,QAAQ,YAAY,UAAU,OAAO,QAAQ;CACxD,OAAO,QAAQ,QAAQ,SAAQ,SAAQ,KAAK,SAAS,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI;AAC3F"}