{"version":3,"file":"prompt-validate.mjs","names":[],"sources":["../../../../../../../ai/src/prompt/prompt-validate.ts"],"sourcesContent":["import type { AgentContract } from \"../contracts/agent/agent.contract\";\nimport type { ModelContract } from \"../contracts/model.contract\";\nimport { judge } from \"../eval/judge-scorer\";\nimport type { PromptValidationNote, PromptValidationReport } from \"./prompt.type\";\n\n/**\n * Placeholder matcher — kept in lock-step with the matcher\n * `renderPlaceholders` uses (`src/system-prompt/render-placeholders.ts`) so the\n * lint sees the same `{{key}}` / `{{a.b}}` / `{{key|default}}` set the renderer\n * substitutes. Global so every occurrence is collected.\n */\nconst PLACEHOLDER_PATTERN = /\\{\\{\\s*([^{}]+?)\\s*\\}\\}/g;\n\n/** Lower bound below which a prompt is suspiciously terse. */\nconst MIN_REASONABLE_LENGTH = 12;\n\n/** Upper bound above which a prompt is likely bloated / unfocused. */\nconst MAX_REASONABLE_LENGTH = 8000;\n\n/**\n * Severity rank for most-severe-first ordering. Higher sorts earlier.\n */\nconst SEVERITY_RANK: Record = {\n error: 2,\n warn: 1,\n info: 0,\n};\n\n/**\n * The fixed rubric the LLM-as-judge grades a prompt body against. Surfaced\n * here (not inline) so the static-lint pass and the judge pass document the\n * same quality dimensions.\n */\nexport const PROMPT_JUDGE_RUBRIC = [\n \"Grade this SYSTEM PROMPT on a 0..1 scale for overall quality:\",\n \"- Clarity: is the intent unambiguous and easy to follow?\",\n \"- Role definition: does it clearly state who/what the assistant is?\",\n \"- Output-format specificity: does it say how the answer should be shaped?\",\n \"- No conflicting instructions: are any directives contradictory?\",\n \"Score 1.0 only when all four hold; deduct for each weakness and explain why.\",\n].join(\"\\n\");\n\n/**\n * Heuristic role-line detector — a prompt that never says \"you are …\" /\n * \"act as …\" / \"your role is …\" typically lacks a persona. Case-insensitive.\n */\nconst ROLE_HINT_PATTERN = /\\b(you are|act as|your role is|you're a|you will act)\\b/i;\n\n/**\n * Run the cheap, model-free static lint over a prompt body. Flags:\n * - length out of the reasonable band (too terse / too bloated),\n * - any `{{placeholder}}` that survives (undeclared / unresolved at lint time),\n * - a missing role line.\n *\n * Pure and synchronous — used standalone (no judge model) and merged with the\n * judge findings when a model is available.\n *\n * @param text - The prompt body to lint.\n */\nexport function staticLint(text: string): PromptValidationNote[] {\n const notes: PromptValidationNote[] = [];\n const trimmed = text.trim();\n\n if (trimmed.length < MIN_REASONABLE_LENGTH) {\n notes.push({\n severity: \"warn\",\n message: `Prompt is very short (${trimmed.length} chars) — it may be too vague to steer the model.`,\n suggestion: \"Add an explicit role and at least one concrete instruction.\",\n });\n }\n\n if (trimmed.length > MAX_REASONABLE_LENGTH) {\n notes.push({\n severity: \"warn\",\n message: `Prompt is very long (${trimmed.length} chars) — long prompts dilute focus and inflate cost.`,\n suggestion: \"Split into a tighter persona plus a few focused instructions.\",\n });\n }\n\n const placeholders = collectPlaceholders(text);\n\n for (const placeholder of placeholders) {\n notes.push({\n severity: \"info\",\n message: `Unresolved placeholder \"{{${placeholder}}}\" — confirm it is supplied at resolve time or give it a default (\"{{${placeholder}|...}}\").`,\n });\n }\n\n if (!ROLE_HINT_PATTERN.test(trimmed)) {\n notes.push({\n severity: \"warn\",\n message: \"No role line found — the prompt never states who the assistant is.\",\n suggestion: 'Open with a role, e.g. \"You are a senior support engineer for …\".',\n });\n }\n\n return notes;\n}\n\n/**\n * Collect every distinct placeholder PATH (the part before any `|default`)\n * from a template, in first-seen order. Matches `renderPlaceholders`' own\n * parsing so the lint never disagrees with the renderer.\n */\nfunction collectPlaceholders(template: string): string[] {\n const found: string[] = [];\n const seen = new Set();\n\n for (const match of template.matchAll(PLACEHOLDER_PATTERN)) {\n const path = match[1].split(\"|\")[0].trim();\n\n if (path.length > 0 && !seen.has(path)) {\n seen.add(path);\n found.push(path);\n }\n }\n\n return found;\n}\n\n/**\n * Stable, most-severe-first ordering: `error` before `warn` before `info`,\n * preserving original order within a severity. Returns a fresh array.\n */\nexport function sortNotesBySeverity(\n notes: PromptValidationNote[],\n): PromptValidationNote[] {\n return notes\n .map((note, index) => ({ note, index }))\n .sort((a, b) => {\n const rankDiff = SEVERITY_RANK[b.note.severity] - SEVERITY_RANK[a.note.severity];\n\n return rankDiff !== 0 ? rankDiff : a.index - b.index;\n })\n .map(entry => entry.note);\n}\n\n/**\n * Score the static-lint findings alone, on a `0..1` scale. Starts at `1.0`\n * and deducts per finding by severity, clamped at `0`. Used as the report\n * score when no judge model is available.\n */\nexport function staticScore(notes: PromptValidationNote[]): number {\n let score = 1;\n\n for (const note of notes) {\n if (note.severity === \"error\") {\n score -= 0.4;\n } else if (note.severity === \"warn\") {\n score -= 0.2;\n } else {\n score -= 0.05;\n }\n }\n\n return Math.max(0, Number(score.toFixed(4)));\n}\n\n/**\n * Run the LLM-as-judge pass over `text` using a judge agent built from\n * `model`, REUSING the eval `judge` scorer so there is no second judging\n * path. Returns the judge `score` (`0..1`) and a single derived note carrying\n * its reason (when present). The judge prompt is the prompt-quality rubric;\n * the \"answer to grade\" is the prompt body itself.\n *\n * @param text - The prompt body under evaluation.\n * @param model - The model that powers the judge agent.\n * @param buildJudgeAgent - Factory that wraps a model into a name-bearing judge agent.\n */\nexport async function judgePrompt(\n text: string,\n model: ModelContract,\n buildJudgeAgent: (model: ModelContract) => AgentContract,\n): Promise<{ score: number; notes: PromptValidationNote[] }> {\n const judgeAgent = buildJudgeAgent(model);\n const scorer = judge({ agent: judgeAgent, rubric: PROMPT_JUDGE_RUBRIC });\n\n const score = await scorer({\n // The judge scorer only reads `case.input` / `case.expected` / `text` /\n // `output` from the context. We feed the rubric question via `input` and\n // the prompt body as the answer to grade via `text`.\n case: { name: \"prompt-quality\", input: \"Grade the system prompt below.\" },\n text,\n // `result` is unused by the judge scorer's prompt builder; a minimal\n // stand-in keeps the structural contract satisfied without a real run.\n result: { text } as never,\n output: undefined,\n });\n\n const notes: PromptValidationNote[] = [];\n\n if (score.reason) {\n notes.push({\n severity: score.passed ? \"info\" : \"warn\",\n message: `LLM-as-judge: ${score.reason}`,\n });\n }\n\n return { score: score.score, notes };\n}\n\n/**\n * Assemble the final {@link PromptValidationReport} from the static-lint\n * findings and (optionally) the judge findings. Notes are merged and sorted\n * most-severe-first. The score is the static score alone when no judge ran,\n * else the mean of the static score and the judge score.\n */\nexport function buildValidationReport(\n staticNotes: PromptValidationNote[],\n judgeResult?: { score: number; notes: PromptValidationNote[] },\n): PromptValidationReport {\n const allNotes = judgeResult\n ? [...staticNotes, ...judgeResult.notes]\n : staticNotes;\n\n const lintScore = staticScore(staticNotes);\n\n const score = judgeResult\n ? Number(((lintScore + judgeResult.score) / 2).toFixed(4))\n : lintScore;\n\n return {\n score,\n notes: sortNotesBySeverity(allNotes),\n };\n}\n"],"mappings":";;;;;;;;;AAWA,MAAM,sBAAsB;;AAG5B,MAAM,wBAAwB;;AAG9B,MAAM,wBAAwB;;;;AAK9B,MAAM,gBAAkE;CACtE,OAAO;CACP,MAAM;CACN,MAAM;AACR;;;;;;AAOA,MAAa,sBAAsB;CACjC;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;;;;AAMX,MAAM,oBAAoB;;;;;;;;;;;;AAa1B,SAAgB,WAAW,MAAsC;CAC/D,MAAM,QAAgC,CAAC;CACvC,MAAM,UAAU,KAAK,KAAK;CAE1B,IAAI,QAAQ,SAAS,uBACnB,MAAM,KAAK;EACT,UAAU;EACV,SAAS,yBAAyB,QAAQ,OAAO;EACjD,YAAY;CACd,CAAC;CAGH,IAAI,QAAQ,SAAS,uBACnB,MAAM,KAAK;EACT,UAAU;EACV,SAAS,wBAAwB,QAAQ,OAAO;EAChD,YAAY;CACd,CAAC;CAGH,MAAM,eAAe,oBAAoB,IAAI;CAE7C,KAAK,MAAM,eAAe,cACxB,MAAM,KAAK;EACT,UAAU;EACV,SAAS,6BAA6B,YAAY,wEAAwE,YAAY;CACxI,CAAC;CAGH,IAAI,CAAC,kBAAkB,KAAK,OAAO,GACjC,MAAM,KAAK;EACT,UAAU;EACV,SAAS;EACT,YAAY;CACd,CAAC;CAGH,OAAO;AACT;;;;;;AAOA,SAAS,oBAAoB,UAA4B;CACvD,MAAM,QAAkB,CAAC;CACzB,MAAM,uBAAO,IAAI,IAAY;CAE7B,KAAK,MAAM,SAAS,SAAS,SAAS,mBAAmB,GAAG;EAC1D,MAAM,OAAO,MAAM,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK;EAEzC,IAAI,KAAK,SAAS,KAAK,CAAC,KAAK,IAAI,IAAI,GAAG;GACtC,KAAK,IAAI,IAAI;GACb,MAAM,KAAK,IAAI;EACjB;CACF;CAEA,OAAO;AACT;;;;;AAMA,SAAgB,oBACd,OACwB;CACxB,OAAO,MACJ,KAAK,MAAM,WAAW;EAAE;EAAM;CAAM,EAAE,CAAC,CACvC,MAAM,GAAG,MAAM;EACd,MAAM,WAAW,cAAc,EAAE,KAAK,YAAY,cAAc,EAAE,KAAK;EAEvE,OAAO,aAAa,IAAI,WAAW,EAAE,QAAQ,EAAE;CACjD,CAAC,CAAC,CACD,KAAI,UAAS,MAAM,IAAI;AAC5B;;;;;;AAOA,SAAgB,YAAY,OAAuC;CACjE,IAAI,QAAQ;CAEZ,KAAK,MAAM,QAAQ,OACjB,IAAI,KAAK,aAAa,SACpB,SAAS;MACJ,IAAI,KAAK,aAAa,QAC3B,SAAS;MAET,SAAS;CAIb,OAAO,KAAK,IAAI,GAAG,OAAO,MAAM,QAAQ,CAAC,CAAC,CAAC;AAC7C;;;;;;;;;;;;AAaA,eAAsB,YACpB,MACA,OACA,iBAC2D;CAI3D,MAAM,QAAQ,MAFC,MAAM;EAAE,OADJ,gBAAgB,KACI;EAAG,QAAQ;CAAoB,CAE7C,CAAC,CAAC;EAIzB,MAAM;GAAE,MAAM;GAAkB,OAAO;EAAiC;EACxE;EAGA,QAAQ,EAAE,KAAK;EACf,QAAQ;CACV,CAAC;CAED,MAAM,QAAgC,CAAC;CAEvC,IAAI,MAAM,QACR,MAAM,KAAK;EACT,UAAU,MAAM,SAAS,SAAS;EAClC,SAAS,iBAAiB,MAAM;CAClC,CAAC;CAGH,OAAO;EAAE,OAAO,MAAM;EAAO;CAAM;AACrC;;;;;;;AAQA,SAAgB,sBACd,aACA,aACwB;CACxB,MAAM,WAAW,cACb,CAAC,GAAG,aAAa,GAAG,YAAY,KAAK,IACrC;CAEJ,MAAM,YAAY,YAAY,WAAW;CAMzC,OAAO;EACL,OALY,cACV,SAAS,YAAY,YAAY,SAAS,EAAC,CAAE,QAAQ,CAAC,CAAC,IACvD;EAIF,OAAO,oBAAoB,QAAQ;CACrC;AACF"}