{"version":3,"file":"trigger-eval.d.ts","sourceRoot":"","sources":["../../../../src/core/extensions/plugins/trigger-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAEpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEtD,4FAA4F;AAC5F,eAAO,MAAM,iBAAiB,QAAqC,CAAC;AAEpE,MAAM,WAAW,WAAW;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,mFAAmF;IACnF,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,GAAG,EAAE,OAAO,CAAC;CACb;AAED,MAAM,WAAW,mBAAmB;IACnC,UAAU,EAAE,gBAAgB,EAAE,CAAC;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,oFAAoF;AACpF,MAAM,MAAM,mBAAmB,GAAG,MAAM,GAAG,IAAI,CAAC;AAEhD;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,mBAAmB,KAAK,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;AAE5F,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,iBAAiB;IACjC,mEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,oFAAoF;IACpF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,iBAAiB,EAAE,CAAC;CAC7B;AAED,MAAM,MAAM,kBAAkB,GAAG;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,iBAAiB,CAAA;CAAE,CAAC;AAOtH,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,GAAG,SAAS,CAkBxE;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,gBAAgB,GAAG,gBAAgB,EAAE,CAI1E;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,aAAa,EAAE,GAAG,gBAAgB,EAAE,CAIxG;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,SAAS,gBAAgB,EAAE,EAAE,KAAK,EAAE,SAAS,WAAW,EAAE,GAAG,MAAM,CAOhH;AAMD;;;;;;;GAOG;AACH,wBAAsB,cAAc,CACnC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,SAAS,gBAAgB,EAAE,EACvC,KAAK,EAAE,SAAS,WAAW,EAAE,GAAG,SAAS,EACzC,KAAK,EAAE,YAAY,GAAG,SAAS,GAC7B,OAAO,CAAC,kBAAkB,CAAC,CA4D7B;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAsC1E","sourcesContent":["/**\n * G4 — trigger eval: does an authored skill fire on the situations it is for,\n * and stay quiet otherwise?\n *\n * G1–G3 ask whether a plugin is well-formed, safe, and runnable. None of them\n * ask whether it is any *good*, and for a passive capability the whole of \"good\"\n * is the `description`: it is the only thing the model sees when deciding\n * whether to reach for the skill. A skill that parses, validates, and never\n * triggers is dead weight that costs context on every request.\n *\n * ## The circularity trap\n *\n * The tempting design is to generate test prompts from the description and then\n * check that the description matches them. That measures self-consistency and\n * always passes. So gold prompts are **supplied**, not generated (`eval/triggers.json`\n * in the plugin, or authored alongside it), and the negatives largely come for\n * free: a prompt whose right answer is a *different* capability is a negative for\n * this one. Discriminating against real siblings is the question that matters —\n * \"does it fire\" is easy, \"does it fire instead of the wrong thing\" is not.\n *\n * ## Reuse of the search eval harness\n *\n * §4.2 asks G4 to reuse `core/search/eval-harness.ts`. Its three *rules* are\n * adopted verbatim below and are the reason this module looks the way it does:\n * pin what you measured, record it machine-readably, and never let a degraded\n * run read like a real one. Its *machinery* could not be called: that harness\n * pins a corpus by checking out a git worktree and stamps provenance with\n * embsearch daemon state, and G4's corpus is a handful of description strings\n * with no repo and no embedder. Calling it would have meant a git worktree per\n * skill eval. The pinning here is therefore a content hash of exactly what was\n * judged — same guarantee, appropriate mechanism.\n *\n * See docs/plugin-system-architecture.md §4.2.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readFileSync } from \"node:fs\";\nimport * as path from \"node:path\";\nimport type { CapabilityDoc } from \"../../capabilities/registry.js\";\nimport { loadSkillsFromDir } from \"../../skills.js\";\nimport type { GateFinding } from \"./gates.js\";\nimport type { NormalizedPlugin } from \"./manifest.js\";\n\n/** Where a plugin declares the prompts its capabilities should (and should not) fire on. */\nexport const TRIGGER_GOLD_FILE = path.join(\"eval\", \"triggers.json\");\n\nexport interface TriggerCase {\n\tprompt: string;\n\t/** Capability name this prompt should select, or null when nothing should fire. */\n\texpect: string | null;\n}\n\n/** One capability offered to the judge as a candidate. */\nexport interface TriggerCandidate {\n\tname: string;\n\tdescription: string;\n\t/** False for the distractors drawn from other plugins. */\n\town: boolean;\n}\n\nexport interface TriggerJudgeRequest {\n\tcandidates: TriggerCandidate[];\n\tprompts: string[];\n}\n\n/** The judge's answer per prompt: the selected candidate name, or null for none. */\nexport type TriggerJudgeVerdict = string | null;\n\n/**\n * The single model call, injected.\n *\n * Injected rather than imported so the scoring, the gold-set handling and the\n * gate mapping are testable without a model — and so a caller with no model\n * configured gets an honest `not-run` instead of a gate that quietly passes.\n */\nexport type TriggerJudge = (request: TriggerJudgeRequest) => Promise<TriggerJudgeVerdict[]>;\n\nexport interface TriggerCaseResult {\n\tprompt: string;\n\texpected: string | null;\n\tactual: string | null;\n\tcorrect: boolean;\n}\n\nexport interface TriggerEvalRecord {\n\t/** Content hash of the candidates and cases judged — the pin. */\n\tcorpusHash: string;\n\ttimestampMs: number;\n\tpluginId: string;\n\tcandidateCount: number;\n\tcaseCount: number;\n\t/** Fires when it should: TP / (TP + FN). Undefined when there were no positives. */\n\trecall?: number;\n\t/** Quiet when it should be: TN / (TN + FP). Undefined when there were no negatives. */\n\tspecificity?: number;\n\tresults: TriggerCaseResult[];\n}\n\nexport type TriggerEvalOutcome = { status: \"not-run\"; reason: string } | { status: \"ran\"; record: TriggerEvalRecord };\n\n/** Recall below this is an error: a skill that does not fire is not a capability. */\nconst MIN_RECALL = 0.8;\n/** Specificity below this is a warning: over-firing costs context but breaks nothing. */\nconst MIN_SPECIFICITY = 0.75;\n\n/** Read the plugin's declared gold set, or undefined when it has none. */\nexport function loadTriggerCases(root: string): TriggerCase[] | undefined {\n\tconst file = path.join(root, TRIGGER_GOLD_FILE);\n\tif (!existsSync(file)) return undefined;\n\ttry {\n\t\tconst raw = JSON.parse(readFileSync(file, \"utf8\")) as { cases?: unknown };\n\t\tif (!Array.isArray(raw.cases)) return undefined;\n\t\tconst cases = raw.cases.filter(\n\t\t\t(c): c is TriggerCase =>\n\t\t\t\t!!c &&\n\t\t\t\ttypeof c === \"object\" &&\n\t\t\t\ttypeof (c as TriggerCase).prompt === \"string\" &&\n\t\t\t\t(c as TriggerCase).prompt.trim().length > 0 &&\n\t\t\t\t((c as TriggerCase).expect === null || typeof (c as TriggerCase).expect === \"string\"),\n\t\t);\n\t\treturn cases.length > 0 ? cases : undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * The plugin's own passive capabilities, as judge candidates.\n *\n * Skills only. Commands are invoked by name (`/foo`) rather than selected from a\n * description, so \"does the description fire\" is not a question about them, and\n * scoring them would dilute the metric with cases that cannot fail.\n */\nexport function ownCandidates(plugin: NormalizedPlugin): TriggerCandidate[] {\n\tif (!plugin.skillsDir) return [];\n\tconst { skills } = loadSkillsFromDir({ dir: plugin.skillsDir, source: plugin.id });\n\treturn skills.map((s) => ({ name: s.name, description: s.description ?? \"\", own: true }));\n}\n\n/**\n * Sibling capabilities to put in front of the judge alongside the plugin's own.\n *\n * Without them the judge picks from a list of one and \"does it fire\" is nearly\n * free. The interesting failure is a description broad enough to win prompts\n * that belong to something else, and that only shows up against real competition.\n */\nexport function distractorCandidates(exclude: string, docs: readonly CapabilityDoc[]): TriggerCandidate[] {\n\treturn docs\n\t\t.filter((d) => (d.kind === \"skill\" || d.kind === \"mcp-tool\") && d.source !== exclude)\n\t\t.map((d) => ({ name: d.name, description: d.description, own: false }));\n}\n\n/**\n * Pin: a hash of exactly what was judged.\n *\n * Over candidates *and* cases, because both move the number. A rerun after a\n * description edit is measuring something else, and a record that cannot say so\n * is the failure mode the search harness was built to prevent.\n */\nexport function triggerCorpusHash(candidates: readonly TriggerCandidate[], cases: readonly TriggerCase[]): string {\n\tconst h = createHash(\"sha256\");\n\tfor (const c of [...candidates].sort((a, b) => a.name.localeCompare(b.name))) {\n\t\th.update(`${c.name}\u0000${c.description}\u0000`);\n\t}\n\tfor (const c of cases) h.update(`${c.prompt}\u0000${c.expect ?? \"\"}\u0000`);\n\treturn h.digest(\"hex\").slice(0, 16);\n}\n\nfunction ratio(correct: number, total: number): number | undefined {\n\treturn total > 0 ? correct / total : undefined;\n}\n\n/**\n * Score a plugin's passive capabilities against its gold set.\n *\n * Never throws and never fabricates: no judge, no gold set, or no candidates all\n * return `not-run` with the reason. A judge that answers the wrong number of\n * prompts is also `not-run` — a partial alignment between prompts and verdicts\n * would silently score the wrong pairs.\n */\nexport async function runTriggerEval(\n\tpluginId: string,\n\tcandidates: readonly TriggerCandidate[],\n\tcases: readonly TriggerCase[] | undefined,\n\tjudge: TriggerJudge | undefined,\n): Promise<TriggerEvalOutcome> {\n\tif (!judge) return { status: \"not-run\", reason: \"no trigger judge configured (G4 needs a model)\" };\n\tif (!cases || cases.length === 0) {\n\t\treturn {\n\t\t\tstatus: \"not-run\",\n\t\t\treason: `no gold set: add ${TRIGGER_GOLD_FILE} with the prompts this plugin should and should not fire on`,\n\t\t};\n\t}\n\tif (candidates.length === 0) return { status: \"not-run\", reason: \"plugin has no passive capabilities to evaluate\" };\n\n\tconst prompts = cases.map((c) => c.prompt);\n\tlet verdicts: TriggerJudgeVerdict[];\n\ttry {\n\t\tverdicts = await judge({ candidates: [...candidates], prompts });\n\t} catch (error) {\n\t\treturn { status: \"not-run\", reason: `judge failed: ${(error as Error).message}` };\n\t}\n\tif (verdicts.length !== prompts.length) {\n\t\treturn {\n\t\t\tstatus: \"not-run\",\n\t\t\treason: `judge returned ${verdicts.length} verdict(s) for ${prompts.length} prompt(s)`,\n\t\t};\n\t}\n\n\tconst own = new Set(candidates.filter((c) => c.own).map((c) => c.name));\n\tlet truePos = 0;\n\tlet positives = 0;\n\tlet trueNeg = 0;\n\tlet negatives = 0;\n\n\tconst results: TriggerCaseResult[] = cases.map((c, i) => {\n\t\tconst actual = verdicts[i] ?? null;\n\t\tconst correct = actual === c.expect;\n\t\t// A \"positive\" is a case this plugin should win. A case expecting a\n\t\t// sibling's capability is a negative *for us* — which is the discriminative\n\t\t// half, and why distractors are in the candidate list at all.\n\t\tif (c.expect !== null && own.has(c.expect)) {\n\t\t\tpositives++;\n\t\t\tif (correct) truePos++;\n\t\t} else {\n\t\t\tnegatives++;\n\t\t\t// Quiet means: did not claim one of ours.\n\t\t\tif (actual === null || !own.has(actual)) trueNeg++;\n\t\t}\n\t\treturn { prompt: c.prompt, expected: c.expect, actual, correct };\n\t});\n\n\treturn {\n\t\tstatus: \"ran\",\n\t\trecord: {\n\t\t\tcorpusHash: triggerCorpusHash(candidates, cases),\n\t\t\ttimestampMs: Date.now(),\n\t\t\tpluginId,\n\t\t\tcandidateCount: candidates.length,\n\t\t\tcaseCount: cases.length,\n\t\t\trecall: ratio(truePos, positives),\n\t\t\tspecificity: ratio(trueNeg, negatives),\n\t\t\tresults,\n\t\t},\n\t};\n}\n\n/**\n * Map an outcome onto gate findings.\n *\n * `not-run` is `info`, never a pass and never a failure: G4 is opt-in and needs a\n * model, so a machine without one must not be blocked from publishing — but the\n * record has to say the check did not happen rather than leaving a silence that\n * reads like a green.\n */\nexport function triggerFindings(outcome: TriggerEvalOutcome): GateFinding[] {\n\tif (outcome.status === \"not-run\") {\n\t\treturn [{ gate: \"G4\", severity: \"info\", message: `G4 trigger eval did not run — ${outcome.reason}` }];\n\t}\n\tconst { recall, specificity, caseCount } = outcome.record;\n\tconst findings: GateFinding[] = [];\n\tconst pct = (v: number) => `${Math.round(v * 100)}%`;\n\n\tif (recall !== undefined && recall < MIN_RECALL) {\n\t\tfindings.push({\n\t\t\tgate: \"G4\",\n\t\t\tseverity: \"error\",\n\t\t\tmessage:\n\t\t\t\t`Skills fired on only ${pct(recall)} of the prompts they are for (need ${pct(MIN_RECALL)}). ` +\n\t\t\t\t\"Rewrite the description around the situation that should trigger it, not the mechanism.\",\n\t\t});\n\t}\n\tif (specificity !== undefined && specificity < MIN_SPECIFICITY) {\n\t\tfindings.push({\n\t\t\tgate: \"G4\",\n\t\t\tseverity: \"warning\",\n\t\t\tmessage:\n\t\t\t\t`Skills also fired on ${pct(1 - specificity)} of the prompts they should have stayed quiet on ` +\n\t\t\t\t`(need ${pct(MIN_SPECIFICITY)} specificity). An over-broad description costs context on every request.`,\n\t\t});\n\t}\n\tif (findings.length === 0) {\n\t\tconst parts = [\n\t\t\trecall !== undefined ? `recall ${pct(recall)}` : undefined,\n\t\t\tspecificity !== undefined ? `specificity ${pct(specificity)}` : undefined,\n\t\t].filter(Boolean);\n\t\tfindings.push({\n\t\t\tgate: \"G4\",\n\t\t\tseverity: \"info\",\n\t\t\tmessage: `G4 trigger eval passed over ${caseCount} case(s): ${parts.join(\", \")}.`,\n\t\t});\n\t}\n\treturn findings;\n}\n"]}