/** * Console AI projection — AiRuntime journal / denial ledger + Manifest * + wide events (console §9.10). * * The UI must not recompute cost or re-derive denials. */ import { mockAiDriver } from "../../drivers/ai-mock.ts"; import { ai, createAiRuntime, type AgentDenial, type AgentRunRecord, type AgentStopReason, type AgentToolEffect, type AiAskOutcome, type AiFallbackAttempt, type AiJournalEntry, type AiRuntime, type EvalSuiteResult, } from "../../elements/ai.ts"; import type { Manifest } from "../../manifest/types.ts"; import type { WideEvent } from "../../runs/types.ts"; /** One numeric sample in a distribution. */ export interface AiSample { readonly value: number; readonly at: number; readonly version?: number; } /** Histogram bucket for a metric distribution. */ export interface AiDistributionBucket { readonly min: number; readonly max: number; readonly count: number; } /** Per-prompt-version quality / cost metrics (real numbers). */ export interface PromptVersionMetrics { readonly prompt: string; readonly version: number; readonly sampleCount: number; readonly cost: { readonly samples: readonly number[]; readonly mean: number; readonly p50: number; readonly p95: number; readonly buckets: readonly AiDistributionBucket[]; }; readonly latencyMs: { readonly samples: readonly number[]; readonly mean: number; readonly p50: number; readonly p95: number; readonly buckets: readonly AiDistributionBucket[]; }; readonly evalScore: { readonly samples: readonly number[]; readonly mean: number; readonly buckets: readonly AiDistributionBucket[]; }; /** Schema-invalid rate — distinct from provider errors. */ readonly schemaInvalidRate: number; readonly providerErrorRate: number; readonly okRate: number; readonly overBudgetRate: number; readonly budgetMaxCostPerCall: number | null; readonly outcomeCounts: Readonly>; } /** Standing security-review row for `allowPii`. */ export interface AllowPiiRow { readonly flowId: string; readonly asks: readonly string[]; readonly pii: "masked" | "allow" | "denied" | null; readonly allowPii: boolean; readonly source: string | null; } /** Fallback chain with cost consequence vs primary-only. */ export interface FallbackChainRow { readonly prompt: string; readonly version: number | undefined; readonly attempts: readonly AiFallbackAttempt[]; readonly actualCost: number; readonly primaryOnlyCost: number | null; readonly costConsequence: number | null; readonly at: number; } /** Agent run projection — trail from the denial ledger. */ export interface AgentRunRow { readonly id: string; readonly agent: string; readonly message: string; readonly ok: boolean; readonly stopReason: AgentStopReason; readonly error?: string; readonly steps: number; readonly cost: number; readonly at: number; readonly trail: ReadonlyArray<{ readonly tool: string; readonly status: "ok" | "denied"; readonly effects: readonly AgentToolEffect[]; readonly denial: AgentDenial | null; readonly at: number; }>; readonly denials: readonly AgentDenial[]; } /** Catalogue prompt row. */ export interface PromptCatalogueRow { readonly name: string; readonly version: number | undefined; readonly model: string | undefined; readonly evals: string | undefined; readonly budgetMaxCostPerCall: number | null; /** Manifest Diff deep-link path for a version bump. */ readonly manifestDiffPath: string; } /** Catalogue agent row. */ export interface AgentCatalogueRow { readonly name: string; readonly tools: readonly string[]; readonly maxSteps: number | undefined; readonly model: string | undefined; readonly budgetMaxCostPerRun: number | null; } /** Full AI panel projection. */ export interface ConsoleAiProjection { readonly prompts: readonly PromptCatalogueRow[]; readonly agents: readonly AgentCatalogueRow[]; readonly versions: readonly PromptVersionMetrics[]; readonly allowPii: readonly AllowPiiRow[]; readonly fallbackChains: readonly FallbackChainRow[]; readonly agentRuns: readonly AgentRunRow[]; readonly denials: readonly AgentDenial[]; readonly journal: readonly AiJournalEntry[]; } /** Options for {@link projectAiPanel}. */ export interface ProjectAiOptions { readonly manifest: Manifest | null; readonly aiRuntime: AiRuntime | null; readonly runs?: readonly WideEvent[]; readonly evalResults?: readonly EvalSuiteResult[]; readonly bucketCount?: number; } /** * Build an {@link AiRuntime} from Manifest AI catalogue (mock driver default). * Agent tool effects come from Manifest flow declarations — the denial ledger * records them; the UI must not re-derive. * * @param manifest - Manifest * @param options - Clock */ export function createManifestAiRuntime( manifest: Manifest | null, options: { readonly now?: () => number; readonly journalStore?: import("../../kernel/journal.ts").JournalStore | null; readonly eventLog?: import("../../elements/ai/run-events.ts").AgentEventLog; } = {}, ): AiRuntime { const models = Object.entries(manifest?.ai?.models ?? {}).map(([name, m]) => ai.model(name, { ...(m.provider !== undefined ? { provider: m.provider } : {}), ...(m.tier !== undefined ? { tier: m.tier } : {}), ...(m.model !== undefined ? { model: m.model } : {}), }), ); const modelByName = new Map(models.map((m) => [m.name, m])); const prompts = Object.entries(manifest?.ai?.prompts ?? {}).map(([name, p]) => { const modelName = p.model ?? models[0]?.name ?? "mock"; const handle = modelByName.get(modelName) ?? ai.model(modelName); return handle.prompt(name, { ...(p.version !== undefined ? { version: p.version } : {}), ...(p.evals !== undefined ? { evals: p.evals } : {}), ...(p.budget !== undefined ? { budget: p.budget } : {}), ...(p.in !== undefined ? { in: p.in } : {}), ...(p.out !== undefined ? { out: p.out } : {}), }); }); const agents = Object.entries(manifest?.ai?.agents ?? {}).map(([name, a]) => ai.agent(name, { tools: a.tools ?? [], ...(a.maxSteps !== undefined ? { maxSteps: a.maxSteps } : {}), ...(a.budget !== undefined ? { budget: a.budget } : {}), ...(a.model !== undefined ? { model: a.model } : {}), }), ); return createAiRuntime({ models: models.length > 0 ? models : prompts.length > 0 ? [ai.model("mock", { provider: "mock" })] : [], prompts, agents, defaultDriver: mockAiDriver, effectsForFlow: (flowName) => effectsForFlowFromManifest(manifest, flowName), now: options.now, ...(options.journalStore ? { journalStore: options.journalStore } : {}), ...(options.eventLog ? { eventLog: options.eventLog } : {}), }); } /** * Project AI panel data from Manifest + AiRuntime + runs + eval history. * * @param options - Sources */ export function projectAiPanel(options: ProjectAiOptions): ConsoleAiProjection { const manifest = options.manifest; const runtime = options.aiRuntime; const bucketCount = options.bucketCount ?? 10; const prompts: PromptCatalogueRow[] = []; const promptMeta = new Map< string, { version?: number; budget?: number | null; model?: string } >(); for (const [name, p] of Object.entries(manifest?.ai?.prompts ?? {})) { const budget = p.budget?.maxCostPerCall ?? null; prompts.push({ name, version: p.version, model: p.model, evals: p.evals, budgetMaxCostPerCall: budget, manifestDiffPath: `/ai/prompts/${name}/version`, }); promptMeta.set(name, { version: p.version, budget, model: p.model, }); } if (runtime) { for (const [name, p] of runtime.prompts) { if (promptMeta.has(name)) continue; const budget = p.budget?.maxCostPerCall ?? null; prompts.push({ name, version: p.version, model: p.model, evals: p.evals, budgetMaxCostPerCall: budget, manifestDiffPath: `/ai/prompts/${name}/version`, }); promptMeta.set(name, { version: p.version, budget, model: p.model, }); } } prompts.sort((a, b) => a.name.localeCompare(b.name)); const agents: AgentCatalogueRow[] = []; for (const [name, a] of Object.entries(manifest?.ai?.agents ?? {})) { agents.push({ name, tools: a.tools ?? [], maxSteps: a.maxSteps, model: a.model, budgetMaxCostPerRun: a.budget?.maxCostPerRun ?? null, }); } if (runtime) { for (const [name, a] of runtime.agents) { if (agents.some((x) => x.name === name)) continue; agents.push({ name, tools: a.tools, maxSteps: a.maxSteps, model: a.model, budgetMaxCostPerRun: a.budget?.maxCostPerRun ?? null, }); } } agents.sort((a, b) => a.name.localeCompare(b.name)); const journal = runtime?.journal ?? []; const versions = buildVersionMetrics({ journal, runs: options.runs ?? [], evalResults: options.evalResults ?? [], promptMeta, bucketCount, }); const allowPii = projectAllowPii(manifest); const fallbackChains = projectFallbackChains(journal); const agentRuns = projectAgentRuns(runtime?.agentRuns ?? []); const denials = runtime?.denials ?? []; return { prompts, agents, versions, allowPii, fallbackChains, agentRuns, denials, journal, }; } /** * Resolve Manifest effects for a tool flow into the agent trail vocabulary. * * @param manifest - Manifest * @param flowName - Tool / flow name */ export function effectsForFlowFromManifest( manifest: Manifest | null, flowName: string, ): readonly AgentToolEffect[] { const flow = manifest?.flows?.[flowName]; if (!flow?.effects) return []; const out: AgentToolEffect[] = []; for (const r of flow.effects.reads ?? []) { out.push({ kind: "read", resource: r }); } for (const r of flow.effects.writes ?? []) { out.push({ kind: "write", resource: r }); } for (const r of flow.effects.emits ?? []) { out.push({ kind: "emit", resource: r }); } for (const r of flow.effects.sends ?? []) { out.push({ kind: "send", resource: r }); } for (const r of flow.effects.asks ?? []) { out.push({ kind: "ask", resource: r }); } for (const r of flow.effects.secrets ?? []) { out.push({ kind: "secret", resource: r }); } for (const r of flow.effects.calls ?? []) { out.push({ kind: "call", resource: r }); } return out; } /** * Standing `allowPii` security-review table from Manifest flows. * * @param manifest - Manifest */ export function projectAllowPii(manifest: Manifest | null): readonly AllowPiiRow[] { if (!manifest?.flows) return []; const rows: AllowPiiRow[] = []; for (const [flowId, flow] of Object.entries(manifest.flows)) { const allow = flow.allowPii === true || flow.pii === "allow"; const asks = flow.effects?.asks ?? []; if (!allow && asks.length === 0) continue; if (!allow && flow.pii === undefined) continue; rows.push({ flowId, asks: [...asks], pii: flow.pii ?? null, allowPii: allow, source: flow.source ?? null, }); } // Prefer rows that actually acknowledge PII egress return rows .filter((r) => r.allowPii || r.asks.length > 0) .sort((a, b) => { if (a.allowPii !== b.allowPii) return a.allowPii ? -1 : 1; return a.flowId.localeCompare(b.flowId); }); } function projectFallbackChains(journal: readonly AiJournalEntry[]): readonly FallbackChainRow[] { const rows: FallbackChainRow[] = []; for (const entry of journal) { if (entry.attempts.length < 2) continue; const primary = entry.attempts[0]; const primaryOnlyCost = primary?.ok && primary.cost !== undefined ? primary.cost : primary && !primary.ok ? null : (primary?.cost ?? null); // Cost consequence: actual chain cost minus what primary-only would have been // when primary succeeded; when primary failed, consequence is the full chain cost. let costConsequence: number | null = null; if (primary && !primary.ok) { costConsequence = entry.cost; } else if (primaryOnlyCost !== null) { costConsequence = entry.cost - primaryOnlyCost; } rows.push({ prompt: entry.prompt, version: entry.version, attempts: entry.attempts, actualCost: entry.cost, primaryOnlyCost, costConsequence, at: entry.at, }); } return rows.sort((a, b) => b.at - a.at); } function projectAgentRuns(runs: readonly AgentRunRecord[]): readonly AgentRunRow[] { return [...runs] .sort((a, b) => b.at - a.at) .map((r) => ({ id: r.id, agent: r.agent, message: r.message, ok: r.ok, stopReason: r.stopReason, ...(r.error !== undefined ? { error: r.error } : {}), steps: r.steps, cost: r.cost, at: r.at, trail: r.trail.map((step) => ({ tool: step.tool, status: step.status, effects: step.effects, denial: step.denial ?? null, at: step.at, })), denials: r.denials, })); } function buildVersionMetrics(input: { readonly journal: readonly AiJournalEntry[]; readonly runs: readonly WideEvent[]; readonly evalResults: readonly EvalSuiteResult[]; readonly promptMeta: Map; readonly bucketCount: number; }): readonly PromptVersionMetrics[] { type Acc = { prompt: string; version: number; costs: number[]; latencies: number[]; evals: number[]; outcomes: Record; overBudget: number; budget: number | null; }; const map = new Map(); const keyOf = (prompt: string, version: number) => `${prompt}@${version}`; const ensure = (prompt: string, version: number): Acc => { const k = keyOf(prompt, version); let acc = map.get(k); if (!acc) { const meta = input.promptMeta.get(prompt); acc = { prompt, version, costs: [], latencies: [], evals: [], outcomes: { ok: 0, provider_error: 0, schema_invalid: 0, budget_exceeded: 0 }, overBudget: 0, budget: meta?.budget ?? null, }; map.set(k, acc); } return acc; }; for (const entry of input.journal) { const version = entry.version ?? input.promptMeta.get(entry.prompt)?.version ?? 0; const acc = ensure(entry.prompt, version); acc.costs.push(entry.cost); acc.latencies.push(entry.latencyMs); acc.outcomes[entry.outcome] += 1; if (acc.budget !== null && entry.cost > acc.budget) { acc.overBudget += 1; } } // Wide events contribute cost / latency when promptVersion is present for (const run of input.runs) { const ask = run.effects.find((e) => e.kind === "ask"); if (!ask) continue; const prompt = ask.resource.replace(/^ask:/, ""); const version = run.promptVersion; if (version === undefined) continue; const acc = ensure(prompt, version); if (run.cost !== undefined) acc.costs.push(run.cost); acc.latencies.push(run.durationMs); if (run.error?.code === "AiSchemaInvalid") { acc.outcomes.schema_invalid += 1; } else if (run.error) { acc.outcomes.provider_error += 1; } else { acc.outcomes.ok += 1; } const budget = acc.budget; if (budget !== null && (run.cost ?? 0) > budget) { acc.overBudget += 1; } } for (const suite of input.evalResults) { const version = suite.version ?? input.promptMeta.get(suite.prompt)?.version; if (version === undefined) continue; const total = suite.passed + suite.failed; if (total === 0) continue; const score = suite.passed / total; ensure(suite.prompt, version).evals.push(score); } const out: PromptVersionMetrics[] = []; for (const acc of map.values()) { const totalOutcomes = acc.outcomes.ok + acc.outcomes.provider_error + acc.outcomes.schema_invalid + acc.outcomes.budget_exceeded; out.push({ prompt: acc.prompt, version: acc.version, sampleCount: Math.max(acc.costs.length, totalOutcomes), cost: metricBlock(acc.costs, input.bucketCount), latencyMs: metricBlock(acc.latencies, input.bucketCount), evalScore: metricBlock(acc.evals, input.bucketCount), schemaInvalidRate: totalOutcomes === 0 ? 0 : acc.outcomes.schema_invalid / totalOutcomes, providerErrorRate: totalOutcomes === 0 ? 0 : acc.outcomes.provider_error / totalOutcomes, okRate: totalOutcomes === 0 ? 0 : acc.outcomes.ok / totalOutcomes, overBudgetRate: acc.costs.length === 0 ? 0 : acc.overBudget / acc.costs.length, budgetMaxCostPerCall: acc.budget, outcomeCounts: acc.outcomes, }); } return out.sort((a, b) => { const byPrompt = a.prompt.localeCompare(b.prompt); if (byPrompt !== 0) return byPrompt; return a.version - b.version; }); } function metricBlock( samples: readonly number[], bucketCount: number, ): PromptVersionMetrics["cost"] { const sorted = [...samples].sort((a, b) => a - b); return { samples: sorted, mean: mean(sorted), p50: percentile(sorted, 0.5), p95: percentile(sorted, 0.95), buckets: histogram(sorted, bucketCount), }; } /** * Arithmetic mean (0 when empty). * * @param values - Samples */ export function mean(values: readonly number[]): number { if (values.length === 0) return 0; return values.reduce((a, b) => a + b, 0) / values.length; } /** * Nearest-rank percentile (0 when empty). * * @param sorted - Ascending samples * @param p - Percentile in [0, 1] */ export function percentile(sorted: readonly number[], p: number): number { if (sorted.length === 0) return 0; if (sorted.length === 1) return sorted[0]!; const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1)); return sorted[idx]!; } /** * Equal-width histogram over samples. * * @param sorted - Ascending samples * @param bucketCount - Buckets */ export function histogram( sorted: readonly number[], bucketCount: number, ): readonly AiDistributionBucket[] { if (sorted.length === 0 || bucketCount < 1) return []; const min = sorted[0]!; const max = sorted[sorted.length - 1]!; if (min === max) { return [{ min, max: max + Number.EPSILON, count: sorted.length }]; } const width = (max - min) / bucketCount; const buckets: AiDistributionBucket[] = []; for (let i = 0; i < bucketCount; i++) { const lo = min + i * width; const hi = i === bucketCount - 1 ? max + Number.EPSILON : min + (i + 1) * width; buckets.push({ min: lo, max: hi, count: 0 }); } for (const v of sorted) { let idx = Math.floor((v - min) / width); if (idx < 0) idx = 0; if (idx >= bucketCount) idx = bucketCount - 1; const b = buckets[idx]!; buckets[idx] = { ...b, count: b.count + 1 }; } return buckets; }