import { createHash } from "node:crypto"; import { readFile, stat } from "node:fs/promises"; import type { UltraConfig } from "../types.js"; import { assertConfig } from "../config/schema.js"; import { loadConfig, type UltraPaths } from "../config/loader.js"; import { TelemetryCollector } from "../telemetry/collector.js"; const CATEGORIES = new Set(["routing", "model", "thinking", "fanout", "budget", "context", "verification", "prompt", "provider"]); const CONFIDENCE = new Set(["low", "medium", "high"]); const THINKING = new Set(["off", "minimal", "low", "medium", "high", "xhigh", "max"]); const ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; const FORBIDDEN_PATCH_ROOTS = new Set(["schemaVersion", "configVersion", "policyVersion", "profile", "compatibility", "telemetry"]); const MAX_FILE_BYTES = 64 * 1024; export interface WeeklyRecommendation { id: string; title: string; category: "routing" | "model" | "thinking" | "fanout" | "budget" | "context" | "verification" | "prompt" | "provider"; evidence: Array<{ metric: string; cohort: string; current: number; comparison: number; sampleSize: number }>; proposedPatch: object; expectedCreditImpactPct: number; expectedQualityImpactPct: number; confidence: "low" | "medium" | "high"; risks: string[]; rollbackCondition: string; experimentRequired: boolean } function object(value: unknown): value is Record { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } function text(value: unknown, label: string, max = 500): asserts value is string { if (typeof value !== "string" || !value.trim() || value.length > max) throw new Error(`Recommendation ${label} must be 1-${max} characters`); } function patchLeaf(value: unknown, prefix: string[] = [], leaves: Array<{ path: string[]; value: unknown }> = []): Array<{ path: string[]; value: unknown }> { if (!object(value)) { leaves.push({ path: prefix, value }); return leaves; } const entries = Object.entries(value); if (!entries.length || prefix.length >= 8) throw new Error("Recommendation patch must contain exactly one leaf"); for (const [key, child] of entries) { if (!/^[A-Za-z][A-Za-z0-9]{0,63}$/.test(key) || ["__proto__", "prototype", "constructor"].includes(key)) throw new Error("Recommendation patch contains an unsafe key"); patchLeaf(child, [...prefix, key], leaves); } return leaves; } export function validateRecommendations(value: unknown): asserts value is WeeklyRecommendation[] { if (!Array.isArray(value) || value.length === 0 || value.length > 3) throw new Error("Weekly protocol permits one to three recommendations"); const ids = new Set(); for (const item of value) { if (!object(item)) throw new Error("Recommendation must be an object"); const allowed = new Set(["id", "title", "category", "evidence", "proposedPatch", "expectedCreditImpactPct", "expectedQualityImpactPct", "confidence", "risks", "rollbackCondition", "experimentRequired"]); if (Object.keys(item).some((key) => !allowed.has(key))) throw new Error("Recommendation contains unknown fields"); text(item.id, "id", 128); if (!ID.test(item.id) || ids.has(item.id)) throw new Error("Recommendation id must be unique and safe"); ids.add(item.id); text(item.title, "title", 200); if (typeof item.category !== "string" || !CATEGORIES.has(item.category)) throw new Error("Recommendation category is invalid"); if (!Array.isArray(item.evidence) || !item.evidence.length || item.evidence.length > 32) throw new Error("Recommendation evidence must contain 1-32 entries"); for (const evidence of item.evidence) { if (!object(evidence) || Object.keys(evidence).some((key) => !["metric", "cohort", "current", "comparison", "sampleSize"].includes(key))) throw new Error("Recommendation evidence is invalid"); text(evidence.metric, "evidence metric", 128); text(evidence.cohort, "evidence cohort", 128); if (![evidence.current, evidence.comparison].every((number) => typeof number === "number" && Number.isFinite(number)) || typeof evidence.sampleSize !== "number" || !Number.isSafeInteger(evidence.sampleSize) || evidence.sampleSize < 0) throw new Error("Recommendation evidence values are invalid"); } if (!object(item.proposedPatch) || patchLeaf(item.proposedPatch).length !== 1) throw new Error("Recommendation patch must contain exactly one leaf"); if (![item.expectedCreditImpactPct, item.expectedQualityImpactPct].every((number) => typeof number === "number" && Number.isFinite(number) && Math.abs(number) <= 1_000)) throw new Error("Recommendation impact must be a finite bounded percentage"); if (typeof item.confidence !== "string" || !CONFIDENCE.has(item.confidence)) throw new Error("Recommendation confidence is invalid"); if (!Array.isArray(item.risks) || item.risks.length > 16) throw new Error("Recommendation risks exceed the limit"); for (const risk of item.risks) text(risk, "risk", 500); text(item.rollbackCondition, "rollback condition", 1_000); if (typeof item.experimentRequired !== "boolean") throw new Error("Recommendation experimentRequired must be boolean"); if (item.confidence === "high" && item.evidence.some((evidence) => evidence.sampleSize < 8)) throw new Error("High-confidence recommendation lacks minimum sample"); const comparablePairs = item.evidence.find((evidence) => evidence.metric === "verified_comparable_pairs"); if (item.confidence === "high" && (!comparablePairs || !Number.isSafeInteger(comparablePairs.current) || comparablePairs.current < 5)) throw new Error("High-confidence recommendation lacks five verified comparable pairs"); } } function assertPatchValue(path: string, current: unknown, next: unknown): void { if (next === null || typeof next === "object" || typeof next !== typeof current || Object.is(current, next)) throw new Error(`Recommendation must change existing config leaf ${path}`); if (typeof next === "string") { if (!next || next.length > 256) throw new Error(`Recommendation value is invalid for ${path}`); if (path === "mode" && !["auto", "direct", "scout", "swarm", "deep", "warroom"].includes(next)) throw new Error("Recommendation mode is invalid"); if (path === "policy" && !["economy", "balanced", "quality", "max"].includes(next)) throw new Error("Recommendation policy is invalid"); if (path.endsWith(".thinking") && !THINKING.has(next)) throw new Error(`Recommendation thinking is invalid for ${path}`); if (path.endsWith(".model") && !/^[A-Za-z0-9._-]+\/[A-Za-z0-9._:+-]+$/.test(next)) throw new Error(`Recommendation model is invalid for ${path}`); } if (typeof next === "number") { if (!Number.isFinite(next) || next < 0 || next > 1_000_000 || (Number.isInteger(current) && !Number.isSafeInteger(next))) throw new Error(`Recommendation number is invalid for ${path}`); if (["context.compactAtRatio", "context.blockWideSwarmAtRatio", "experiment.challengerAllocation", "experiment.stopLossSuccessDrop"].includes(path) && next > 1) throw new Error(`Recommendation ratio is invalid for ${path}`); } } export function applyRecommendation(current: UltraConfig, recommendation: WeeklyRecommendation): UltraConfig { validateRecommendations([recommendation]); const { path, value } = patchLeaf(recommendation.proposedPatch)[0]!; if (!path?.length || FORBIDDEN_PATCH_ROOTS.has(path[0]!)) throw new Error("Recommendation cannot patch config metadata"); const next = structuredClone(current) as unknown as Record; let parent = next; for (const key of path.slice(0, -1)) { if (!Object.hasOwn(parent, key) || !object(parent[key])) throw new Error(`Recommendation must change existing config leaf ${path.join(".")}`); parent = parent[key]; } const key = path.at(-1)!; if (!Object.hasOwn(parent, key)) throw new Error(`Recommendation must change existing config leaf ${path.join(".")}`); assertPatchValue(path.join("."), parent[key], value); parent[key] = value; const validated = next as unknown as UltraConfig; assertConfig(validated); return validated; } export async function loadRecommendation(file: string, recommendationId?: string): Promise { const info = await stat(file); if (!info.isFile() || info.size > MAX_FILE_BYTES) throw new Error(`Recommendation file must be at most ${MAX_FILE_BYTES} bytes`); const parsed = JSON.parse(await readFile(file, "utf8")) as unknown; const recommendations = Array.isArray(parsed) ? parsed : [parsed]; validateRecommendations(recommendations); if (recommendationId) { const selected = recommendations.find((item) => item.id === recommendationId); if (!selected) throw new Error(`Unknown recommendation ${recommendationId}`); return selected; } if (recommendations.length !== 1) throw new Error("Recommendation id is required when the file contains multiple recommendations"); return recommendations[0]!; } export async function recordRecommendationDecision(paths: UltraPaths, recommendation: WeeklyRecommendation, decision: "accepted" | "rejected", details: { challengerVersion?: string; experimentStarted?: boolean } = {}): Promise { const config = await loadConfig(paths); const digest = createHash("sha256").update(recommendation.id).digest("hex"); const runId = `recommendation:${digest.slice(0, 16)}`; const collector = new TelemetryCollector(paths.events, paths.database, paths.hmacKey, config, config.profile); try { await collector.record({ sessionId: "system", taskId: runId, runId }, `recommendation.${decision}`, { recommendationHash: digest, category: recommendation.category, confidence: recommendation.confidence, experimentRequired: recommendation.experimentRequired, ...details }); } finally { collector.close(); } }