/* evaluators registry — register certification evaluators keyed by risk class. Do not evaluate — the registry only maps risk classes to evaluators. RFC-1097: sweep — werkstatt-engine clean Sweep batch 4: 73 Compass headers on headerless engine files (certification, component-runtime, isolation, evolution, testing), real KEY_DECISIONS on 75 files (kernel, cache, dht, swim, gitmesh, runtime), ~80 purpose expansions (CONTRACT-02/PURPOSE-02), non-goals on 13 CONTRACT-03 files, CS-07 history literal fix repo-wide (253 files). Policy: .template.ts/.template.astro excludedPaths. werkstatt-engine now 0 diagnostics. */ import type { Sha256Digest } from "@warpgogol/werkstatt-shared/fingerprint"; import type { Diagnostic } from "@warpgogol/werkstatt-shared/kernel"; import type { EvidenceEnvelopeV1 } from "../contracts/evidence.ts"; export type RiskClass = "ordinary" | "critical" | "borderline"; export type EvaluatorVerdict = "pass" | "fail" | "borderline" | "incomplete"; export interface QualitativeRubricV1 { schema: "werkstatt/site-qualitative-rubric@1"; rubricId: string; version: string; criteria: RubricCriterionV1[]; } export interface RubricCriterionV1 { criterionId: string; title: string; description: string; dimension: string; weight: number; } export interface EvaluatorInputBundleV1 { schema: "werkstatt/evaluator-input-bundle@1"; bundleId: string; candidateId: string; rubricId: string; rubricVersion: string; coverageManifest: CoverageManifestV1; deterministicEvidence: EvidenceEnvelopeV1[]; changedRoutes: string[]; changedStates: string[]; redactedContent: string; bundleHash: Sha256Digest; } export interface CoverageManifestV1 { schema: "werkstatt/coverage-manifest@1"; routes: string[]; states: string[]; viewports: string[]; combinations: CoverageCombinationV1[]; } export interface CoverageCombinationV1 { route: string; state: string; viewport: string; covered: boolean; evidenceId: string | null; } export interface QualitativeEvaluationPayloadV1 { schema: "werkstatt/site-qualitative-evaluation@1"; evaluatorId: string; evaluatorRunId: string; modelProvider: string; modelVersion: string; rubricId: string; rubricVersion: string; inputBundleHash: Sha256Digest; riskClass: RiskClass; verdict: EvaluatorVerdict; confidence: number; criteria: CriterionVerdictV1[]; diagnostics: Diagnostic[]; } export interface CriterionVerdictV1 { criterionId: string; verdict: EvaluatorVerdict; rationale: string; diagnosticIds: string[]; evidenceAnchors: string[]; } export interface EvaluatorIdentityV1 { evaluatorId: string; modelProvider: string; modelVersion: string; } export type EvaluatorHandlerV1 = ( input: EvaluatorInputBundleV1, identity: EvaluatorIdentityV1, ) => Promise; export interface RegisteredEvaluatorV1 { identity: EvaluatorIdentityV1; handler: EvaluatorHandlerV1; } export interface EvaluatorRegistryV1 { register(identity: EvaluatorIdentityV1, handler: EvaluatorHandlerV1): EvaluatorRegisterOutcomeV1; get(evaluatorId: string): RegisteredEvaluatorV1 | null; list(): readonly string[]; } export interface EvaluatorRegisterResultV1 { ok: true; evaluatorId: string; } export interface EvaluatorRegisterFailureV1 { ok: false; ruleId: "CERT-EVAL-01"; message: string; } export type EvaluatorRegisterOutcomeV1 = EvaluatorRegisterResultV1 | EvaluatorRegisterFailureV1; export function createEvaluatorRegistry(): EvaluatorRegistryV1 { const evaluators = new Map(); return { register( identity: EvaluatorIdentityV1, handler: EvaluatorHandlerV1, ): EvaluatorRegisterOutcomeV1 { if (evaluators.has(identity.evaluatorId)) { return { ok: false, ruleId: "CERT-EVAL-01", message: `evaluator "${identity.evaluatorId}" is already registered`, }; } evaluators.set(identity.evaluatorId, { identity, handler }); return { ok: true, evaluatorId: identity.evaluatorId }; }, get(evaluatorId: string): RegisteredEvaluatorV1 | null { return evaluators.get(evaluatorId) ?? null; }, list(): readonly string[] { return [...evaluators.keys()]; }, }; } export interface RiskRuleV1 { ruleId: string; dimension: string; description: string; } export interface RiskRoutingResultV1 { riskClass: RiskClass; requiredEvaluatorCount: number; matchedRules: RiskRuleV1[]; } export function routeRisk( changeProfile: ChangeProfileV1, criticalRules: readonly RiskRuleV1[], borderlineRules: readonly RiskRuleV1[], ): RiskRoutingResultV1 { const matchedCritical: RiskRuleV1[] = []; for (const rule of criticalRules) { if (changeProfile.dimensions.includes(rule.dimension)) { matchedCritical.push(rule); } } if (matchedCritical.length > 0) { return { riskClass: "critical", requiredEvaluatorCount: 2, matchedRules: matchedCritical, }; } const matchedBorderline: RiskRuleV1[] = []; for (const rule of borderlineRules) { if (changeProfile.dimensions.includes(rule.dimension)) { matchedBorderline.push(rule); } } if (matchedBorderline.length > 0) { return { riskClass: "borderline", requiredEvaluatorCount: 2, matchedRules: matchedBorderline, }; } return { riskClass: "ordinary", requiredEvaluatorCount: 1, matchedRules: [], }; } export interface ChangeProfileV1 { changedRoutes: string[]; changedStates: string[]; dimensions: string[]; hasStructuralChanges: boolean; hasContentChanges: boolean; hasStyleChanges: boolean; } export interface EvaluatorIsolationCheckV1 { ok: true; passed: true; } export interface EvaluatorIsolationFailureV1 { ok: false; ruleId: "CERT-EVAL-02" | "CERT-EVAL-03" | "CERT-EVAL-04"; message: string; } export type EvaluatorIsolationOutcomeV1 = EvaluatorIsolationCheckV1 | EvaluatorIsolationFailureV1; export function checkEvaluatorIsolation( evaluatorIds: readonly string[], authorAgentId: string, ): EvaluatorIsolationOutcomeV1 { const seen = new Set(); for (const id of evaluatorIds) { if (id === authorAgentId) { return { ok: false, ruleId: "CERT-EVAL-02", message: `evaluator "${id}" matches author agent "${authorAgentId}" — self-review is not permitted`, }; } if (seen.has(id)) { return { ok: false, ruleId: "CERT-EVAL-03", message: `duplicate evaluator identity "${id}" — distinct evaluator IDs are required`, }; } seen.add(id); } return { ok: true, passed: true }; } export interface ConsensusResultV1 { ok: true; consensus: "pass" | "fail" | "incomplete"; verdicts: EvaluatorVerdict[]; } export interface ConsensusFailureV1 { ok: false; ruleId: "CERT-EVAL-05" | "CERT-EVAL-06"; message: string; } export type ConsensusOutcomeV1 = ConsensusResultV1 | ConsensusFailureV1; export function aggregateConsensus( payloads: readonly QualitativeEvaluationPayloadV1[], ): ConsensusOutcomeV1 { if (payloads.length === 0) { return { ok: false, ruleId: "CERT-EVAL-05", message: "no evaluator payloads provided — missing runs map to incomplete", }; } const verdicts = payloads.map((p) => p.verdict); const allPass = verdicts.every((v) => v === "pass"); const allFail = verdicts.every((v) => v === "fail"); if (allPass) { return { ok: true, consensus: "pass", verdicts }; } if (allFail) { return { ok: true, consensus: "fail", verdicts }; } return { ok: true, consensus: "incomplete", verdicts }; } export interface EvaluatorValidationResultV1 { ok: true; passed: true; } export interface EvaluatorValidationFailureV1 { ok: false; ruleId: "CERT-EVAL-07" | "CERT-EVAL-08" | "CERT-EVAL-09" | "CERT-EVAL-10"; message: string; } export type EvaluatorValidationOutcomeV1 = EvaluatorValidationResultV1 | EvaluatorValidationFailureV1; export function validateEvaluatorPayload( payload: QualitativeEvaluationPayloadV1, expectedBundleHash: Sha256Digest, rubric: QualitativeRubricV1, ): EvaluatorValidationOutcomeV1 { if (payload.inputBundleHash !== expectedBundleHash) { return { ok: false, ruleId: "CERT-EVAL-07", message: `input bundle hash mismatch: expected ${expectedBundleHash}, got ${payload.inputBundleHash}`, }; } if (payload.rubricId !== rubric.rubricId || payload.rubricVersion !== rubric.version) { return { ok: false, ruleId: "CERT-EVAL-08", message: `rubric mismatch: expected ${rubric.rubricId}@${rubric.version}, got ${payload.rubricId}@${payload.rubricVersion}`, }; } if (payload.confidence < 0 || payload.confidence > 100) { return { ok: false, ruleId: "CERT-EVAL-09", message: `confidence ${payload.confidence} is out of range [0, 100]`, }; } const rubricCriterionIds = new Set(rubric.criteria.map((c) => c.criterionId)); for (const cv of payload.criteria) { if (!rubricCriterionIds.has(cv.criterionId)) { return { ok: false, ruleId: "CERT-EVAL-10", message: `criterion "${cv.criterionId}" is not in rubric "${rubric.rubricId}"`, }; } if (!cv.rationale || cv.rationale.trim().length === 0) { return { ok: false, ruleId: "CERT-EVAL-10", message: `criterion "${cv.criterionId}" has empty rationale — ungrounded prose is not permitted`, }; } } return { ok: true, passed: true }; } export interface EvaluatorExecutionRequestV1 { evaluatorIds: readonly string[]; inputBundle: EvaluatorInputBundleV1; authorAgentId: string; rubric: QualitativeRubricV1; } export interface EvaluatorExecutionSuccessV1 { ok: true; payloads: QualitativeEvaluationPayloadV1[]; consensus: "pass" | "fail" | "incomplete"; } export interface EvaluatorExecutionFailureV1 { ok: false; ruleId: "CERT-EVAL-11" | "CERT-EVAL-12" | "CERT-EVAL-13"; message: string; } export type EvaluatorExecutionOutcomeV1 = EvaluatorExecutionSuccessV1 | EvaluatorExecutionFailureV1; export async function executeEvaluators( registry: EvaluatorRegistryV1, request: EvaluatorExecutionRequestV1, ): Promise { const isolationCheck = checkEvaluatorIsolation(request.evaluatorIds, request.authorAgentId); if (!isolationCheck.ok) { return { ok: false, ruleId: "CERT-EVAL-11", message: isolationCheck.message, }; } const payloads: QualitativeEvaluationPayloadV1[] = []; for (const evaluatorId of request.evaluatorIds) { const registered = registry.get(evaluatorId); if (!registered) { return { ok: false, ruleId: "CERT-EVAL-12", message: `evaluator "${evaluatorId}" is not registered`, }; } let payload: QualitativeEvaluationPayloadV1; try { payload = await registered.handler(request.inputBundle, registered.identity); } catch (err) { return { ok: false, ruleId: "CERT-EVAL-13", message: `evaluator "${evaluatorId}" threw: ${err instanceof Error ? err.message : String(err)}`, }; } const validation = validateEvaluatorPayload( payload, request.inputBundle.bundleHash, request.rubric, ); if (!validation.ok) { return { ok: false, ruleId: "CERT-EVAL-13", message: validation.message, }; } payloads.push(payload); } const consensusResult = aggregateConsensus(payloads); if (!consensusResult.ok) { return { ok: false, ruleId: "CERT-EVAL-13", message: consensusResult.message, }; } return { ok: true, payloads, consensus: consensusResult.consensus, }; } export function buildCoverageManifest( routes: readonly string[], states: readonly string[], viewports: readonly string[], evidence: readonly EvidenceEnvelopeV1[], ): CoverageManifestV1 { const evidenceByRoute = new Map(); for (const ev of evidence) { for (const reqId of ev.result.applicability.appliesTo) { evidenceByRoute.set(reqId, ev.evidenceId); } } const combinations: CoverageCombinationV1[] = []; for (const route of routes) { for (const state of states) { for (const viewport of viewports) { const key = `${route}::${state}::${viewport}`; const evidenceId = evidenceByRoute.get(key) ?? null; combinations.push({ route, state, viewport, covered: evidenceId !== null, evidenceId, }); } } } return { schema: "werkstatt/coverage-manifest@1", routes: [...routes], states: [...states], viewports: [...viewports], combinations, }; }