import { z } from "zod"; import { AnalysisReviewError, DiagnosticValidationError, canonicalJson, createAnalysisReference, verifyAnalysisReference, snapshotDiagnosticJson, isDiagnosticToken, isDiagnosticPlainRecord, MODEL_DIAGNOSTIC_CHECKS, assessModelDiagnosticEligibility, runModelDiagnostics, type AnalysisReference, type AnalysisReviewFinding, type DiagnosticMetricEvaluation } from "@actuarial-ts/core"; import { validateDiagnosticRunInput, runValidatedMetricDiagnostics, type ValidatedDiagnosticRunInput } from "./diagnosticInput.js"; import { listDiagnosticDataChecks } from "./diagnosticPreparedReview.js"; import { parseModelDiagnosticRequest } from "./modelDiagnosticsContracts.js"; import { parseReviewBoundary } from "./reviewBoundary.js"; import { DATA_PACKAGE_VERSION } from "./version.js"; import { DIAGNOSTIC_SUITE_CONTRACT_VERSION, DIAGNOSTIC_SUITE_LIMITS, DIAGNOSTIC_SUITE_PURPOSES, type DiagnosticSuiteRequest, type DiagnosticSuiteDataInput, type DiagnosticSuiteDataEvidence, type DiagnosticSuiteResult, type DiagnosticSuiteCheck, type DiagnosticSuiteCheckResult, type DiagnosticSuiteMetricEvidence } from "./diagnosticSuiteTypes.js"; const token = z.string().refine(isDiagnosticToken), base = { id: token, rationale: token, interpretationLimits: z.array(token).min(1) }; const policy = z.object({ allowedReviewStatuses: z.array(z.enum(["pass", "warning", "fail", "not-evaluated"])), allowedMetricFindingSeverities: z.array(z.enum(["info", "warning", "fail"])), rationaleRef: token }).strict(); export const diagnosticSuiteCheckSchema: z.ZodType = z.discriminatedUnion("kind", [ z.object({ ...base, kind: z.literal("model"), model: token, check: z.enum(MODEL_DIAGNOSTIC_CHECKS) }).strict(), z.object({ ...base, kind: z.literal("data-check"), dataset: token, checkId: token }).strict(), z.object({ ...base, kind: z.literal("metric"), dataset: token, instanceId: token }).strict(), z.object({ ...base, kind: z.literal("metric-rule"), dataset: token, instanceId: token, ruleId: token }).strict(), ]); const limitsSchema = z.object({ maximumDatasets: z.number().int().positive().max(DIAGNOSTIC_SUITE_LIMITS.maximumDatasets), maximumModels: z.number().int().positive().max(DIAGNOSTIC_SUITE_LIMITS.maximumModels), maximumSuites: z.number().int().positive().max(DIAGNOSTIC_SUITE_LIMITS.maximumSuites), maximumChecks: z.number().int().positive().max(DIAGNOSTIC_SUITE_LIMITS.maximumChecks), maximumInputRows: z.number().int().positive().max(DIAGNOSTIC_SUITE_LIMITS.maximumInputRows), maximumEvaluations: z.number().int().positive().max(DIAGNOSTIC_SUITE_LIMITS.maximumEvaluations), maximumInputCharacters: z.number().int().positive().max(DIAGNOSTIC_SUITE_LIMITS.maximumInputCharacters), maximumOutputCharacters: z.number().int().positive().max(DIAGNOSTIC_SUITE_LIMITS.maximumOutputCharacters), }).strict(); /** Nested native requests are validated intact by their existing SDK boundaries. */ export const diagnosticSuiteRequestSchema: z.ZodType = z.object({ id: token, version: token, datasets: z.array(z.object({ id: token, input: z.custom(isDiagnosticPlainRecord) }).strict()).max(DIAGNOSTIC_SUITE_LIMITS.maximumDatasets), models: z.array(z.object({ id: token, request: z.custom(isDiagnosticPlainRecord) }).strict()).max(DIAGNOSTIC_SUITE_LIMITS.maximumModels), suites: z.array(z.object({ id: token, title: token, purpose: z.enum(DIAGNOSTIC_SUITE_PURPOSES), checks: z.array(diagnosticSuiteCheckSchema).min(1).max(DIAGNOSTIC_SUITE_LIMITS.maximumChecks) }).strict()).min(1).max(DIAGNOSTIC_SUITE_LIMITS.maximumSuites), limits: limitsSchema, }).strict(); function invalid(path: string, message: string, code = "INVALID_DIAGNOSTIC_SUITE", coordinates?: AnalysisReviewFinding["coordinates"]): never { throw new AnalysisReviewError([{ code, path, message, ...(coordinates ? { coordinates } : {}), remedies: ["Inspect the selected native source, check definition and explicit suite limits"] }]); } function distinct(ids: readonly string[], path: string): void { if (new Set(ids).size !== ids.length) invalid(path, "IDs must be distinct"); } function localPath(path: string): string { return path.startsWith("$") ? path.slice(1) : path === "input" || path.startsWith("input.") || path.startsWith("input[") ? path.slice(5) : path.startsWith("[") ? path : `.${path}`; } function subjectFindings(findings: readonly AnalysisReviewFinding[], path: string, subject: { dataset?: string; model?: string }): AnalysisReviewFinding[] { return findings.map((finding) => { return { ...finding, path: `${path}${localPath(finding.path)}`, coordinates: { ...finding.coordinates, ...subject } }; }); } function atSubject(path: string, subject: { dataset?: string; model?: string }, operation: () => T): T { try { return operation(); } catch (error) { if (!(error instanceof AnalysisReviewError) && !(error instanceof DiagnosticValidationError)) throw error; const findings = error instanceof AnalysisReviewError ? error.findings : error.issues.map((issue) => ({ code: issue.code, message: issue.message, path: issue.domain === "definition" && subject.dataset !== undefined ? `$.definition${localPath(issue.path)}` : issue.path, remedies: ["Resolve the native diagnostic input at this dataset coordinate"] })); throw new AnalysisReviewError(subjectFindings(findings, path, subject)); } } function prepare(input: unknown): { request: DiagnosticSuiteRequest; datasets: Map; evaluationUpperBound: number; findings: readonly AnalysisReviewFinding[] } { const request = snapshotDiagnosticJson(parseReviewBoundary(diagnosticSuiteRequestSchema, input)), limits = request.limits; const locatedChecks = request.suites.flatMap((suite, i) => suite.checks.map((check, j) => ({ check, path: `$.suites[${i}].checks[${j}]`, suite: suite.id }))), checks = locatedChecks.map((item) => item.check); if (canonicalJson(request).length > limits.maximumInputCharacters || request.datasets.length > limits.maximumDatasets || request.models.length > limits.maximumModels || request.suites.length > limits.maximumSuites || checks.length > limits.maximumChecks) invalid("$", "Diagnostic suite exceeds its declared input or source limits", "RESOURCE_LIMIT_EXCEEDED"); distinct(request.datasets.map((item) => item.id), "$.datasets"); distinct(request.models.map((item) => item.id), "$.models"); distinct(request.suites.map((item) => item.id), "$.suites"); distinct(checks.map((item) => item.id), "$.suites"); let rows = 0, evaluationUpperBound = 0; const datasets = new Map(), findings: AnalysisReviewFinding[] = []; for (const [index, dataset] of request.datasets.entries()) { const path = `$.datasets[${index}].input`, subject = { dataset: dataset.id }; if (!checks.some((check) => check.kind !== "model" && check.dataset === dataset.id)) invalid(`$.datasets[${index}].id`, "Remove unreferenced datasets; suites execute only declared subjects", "INVALID_DIAGNOSTIC_SUITE", subject); atSubject(`${path}.policy`, subject, () => parseReviewBoundary(policy, dataset.input.policy)); // Bound the raw dimensions before the existing validator prepares native cells. for (const key of ["losses", "exposures", "expectedCells"] as const) { const items = dataset.input[key]; if (items !== undefined && !Array.isArray(items)) invalid(`${path}.${key}`, "Expected native input rows"); rows += items?.length ?? 0; } if (rows > limits.maximumInputRows) invalid("$.datasets", "Diagnostic suite exceeds its input-row limit", "RESOURCE_LIMIT_EXCEEDED"); const validated = atSubject(path, subject, () => validateDiagnosticRunInput(dataset.input)), definition = validated.definition.definition; const cells = validated.losses.length + validated.exposures.length + (validated.expectedCells?.length ?? 0); const instances = definition.instances.filter((instance) => validated.filter?.instanceIds === undefined || validated.filter.instanceIds.includes(instance.id)); evaluationUpperBound += Math.max(1, cells) * (definition.reviewRules.length + instances.reduce((sum, instance) => sum + 1 + instance.rules.length, 0)); datasets.set(dataset.id, validated); } for (const [index, model] of request.models.entries()) { if (!checks.some((check) => check.kind === "model" && check.model === model.id)) invalid(`$.models[${index}].id`, "Remove unreferenced model subjects", "INVALID_DIAGNOSTIC_SUITE", { model: model.id }); const path = `$.models[${index}].request`, subject = { model: model.id }; const native = atSubject(path, subject, () => parseModelDiagnosticRequest(model.request)), source = native.run.request.source; const eligibility = assessModelDiagnosticEligibility(native), located = subjectFindings(eligibility.findings, path, subject); // No check rows means the shared model boundary refused the request. Unsupported // individual checks remain executable as explicit not-evaluated evidence. if (eligibility.checks.length === 0) throw new AnalysisReviewError(located); findings.push(...located); evaluationUpperBound += Math.max(1, (source?.calendar.development.origins.length ?? 0) * (source?.calendar.development.ages.length ?? 0)) * native.checks.length; } if (!Number.isSafeInteger(evaluationUpperBound) || evaluationUpperBound > limits.maximumEvaluations) invalid("$.limits.maximumEvaluations", "Native diagnostic evaluation upper bound exceeds the declared limit", "RESOURCE_LIMIT_EXCEEDED"); const sourceChecks = new Set(listDiagnosticDataChecks().map((check) => check.id)); for (const { check, path, suite } of locatedChecks) { if (check.kind === "model") { const model = request.models.find((model) => model.id === check.model); if (!model || !model.request.checks.includes(check.check)) invalid(`${path}.${model ? "check" : "model"}`, "Model check is not present in the referenced native diagnostic request", "DIAGNOSTIC_CHECK_UNRESOLVED", { suite, check: check.id, model: check.model }); } else { const dataset = datasets.get(check.dataset), definition = dataset?.definition.definition, coordinates = { suite, check: check.id, dataset: check.dataset }; if (!definition) invalid(`${path}.dataset`, "Dataset reference is unresolved", "DIAGNOSTIC_SOURCE_UNRESOLVED", coordinates); if (check.kind === "data-check") { if (!sourceChecks.has(check.checkId) && !definition.reviewRules.some((rule) => rule.id === check.checkId)) invalid(`${path}.checkId`, "Data check is absent from native source checks and authored review rules", "DIAGNOSTIC_CHECK_UNRESOLVED", coordinates); } else { const instance = definition.instances.find((instance) => instance.id === check.instanceId); if (!instance || check.kind === "metric-rule" && !instance.rules.some((rule) => rule.id === check.ruleId)) invalid(`${path}.${instance ? "ruleId" : "instanceId"}`, "Metric or metric rule is absent from the authored definition", "DIAGNOSTIC_CHECK_UNRESOLVED", coordinates); if (dataset!.filter?.instanceIds !== undefined && !dataset!.filter.instanceIds.includes(check.instanceId)) findings.push({ code: "DIAGNOSTIC_INSTANCE_FILTERED", path: `${path}.instanceId`, coordinates: { ...coordinates, instanceId: check.instanceId }, message: "The referenced dataset's explicit instance filter excludes this selected check", remedies: ["Change the explicit instance filter or retain this check as unavailable evidence"] }); } } } return { request, datasets, evaluationUpperBound, findings }; } /** Compiles native data definitions and validates selected subjects without executing diagnostic tests or model fits. */ export function validateDiagnosticSuiteRequest(input: unknown): DiagnosticSuiteRequest { return prepare(input).request; } /** Internal discovery adapter; validation and execution use this same preparation. */ export function assessDiagnosticSuiteInput(input: unknown): { findings: readonly AnalysisReviewFinding[] } { return { findings: prepare(input).findings }; } function dataEvidence(id: string, input: ValidatedDiagnosticRunInput, version: string): DiagnosticSuiteDataEvidence { const outcome = runValidatedMetricDiagnostics(input), p = outcome.prepared; const content = { dataset: id, status: outcome.status, stage: outcome.status === "completed" ? null : outcome.stage, gate: outcome.gate, definition: p.definition.definition, preparation: { preparationFingerprint: p.preparationFingerprint, filter: p.filter, completePeriodCutoffs: p.completePeriodCutoffs, inputAudit: p.inputAudit, cells: p.cells, exposures: p.exposures, expectedCellsProvided: p.expectedCellsProvided, expectedCells: p.expectedCells, findings: p.findings }, review: outcome.review, metrics: outcome.result }; return { ...content, reference: createAnalysisReference("result", id, version, content) }; } const fixedLimits = ["The suite covers only caller-selected checks; completeness and publication readiness are not certified", "Descriptive patterns and test signals do not establish causal explanations or reserve adequacy", "Source authenticity and comparability between different subjects are not independently established"] as const; const selectedMetricRows = (source: DiagnosticSuiteDataEvidence, instanceId: string): DiagnosticSuiteMetricEvidence[] => (source.metrics?.emergence ?? []).flatMap((point) => { const evaluation = point.metrics[instanceId]; return evaluation === undefined ? [] : [{ coordinate: { group: point.group, sourceGroups: point.sourceGroups, origin: point.origin, valuation: point.valuation, developmentAge: point.developmentAge, ageUnit: point.ageUnit, dimensions: point.dimensions ?? null }, evaluation }]; }); function summarizeCheck(check: DiagnosticSuiteCheck, data: readonly DiagnosticSuiteDataEvidence[], models: DiagnosticSuiteResult["models"]): DiagnosticSuiteCheckResult { const interpretationLimits = [...check.interpretationLimits, ...fixedLimits]; if (check.kind === "model") { const result = models.find((model) => model.id === check.model)!.result, native = result.checks.find((item) => item.kind === check.check)!; return { check, status: native.status, signal: native.signal, reason: native.reason, coverage: { observations: 1, evaluated: native.status === "not-evaluated" ? 0 : 1, notEvaluated: native.status === "not-evaluated" ? 1 : 0 }, interpretationLimits: [...interpretationLimits, ...native.interpretationLimits], evidence: { kind: "model", source: result.reference, native } }; } const source = data.find((source) => source.dataset === check.dataset)!; if (check.kind === "data-check") { const native = source.review.report.checks.find((item) => item.id === check.checkId)!, evaluations = source.review.evaluations.filter((item) => item.ruleId === check.checkId); const configured = source.definition.reviewRules.some((rule) => rule.id === check.checkId), observations = configured ? evaluations.length : 1; const notEvaluated = configured ? evaluations.filter((item) => item.status === "not-evaluated").length : native.status === "not-evaluated" ? 1 : 0; const evaluated = observations - notEvaluated, flagged = native.status === "fail" || native.status === "warning"; return { check, status: observations === 0 || evaluated === 0 ? "not-evaluated" : notEvaluated > 0 ? "qualified" : "evaluated", signal: flagged ? "flagged" : native.status === "not-evaluated" ? "not-evaluated" : "not-flagged", reason: observations === 0 ? "NO_DIAGNOSTIC_COORDINATES" : notEvaluated > 0 ? "DIAGNOSTIC_REVIEW_NOT_EVALUATED" : null, coverage: { observations, evaluated, notEvaluated }, interpretationLimits, evidence: { kind: "data-check", source: source.reference, native, evaluations } }; } const native = selectedMetricRows(source, check.instanceId), evaluations = native.map((row) => row.evaluation); const rules = check.kind === "metric-rule" ? evaluations.map((evaluation) => evaluation.rules.find((rule) => rule.ruleId === check.ruleId)!) : []; const notEvaluated = check.kind === "metric" ? evaluations.filter((evaluation) => evaluation.calculation.value === null).length : rules.filter((rule) => rule.status === "not-evaluated").length; const evaluated = native.length - notEvaluated; const flagged = rules.some((rule) => rule.status === "triggered" || rule.expressionOverflows.length > 0); const qualified = evaluations.some((evaluation: DiagnosticMetricEvaluation) => evaluation.findings.length > 0); const reason = source.metrics === null ? "DIAGNOSTIC_EXECUTION_GATE_BLOCKED" : native.length === 0 ? "NO_DIAGNOSTIC_COORDINATES" : notEvaluated > 0 ? "DIAGNOSTIC_VALUE_UNAVAILABLE" : source.status === "blocked" ? "DIAGNOSTIC_METRIC_GATE_BLOCKED" : null; return { check, status: evaluated === 0 ? "not-evaluated" : notEvaluated > 0 || qualified || source.status === "blocked" ? "qualified" : "evaluated", signal: check.kind === "metric" ? evaluated > 0 ? "descriptive-only" : "not-evaluated" : flagged ? "flagged" : notEvaluated > 0 || native.length === 0 ? "not-evaluated" : "not-flagged", reason, coverage: { observations: native.length, evaluated, notEvaluated }, interpretationLimits, evidence: { kind: check.kind, source: source.reference, native } }; } /** Deterministic SDK coordination. All mathematical checks use existing core engines. */ export function runDiagnosticSuite(input: unknown): DiagnosticSuiteResult { const { request, datasets: inputs, evaluationUpperBound } = prepare(input), datasets: DiagnosticSuiteDataEvidence[] = [], models: DiagnosticSuiteResult["models"][number][] = []; let characters = 0; const retain = (value: unknown) => { characters += canonicalJson(value).length; if (characters > request.limits.maximumOutputCharacters) invalid("$", "Diagnostic suite exceeds its retained evidence limit", "RESOURCE_LIMIT_EXCEEDED"); }; for (const [index, [id, input]] of [...inputs].entries()) { const evidence = atSubject(`$.datasets[${index}].input`, { dataset: id }, () => dataEvidence(id, input, request.version)); retain(evidence); datasets.push(evidence); } for (const [index, model] of request.models.entries()) { const evidence = { id: model.id, result: atSubject(`$.models[${index}].request`, { model: model.id }, () => runModelDiagnostics(model.request)) }; retain(evidence); models.push(evidence); } const suites = request.suites.map((suite) => ({ ...suite, checks: suite.checks.map((check) => summarizeCheck(check, datasets, models)) })), checks = suites.flatMap((suite) => suite.checks); const content = { contractVersion: DIAGNOSTIC_SUITE_CONTRACT_VERSION, sdkVersion: DATA_PACKAGE_VERSION, request, datasets, models, suites, summary: { requested: checks.length, evaluated: checks.filter((check) => check.status === "evaluated").length, qualified: checks.filter((check) => check.status === "qualified").length, notEvaluated: checks.filter((check) => check.status === "not-evaluated").length, flagged: checks.filter((check) => check.signal === "flagged").length }, execution: { evaluationUpperBound }, assurance: { calculations: "existing-native-engines", modelEvidence: "replayed-current-sdk", sourceAuthenticity: "not-verified", crossSourceComparability: "not-established", causality: "not-established", completeness: "caller-selected-checks-only", publication: "not-reported" } as const }; const result = { ...content, reference: createAnalysisReference("result", request.id, request.version, content) }; if (canonicalJson(result).length > request.limits.maximumOutputCharacters) invalid("$", "Diagnostic suite exceeds its complete output evidence limit", "RESOURCE_LIMIT_EXCEEDED"); return snapshotDiagnosticJson(result); } /** Owns the saved envelope and verifies its reference; native calculations remain unverified. */ export function validateDiagnosticSuiteReplayInput(input: unknown): DiagnosticSuiteResult { const raw = parseReviewBoundary(z.object({ contractVersion: z.literal(DIAGNOSTIC_SUITE_CONTRACT_VERSION), sdkVersion: z.literal(DATA_PACKAGE_VERSION), request: diagnosticSuiteRequestSchema, datasets: z.array(z.unknown()), models: z.array(z.unknown()), suites: z.array(z.unknown()), summary: z.unknown(), execution: z.unknown(), assurance: z.unknown(), reference: z.custom(isDiagnosticPlainRecord) }).strict(), input); if (canonicalJson(raw).length > DIAGNOSTIC_SUITE_LIMITS.maximumOutputCharacters) invalid("$", "Saved diagnostic suite exceeds the replay input limit", "RESOURCE_LIMIT_EXCEEDED"); const { reference, ...content } = raw; verifyAnalysisReference(reference, content); return snapshotDiagnosticJson(raw) as unknown as DiagnosticSuiteResult; } /** Header validation alone never certifies retained values; exact native replay remains required. */ export function replayDiagnosticSuite(input: unknown): DiagnosticSuiteResult { const raw = validateDiagnosticSuiteReplayInput(input); const replay = runDiagnosticSuite(raw.request); if (canonicalJson(raw) !== canonicalJson(replay)) invalid("$", "Saved diagnostic suite differs from exact native replay", "DIAGNOSTIC_SUITE_REPLAY_MISMATCH"); return replay; }