import { DiagnosticValidationError, canonicalJson, diagnosticJsonPreflight, isDiagnosticToken, isRealIsoDate, snapshotDiagnosticJson, type HistoricalObservationRecord, type HistoricalScalar, } from "@actuarial-ts/core"; import { z } from "zod"; export interface HistoricalColumnSelector { readonly columns: readonly string[]; } /** Identity parts are required, ordered strings; each part may declare column aliases. */ export type HistoricalIdentitySelector = HistoricalColumnSelector | { readonly kind: "composite"; readonly parts: readonly (HistoricalColumnSelector & { readonly format?: HistoricalDateSelector["format"] })[]; }; export interface HistoricalDateSelector extends HistoricalColumnSelector { readonly format: "iso-yyyy-mm-dd" | "mdy-slash" | "dmy-slash"; } export interface HistoricalNumberSelector extends HistoricalColumnSelector { readonly decimalSeparator: "." | ","; readonly thousandsSeparator: "," | "." | " " | null; readonly negative: "leading-minus" | "parentheses-or-leading-minus"; readonly blank: "null" | "error"; readonly scale: number; } export interface HistoricalDimensionSelector extends HistoricalColumnSelector { readonly type: "string" | "number" | "boolean" | "date"; readonly dateFormat?: HistoricalDateSelector["format"]; readonly numberFormat?: Omit; readonly multipleDelimiter?: string; } export interface HistoricalSourceMapping { readonly id: string; readonly version: string; readonly sourceNamespace: string; readonly artifactId: string; readonly grain: HistoricalObservationRecord["grain"]; readonly completeness: HistoricalObservationRecord["completeness"]; readonly fields: { readonly recordId: HistoricalIdentitySelector; readonly claimId: HistoricalIdentitySelector; readonly componentId?: HistoricalColumnSelector; readonly accidentDate?: HistoricalDateSelector; readonly reportDate?: HistoricalDateSelector; readonly evaluationDate?: HistoricalDateSelector; readonly effectiveDate?: HistoricalDateSelector; readonly status?: HistoricalColumnSelector; }; readonly measures: Readonly>; readonly dimensions?: Readonly>; readonly relationships?: { readonly claimantIds?: HistoricalColumnSelector & { readonly delimiter: string }; readonly occurrenceId?: HistoricalColumnSelector; readonly policyIds?: HistoricalColumnSelector & { readonly delimiter: string }; readonly coverageIds?: HistoricalColumnSelector & { readonly delimiter: string }; }; readonly revision: { readonly id: HistoricalIdentitySelector; readonly action?: HistoricalColumnSelector; readonly sequence?: HistoricalNumberSelector; readonly correctedAt?: HistoricalColumnSelector; }; readonly preserveUnmappedAttributes: boolean; } export interface HistoricalSourceRow { readonly rowNumber: number; readonly values: Readonly>; } export interface HistoricalMappedRowRejection { readonly rowNumber: number; readonly source: { readonly artifactId: string; readonly sourceRow: number }; readonly issues: readonly { readonly path: string; readonly message: string }[]; } export interface HistoricalMappingResult { readonly mappingId: string; readonly mappingVersion: string; readonly sourceNamespace: string; readonly artifactId: string; readonly records: readonly HistoricalObservationRecord[]; readonly rejected: readonly HistoricalMappedRowRejection[]; readonly attributeCatalog: readonly { readonly id: string; readonly sourceColumn: string; readonly observedTypes: readonly ("string" | "number" | "boolean" | "null")[]; }[]; readonly reconciliation: { readonly inputRows: number; readonly acceptedRows: number; readonly rejectedRows: number; }; } const token = z.string().refine(isDiagnosticToken, "Expected a valid nonempty token"); const columnSelector = z.object({ columns: z.array(token).min(1) }).strict(); const identitySelector = z.union([ columnSelector, z.object({ kind: z.literal("composite"), parts: z.array(columnSelector.extend({ format: z.enum(["iso-yyyy-mm-dd", "mdy-slash", "dmy-slash"]).optional(), }).strict()).min(2).max(8), }).strict(), ]); const dateSelector = columnSelector.extend({ format: z.enum(["iso-yyyy-mm-dd", "mdy-slash", "dmy-slash"]), }).strict(); const numberFormat = { decimalSeparator: z.enum([".", ","]), thousandsSeparator: z.enum([",", ".", " "]).nullable(), negative: z.enum(["leading-minus", "parentheses-or-leading-minus"]), blank: z.enum(["null", "error"]), scale: z.number().finite().positive(), }; const numberSelector = columnSelector.extend(numberFormat).strict().superRefine((value, context) => { if (value.thousandsSeparator === value.decimalSeparator) context.addIssue({ code: z.ZodIssueCode.custom, path: ["thousandsSeparator"], message: "Thousands and decimal separators must differ", }); }); const dimensionSelector = columnSelector .extend({ type: z.enum(["string", "number", "boolean", "date"]), dateFormat: z.enum(["iso-yyyy-mm-dd", "mdy-slash", "dmy-slash"]).optional(), numberFormat: z.object(numberFormat).strict().optional(), multipleDelimiter: z.string().min(1).optional(), }) .strict() .superRefine((value, context) => { if (value.type === "date" && value.dateFormat === undefined) context.addIssue({ code: z.ZodIssueCode.custom, path: ["dateFormat"], message: "Date dimension requires dateFormat" }); if (value.type === "number" && value.numberFormat === undefined) context.addIssue({ code: z.ZodIssueCode.custom, path: ["numberFormat"], message: "Number dimension requires numberFormat" }); if (value.multipleDelimiter !== undefined && value.type !== "string") context.addIssue({ code: z.ZodIssueCode.custom, path: ["multipleDelimiter"], message: "Only string dimensions support multipleDelimiter", }); }); const multiSelector = columnSelector.extend({ delimiter: z.string().min(1) }).strict(); export const historicalSourceMappingSchema = z .object({ id: token, version: token, sourceNamespace: token, artifactId: token, grain: z.enum(["claim-snapshot", "claim-component-snapshot", "transaction"]), completeness: z.enum(["complete-snapshot", "partial-snapshot", "change-only"]), fields: z .object({ recordId: identitySelector, claimId: identitySelector, componentId: columnSelector.optional(), accidentDate: dateSelector.optional(), reportDate: dateSelector.optional(), evaluationDate: dateSelector.optional(), effectiveDate: dateSelector.optional(), status: columnSelector.optional(), }) .strict(), measures: z.record(token, numberSelector), dimensions: z.record(token, dimensionSelector).optional(), relationships: z .object({ claimantIds: multiSelector.optional(), occurrenceId: columnSelector.optional(), policyIds: multiSelector.optional(), coverageIds: multiSelector.optional(), }) .strict() .optional(), revision: z .object({ id: identitySelector, action: columnSelector.optional(), sequence: numberSelector.optional(), correctedAt: columnSelector.optional(), }) .strict(), preserveUnmappedAttributes: z.boolean(), }) .strict() .superRefine((mapping, context) => { if (mapping.grain === "claim-component-snapshot" && mapping.fields.componentId === undefined) context.addIssue({ code: z.ZodIssueCode.custom, path: ["fields", "componentId"], message: "Component grain requires componentId mapping" }); if (mapping.grain === "transaction" && mapping.fields.effectiveDate === undefined) context.addIssue({ code: z.ZodIssueCode.custom, path: ["fields", "effectiveDate"], message: "Transaction grain requires effectiveDate mapping" }); if (mapping.grain !== "transaction" && mapping.fields.evaluationDate === undefined) context.addIssue({ code: z.ZodIssueCode.custom, path: ["fields", "evaluationDate"], message: "Snapshot grain requires evaluationDate mapping" }); if (mapping.grain === "transaction" && mapping.completeness !== "change-only") context.addIssue({ code: z.ZodIssueCode.custom, path: ["completeness"], message: "Transaction grain requires change-only completeness" }); if (Object.keys(mapping.measures).length === 0) context.addIssue({ code: z.ZodIssueCode.custom, path: ["measures"], message: "At least one measure mapping is required" }); }); function mappingError(error: z.ZodError): DiagnosticValidationError { return new DiagnosticValidationError( error.issues.map((issue) => ({ domain: "configuration" as const, code: issue.code === z.ZodIssueCode.unrecognized_keys ? ("unknown-key" as const) : ("invalid-configuration" as const), path: `$${issue.path.map((part) => (typeof part === "number" ? `[${part}]` : `.${part}`)).join("")}`, message: issue.message, })), ); } export function parseHistoricalSourceMapping(value: unknown): HistoricalSourceMapping { const parsed = historicalSourceMappingSchema.safeParse(value); if (!parsed.success) throw mappingError(parsed.error); return snapshotDiagnosticJson(parsed.data) as HistoricalSourceMapping; } function scalarText(value: HistoricalScalar): string { return value === null ? "" : typeof value === "string" ? value.trim() : String(value); } function selectedValue( row: HistoricalSourceRow, selector: HistoricalColumnSelector, path: string, issues: { path: string; message: string }[], required = true, ): HistoricalScalar | undefined { const selected = selector.columns .filter((column) => Object.hasOwn(row.values, column)) .map((column) => ({ column, value: row.values[column]! })) .filter(({ value }) => scalarText(value) !== ""); if (selected.length === 0) { if (required) issues.push({ path, message: `No value found in columns ${selector.columns.join(", ")}` }); return undefined; } if (new Set(selected.map(({ value }) => scalarText(value))).size > 1) { issues.push({ path, message: `Alias columns disagree: ${selected.map(({ column }) => column).join(", ")}` }); return undefined; } return selected[0]!.value; } function parseDateValue( value: HistoricalScalar | undefined, format: HistoricalDateSelector["format"], path: string, issues: { path: string; message: string }[], ): string | undefined { if (value === undefined) return undefined; const text = scalarText(value); let result = text; const match = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/.exec(text); if (format !== "iso-yyyy-mm-dd") { if (!match) { issues.push({ path, message: `Expected ${format} date` }); return undefined; } const month = format === "mdy-slash" ? match[1]! : match[2]!; const day = format === "mdy-slash" ? match[2]! : match[1]!; result = `${match[3]}-${month.padStart(2, "0")}-${day.padStart(2, "0")}`; } if (!isRealIsoDate(result)) { issues.push({ path, message: `Expected a real date; got ${text}` }); return undefined; } return result; } function parseNumberValue( value: HistoricalScalar | undefined, selector: Omit, path: string, issues: { path: string; message: string }[], ): number | null | undefined { if (value === undefined || scalarText(value) === "") { if (selector.blank === "error") issues.push({ path, message: "Blank numeric value is not allowed" }); return selector.blank === "null" ? null : undefined; } if (typeof value === "number") { if (!Number.isFinite(value)) { issues.push({ path, message: "Numeric value is not finite" }); return undefined; } const result = value * selector.scale; if (!Number.isFinite(result)) { issues.push({ path, message: "Scaled numeric value is not finite" }); return undefined; } return Object.is(result, -0) ? 0 : result; } let text = scalarText(value); let negative = false; if (/^\(.*\)$/.test(text)) { if (selector.negative !== "parentheses-or-leading-minus") { issues.push({ path, message: "Parenthesized negative is not allowed" }); return undefined; } negative = true; text = text.slice(1, -1); } if (selector.thousandsSeparator !== null) text = text.split(selector.thousandsSeparator).join(""); if (selector.decimalSeparator === ",") text = text.replace(",", "."); if (!/^-?\d+(?:\.\d+)?$/.test(text)) { issues.push({ path, message: `Invalid number ${scalarText(value)} for declared format` }); return undefined; } const result = Number(text) * selector.scale * (negative ? -1 : 1); if (!Number.isFinite(result)) { issues.push({ path, message: "Numeric value is not finite" }); return undefined; } return Object.is(result, -0) ? 0 : result; } function splitIds(value: HistoricalScalar | undefined, delimiter: string): string[] | undefined { if (value === undefined) return undefined; const values = scalarText(value).split(delimiter).map((item) => item.trim()).filter(Boolean); return values.length === 0 ? undefined : [...new Set(values)]; } function selectorColumns(mapping: HistoricalSourceMapping): Set { const result = new Set(); const add = (selector: HistoricalIdentitySelector | undefined) => { if (selector === undefined) return; for (const part of "parts" in selector ? selector.parts : [selector]) part.columns.forEach((column) => result.add(column)); }; Object.values(mapping.fields).forEach(add); Object.values(mapping.measures).forEach(add); Object.values(mapping.dimensions ?? {}).forEach(add); Object.values(mapping.relationships ?? {}).forEach(add); Object.values(mapping.revision).forEach(add); return result; } export function mapHistoricalSourceRows( mappingInput: HistoricalSourceMapping | unknown, rowsInput: readonly HistoricalSourceRow[], ): HistoricalMappingResult { const mapping = parseHistoricalSourceMapping(mappingInput); const records: HistoricalObservationRecord[] = []; const rejected: HistoricalMappedRowRejection[] = []; const usedColumns = selectorColumns(mapping); const attributeTypes = new Map>(); for (let rowIndex = 0; rowIndex < rowsInput.length; rowIndex += 1) { const raw = rowsInput[rowIndex]!; const preflight = diagnosticJsonPreflight(raw, "input"); if (preflight.length > 0) { const candidate = (raw as { readonly rowNumber?: unknown }).rowNumber; const rowNumber = typeof candidate === "number" && Number.isSafeInteger(candidate) && candidate > 0 ? candidate : rowIndex + 1; rejected.push({ rowNumber, source: { artifactId: mapping.artifactId, sourceRow: rowNumber }, issues: preflight.map((issue) => ({ path: issue.path, message: issue.message })), }); continue; } const row = snapshotDiagnosticJson(raw); const issues: { path: string; message: string }[] = []; const pendingAttributeTypes = new Map>(); if (!Number.isSafeInteger(row.rowNumber) || row.rowNumber < 1) issues.push({ path: "$.rowNumber", message: "rowNumber must be a positive safe integer" }); const field = (selector: HistoricalColumnSelector | undefined, name: string, required = false) => selector === undefined ? undefined : selectedValue(row, selector, `$.${name}`, issues, required); const identity = (selector: HistoricalIdentitySelector, name: string) => { if (!("parts" in selector)) return scalarText(field(selector, name, true) ?? ""); const parts = selector.parts.map((part, index) => { const partName = `${name}.parts[${index}]`; const value = field(part, partName, true); return part.format === undefined ? scalarText(value ?? "") : parseDateValue(value, part.format, `$.${partName}`, issues) ?? ""; }); return `tuple/1:${canonicalJson(parts)}`; }; const recordId = identity(mapping.fields.recordId, "recordId"); const claimId = identity(mapping.fields.claimId, "claimId"); const componentIdValue = field(mapping.fields.componentId, "componentId", mapping.grain === "claim-component-snapshot"); const statusValue = field(mapping.fields.status, "status"); const mappedDate = (selector: HistoricalDateSelector | undefined, name: string, required = false) => { if (selector === undefined) return undefined; return parseDateValue(field(selector, name, required), selector.format, `$.${name}`, issues); }; const accidentDate = mappedDate(mapping.fields.accidentDate, "accidentDate"); const reportDate = mappedDate(mapping.fields.reportDate, "reportDate"); const evaluationDate = mappedDate( mapping.fields.evaluationDate, "evaluationDate", mapping.grain !== "transaction", ); const effectiveDate = mappedDate( mapping.fields.effectiveDate, "effectiveDate", mapping.grain === "transaction", ); const measures: Record = Object.create(null); for (const [id, selector] of Object.entries(mapping.measures)) { const value = selectedValue(row, selector, `$.measures.${id}`, issues, selector.blank === "error"); const amount = parseNumberValue(value, selector, `$.measures.${id}`, issues); if (amount !== undefined) measures[id] = amount; } const dimensions: Record = Object.create(null); for (const [id, selector] of Object.entries(mapping.dimensions ?? {})) { const value = selectedValue(row, selector, `$.dimensions.${id}`, issues, false); if (value === undefined) continue; if (selector.multipleDelimiter !== undefined) { dimensions[id] = splitIds(value, selector.multipleDelimiter) ?? []; } else if (selector.type === "date") { const parsed = parseDateValue(value, selector.dateFormat!, `$.dimensions.${id}`, issues); if (parsed !== undefined) dimensions[id] = parsed; } else if (selector.type === "number") { const parsed = parseNumberValue(value, selector.numberFormat!, `$.dimensions.${id}`, issues); if (parsed !== undefined) dimensions[id] = parsed; } else if (selector.type === "boolean") { const text = scalarText(value).toLowerCase(); if (text === "true" || text === "1") dimensions[id] = true; else if (text === "false" || text === "0") dimensions[id] = false; else issues.push({ path: `$.dimensions.${id}`, message: "Expected boolean true/false/1/0" }); } else dimensions[id] = scalarText(value); } if (mapping.preserveUnmappedAttributes) for (const [column, value] of Object.entries(row.values)) { if (usedColumns.has(column)) continue; const id = `source:${mapping.sourceNamespace}:${column}`; dimensions[id] = value; const type = value === null ? "null" : typeof value; if (type === "string" || type === "number" || type === "boolean" || type === "null") { const current = pendingAttributeTypes.get(id) ?? new Set(); current.add(type); pendingAttributeTypes.set(id, current); } } const relationship = mapping.relationships; const occurrence = scalarText(field(relationship?.occurrenceId, "relationships.occurrenceId") ?? ""); const claimantIds = relationship?.claimantIds ? splitIds(field(relationship.claimantIds, "relationships.claimantIds"), relationship.claimantIds.delimiter) : undefined; const policyIds = relationship?.policyIds ? splitIds(field(relationship.policyIds, "relationships.policyIds"), relationship.policyIds.delimiter) : undefined; const coverageIds = relationship?.coverageIds ? splitIds(field(relationship.coverageIds, "relationships.coverageIds"), relationship.coverageIds.delimiter) : undefined; const revisionId = identity(mapping.revision.id, "revision.id"); const actionText = scalarText(field(mapping.revision.action, "revision.action") ?? "upsert").toLowerCase(); if (actionText !== "upsert" && actionText !== "delete") issues.push({ path: "$.revision.action", message: "Revision action must be upsert or delete" }); const sequenceValue = mapping.revision.sequence ? parseNumberValue( field(mapping.revision.sequence, "revision.sequence", true), mapping.revision.sequence, "$.revision.sequence", issues, ) : undefined; if (typeof sequenceValue === "number" && !Number.isSafeInteger(sequenceValue)) issues.push({ path: "$.revision.sequence", message: "Revision sequence must be a safe integer" }); const correctedAtValue = field(mapping.revision.correctedAt, "revision.correctedAt"); const correctedAt = correctedAtValue === undefined ? undefined : scalarText(correctedAtValue); if (correctedAt !== undefined && !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.test(correctedAt)) issues.push({ path: "$.revision.correctedAt", message: "correctedAt must be an ISO timestamp with offset" }); if (!isDiagnosticToken(recordId)) issues.push({ path: "$.recordId", message: "recordId is invalid" }); if (!isDiagnosticToken(claimId)) issues.push({ path: "$.claimId", message: "claimId is invalid" }); if (!isDiagnosticToken(revisionId)) issues.push({ path: "$.revision.id", message: "revision id is invalid" }); if (issues.length > 0) { rejected.push({ rowNumber: row.rowNumber, source: { artifactId: mapping.artifactId, sourceRow: row.rowNumber }, issues, }); continue; } for (const [id, types] of pendingAttributeTypes) { const current = attributeTypes.get(id) ?? new Set(); types.forEach((type) => current.add(type)); attributeTypes.set(id, current); } const relationships = { ...(claimantIds === undefined ? {} : { claimantIds }), ...(occurrence === "" ? {} : { occurrenceId: occurrence }), ...(policyIds === undefined ? {} : { policyIds }), ...(coverageIds === undefined ? {} : { coverageIds }), }; records.push({ recordId, sourceNamespace: mapping.sourceNamespace, claimId, grain: mapping.grain, ...(componentIdValue === undefined ? {} : { componentId: scalarText(componentIdValue) }), ...(accidentDate === undefined ? {} : { accidentDate }), ...(reportDate === undefined ? {} : { reportDate }), ...(evaluationDate === undefined ? {} : { evaluationDate }), ...(effectiveDate === undefined ? {} : { effectiveDate }), measures: actionText === "delete" ? {} : measures, ...(statusValue === undefined ? {} : { status: scalarText(statusValue) }), ...(Object.keys(dimensions).length === 0 ? {} : { dimensions }), ...(Object.keys(relationships).length === 0 ? {} : { relationships }), completeness: mapping.completeness, revision: { id: revisionId, action: actionText as "upsert" | "delete", ...(typeof sequenceValue !== "number" ? {} : { sequence: sequenceValue }), ...(correctedAt === undefined ? {} : { correctedAt }), }, source: { artifactId: mapping.artifactId, sourceRow: row.rowNumber }, }); } records.sort((left, right) => left.recordId === right.recordId ? 0 : left.recordId < right.recordId ? -1 : 1, ); rejected.sort((left, right) => left.rowNumber - right.rowNumber); return snapshotDiagnosticJson({ mappingId: mapping.id, mappingVersion: mapping.version, sourceNamespace: mapping.sourceNamespace, artifactId: mapping.artifactId, records, rejected, attributeCatalog: [...attributeTypes.entries()] .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) .map(([id, types]) => ({ id, sourceColumn: id.slice(`source:${mapping.sourceNamespace}:`.length), observedTypes: [...types].sort(), })), reconciliation: { inputRows: rowsInput.length, acceptedRows: records.length, rejectedRows: rejected.length, }, }); }