/** * Ledger unification: the v1 shapes become the hashed journal (additive, and * resumable after a crash at any point — every write is idempotent or commits * together with the marker that makes a rerun skip it): * * cents backfill — every legacy JournalEntry line gains exact-cent fields * (amountCents/side/counterAccount/buKey) next to the kept float * fields; sub-cent drift beyond the guard aborts loudly. A rerun * derives the same values from the kept floats. * unify journal — standalone legacy Transactions (not referenced by any * JournalEntry) become synthesized two-line entries (legacyImport). * The entry's id derives from the transaction's id, a unique index * admits one entry per `legacyTransactionId`, and the entry and the * transaction's `synthesizedIntoJournal` marker commit in one * transaction. An entry an interrupted 6.x `migrateToV2()` left behind * is adopted only when identical to the derived one; anything else * stops the step. Legacy entries get schemaVersion 2 so aggregations * see them; v1 'reversed' status maps to 'posted' (kept as * legacyStatus) so original+reversal pairs net to zero. * sequence + chain — all schemaVersion-2 entries without a sequenceNumber * are appended to the hash chain per (skrType, fiscalYear) in * deterministic order (postedAt/createdAt/date, then id). Each chain * resumes from its highest chained entry (the journal is authoritative), * and every entry commits together with the LedgerSequence counter and * head it advances. The chained prefix must verify before anything is * appended to it; a counter a 6.x `migrateToV2()` left behind the * journal is taken forward over it, a counter ahead of the journal or a * differing head stops the step. * Every chain the step resumed or wrote to, and every chain holding * imported legacy entries, is verified end to end afterwards. * * Legacy float fields are never removed; the Transaction collection is kept * as a frozen read model. The report is persisted with the migration record * (Verfahrensdokumentation artifact, `SkrApi.getMigrationRecords()`). */ import * as plugins from '../ts/plugins.js'; import * as core from '../ts/core/index.js'; import { JOURNAL_ENTRY_COLLECTION, JOURNAL_LEGACY_TRANSACTION_INDEX_NAME, } from '../ts/skr.classes.journalentry.js'; import { LEDGER_SEQUENCE_COLLECTION } from '../ts/skr.sequence.js'; import { buildHashPayload } from '../ts/skr.posting.js'; import { verifyChainDocuments, type IChainedEntryDocument } from '../ts/skr.verify.js'; import { mapNumericPostingKey } from '../ts/skr.txview.js'; import { LedgerMigrationConflictError } from '../ts/skr.errors.js'; import type { TSKRType } from '../ts/skr.types.js'; import type { ISkrMigrationRunContext, ISkrMigrationStep } from './helpers.migration.js'; import type { ILedgerUnificationReport } from './classes.migrationrecord.js'; const { centsFromFloat } = plugins.calculation; type TRawSession = ReturnType; type TRawDocument = Record; /** * Test-only fault seams: each runs at one crash point of the step and may * throw to simulate a crash there. Production runs pass none. */ export interface ILedgerUnificationFaults { /** inside the import transaction, after the entry is inserted, before the Transaction is marked */ afterSynthesizedInsert?(legacyTransactionId: string): Promise | void; /** inside the chain transaction, after the entry is chained, before the counter advances */ beforeCounterAdvance?(chained: IChainedPosition): Promise | void; /** after the transaction that chained one entry and advanced its counter has committed */ afterChainedEntry?(chained: IChainedPosition): Promise | void; } /** The chain position one entry was given. */ export interface IChainedPosition { skrType: string; fiscalYear: number; sequenceNumber: number; } /** drift guard for legacy float data (plan: abort on drift > 0.5 cent) */ const LEGACY_DRIFT_CENTS = 0.5; /** The id of the journal entry synthesized from one legacy Transaction. */ export const synthesizedEntryId = (legacyTransactionId: string): string => `legacy-tx-${legacyTransactionId}`; function toCents(value: number, driftTracker: { max: number }): number { const exact = value * 100; const rounded = Math.round(exact); const drift = Math.abs(exact - rounded); if (drift > driftTracker.max) driftTracker.max = drift; return centsFromFloat(value, LEGACY_DRIFT_CENTS); } /** Runs `work` in one transaction of a raw session and ends the session. */ const inTransaction = async ( db: plugins.nosqldb.SmartdataDb, work: (session: TRawSession) => Promise, ): Promise => { const session = db.startSession(); try { await session.withTransaction(() => work(session)); } finally { await session.endSession(); } }; const isAbsent = (value: unknown): boolean => value === undefined || value === null; /** Stored-value equality: an absent field equals null, dates compare by instant. */ const sameValue = (a: unknown, b: unknown): boolean => { if (isAbsent(a) || isAbsent(b)) return isAbsent(a) && isAbsent(b); if (a instanceof Date || b instanceof Date) { return a instanceof Date && b instanceof Date && a.getTime() === b.getTime(); } if (Array.isArray(a) || Array.isArray(b)) { return ( Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => sameValue(item, b[index])) ); } if (typeof a === 'object' && typeof b === 'object') { const left = a as TRawDocument; const right = b as TRawDocument; const keys = new Set([...Object.keys(left), ...Object.keys(right)]); return [...keys].every((key) => sameValue(left[key], right[key])); } return a === b; }; /** The fields of `expected` a stored entry does not hold identically (its id aside). */ const differingFields = (stored: TRawDocument, expected: TRawDocument): string[] => Object.keys(expected).filter((key) => key !== 'id' && !sameValue(stored[key], expected[key])); /** Drops undefined members, so the stored document and the derived one compare field by field. */ const withoutUndefined = (document: T): T => Object.fromEntries(Object.entries(document).filter(([, value]) => value !== undefined)) as T; /** The two-line journal entry one standalone legacy Transaction becomes. */ const synthesizeEntry = (doc: TRawDocument, drift: { max: number }): TRawDocument => { const legacyTransactionId = doc.id as string; const amount = (doc.amount as number) ?? 0; const amountCents = (doc.amountCents as number) ?? toCents(amount, drift); const date = doc.date as Date; const description = (doc.description as string) ?? ''; return { id: synthesizedEntryId(legacyTransactionId), journalNumber: `JRN-LEGACY-${doc.transactionNumber ?? legacyTransactionId}`, date, description, reference: (doc.reference as string) ?? '', skrType: doc.skrType as TSKRType, currency: 'EUR', schemaVersion: 2, legacyImport: true, legacyStatus: doc.status, legacyTransactionId, status: doc.status === 'reversed' ? 'posted' : (doc.status as string), period: doc.period as string, fiscalYear: doc.fiscalYear as number, createdAt: (doc.createdAt as Date) ?? date, postedAt: (doc.postedAt as Date) ?? date, createdBy: (doc.createdBy as string) ?? 'migration', totalDebits: amount, totalCredits: amount, isBalanced: true, transactionIds: [], lines: [ withoutUndefined({ accountNumber: doc.debitAccount as string, debit: amount, credit: 0, description, side: 'debit', amountCents, counterAccount: doc.creditAccount as string, buKey: '', buchungstext: description, vatAmountCents: (doc.vatAmountCents as number) ?? 0, costCenter1: (doc.costCenter as string) || undefined, }), { accountNumber: doc.creditAccount as string, debit: 0, credit: amount, description, side: 'credit', amountCents, counterAccount: doc.debitAccount as string, buKey: '', buchungstext: description, }, ], }; }; /** The persisted fields the chain hashes and verifies, read from a stored entry. */ const toChainedEntry = (doc: TRawDocument): IChainedEntryDocument => ({ sequenceNumber: doc.sequenceNumber as number | undefined, prevHash: doc.prevHash as string | undefined, entryHash: doc.entryHash as string | undefined, skrType: doc.skrType as string, fiscalYear: doc.fiscalYear as number, date: doc.date as Date, postedAt: doc.postedAt as Date | undefined, description: doc.description as string, reference: doc.reference as string | undefined, taxScenario: doc.taxScenario as string | undefined, currency: doc.currency as string | undefined, reversalOf: doc.reversalOf as string | undefined, lines: (doc.lines ?? []) as IChainedEntryDocument['lines'], }); const groupKey = (skrType: string, fiscalYear: number): string => JSON.stringify([skrType, fiscalYear]); async function unifyLedger( db: plugins.nosqldb.SmartdataDb, { signal }: ISkrMigrationRunContext, faults: ILedgerUnificationFaults, ): Promise { const entries = db.mongoDb.collection(JOURNAL_ENTRY_COLLECTION); const transactions = db.mongoDb.collection('Transaction'); const sequences = db.mongoDb.collection(LEDGER_SEQUENCE_COLLECTION); const drift = { max: 0 }; const report: ILedgerUnificationReport = { kind: 'ledger-unification', journalEntriesBackfilled: 0, transactionsBackfilled: 0, maxDriftCents: 0, legacyEntriesUpgraded: 0, transactionsSynthesized: 0, synthesizedEntriesAdopted: 0, chains: [], verifiedChains: [], }; // ---------- cents backfill on legacy journal entries ---------- const legacyEntries = await entries .find({ schemaVersion: { $exists: false } }) .toArray(); for (const doc of legacyEntries) { const lines = (doc.lines ?? []) as Array>; const debitLines = lines.filter((line) => ((line.debit as number) ?? 0) > 0); const creditLines = lines.filter((line) => ((line.credit as number) ?? 0) > 0); const counterFor = (isDebit: boolean): string => { const opposite = isDebit ? creditLines : debitLines; return (opposite[0]?.accountNumber as string) ?? ''; }; const upgraded = lines.map((line) => { const isDebit = ((line.debit as number) ?? 0) > 0; const amount = isDebit ? (line.debit as number) : ((line.credit as number) ?? 0); return { ...line, side: line.side ?? (isDebit ? 'debit' : 'credit'), amountCents: line.amountCents ?? toCents(amount, drift), counterAccount: line.counterAccount ?? counterFor(isDebit), buKey: line.buKey ?? mapNumericPostingKey(line.postingKey as number), buchungstext: line.buchungstext ?? line.description ?? doc.description, }; }); signal.throwIfAborted(); await entries.updateOne({ _id: doc._id }, { $set: { lines: upgraded } }); report.journalEntriesBackfilled++; } // cents mirrors on frozen legacy transactions (informational) const legacyTransactions = await transactions .find({ amountCents: { $exists: false } }) .toArray(); for (const doc of legacyTransactions) { signal.throwIfAborted(); await transactions.updateOne( { _id: doc._id }, { $set: { amountCents: toCents((doc.amount as number) ?? 0, drift), vatAmountCents: toCents((doc.vatAmount as number) ?? 0, drift), }, }, ); report.transactionsBackfilled++; } // ---------- unify into the hashed journal ---------- // referenced transactions already live inside a journal entry const referencedIds = new Set(); const entryDocs = await entries.find({}).toArray(); for (const doc of entryDocs) { for (const id of (doc.transactionIds as string[]) ?? []) referencedIds.add(id); } for (const doc of entryDocs) { if (doc.schemaVersion === 2) continue; signal.throwIfAborted(); await entries.updateOne( { _id: doc._id }, { $set: { schemaVersion: 2, currency: doc.currency ?? 'EUR', legacyImport: true, legacyStatus: doc.status, status: doc.status === 'reversed' ? 'posted' : doc.status, }, }, ); report.legacyEntriesUpgraded++; } // one journal entry per legacy transaction: refuse duplicates an earlier // import left behind, then let a unique index hold the rule from here on const imported = await entries .find( { legacyTransactionId: { $exists: true } }, { projection: { _id: 0, id: 1, legacyTransactionId: 1 } }, ) .toArray(); const importedAs = new Map(); for (const doc of imported) { const key = doc.legacyTransactionId as string; importedAs.set(key, [...(importedAs.get(key) ?? []), doc.id as string]); } const duplicateImports = [...importedAs].filter(([, ids]) => ids.length > 1); if (duplicateImports.length > 0) { throw new LedgerMigrationConflictError( duplicateImports.map( ([legacyTransactionId, ids]) => `legacy transaction ${legacyTransactionId} is booked as ${ids.length} journal entries (${ids.join(', ')})`, ), ); } signal.throwIfAborted(); await entries.createIndex( { legacyTransactionId: 1 }, { name: JOURNAL_LEGACY_TRANSACTION_INDEX_NAME, unique: true, partialFilterExpression: { legacyTransactionId: { $exists: true } }, }, ); const standalone = await transactions .find({ synthesizedIntoJournal: { $exists: false } }) .toArray(); for (const doc of standalone) { const legacyTransactionId = doc.id as string; if (referencedIds.has(legacyTransactionId)) continue; const expected = synthesizeEntry(doc, drift); const expectedId = expected.id as string; const existing = await entries.findOne({ legacyTransactionId }); if (existing) { // written by an interrupted 6.x migrateToV2(), which marked the // transaction in a separate write: adopt it only if it is the same booking const differing = differingFields(existing, expected); if (differing.length > 0) { throw new LedgerMigrationConflictError([ `legacy transaction ${legacyTransactionId} is already booked as journal entry ${String(existing.id)}, ` + `which differs in ${differing.join(', ')}`, ]); } signal.throwIfAborted(); const marked = await transactions.updateOne( { _id: doc._id, synthesizedIntoJournal: { $exists: false } }, { $set: { synthesizedIntoJournal: existing.id } }, ); if (marked.matchedCount !== 1) { throw new LedgerMigrationConflictError([ `legacy transaction ${legacyTransactionId} was marked by another writer during the migration`, ]); } report.synthesizedEntriesAdopted++; continue; } const taken = await entries.findOne({ id: expectedId }, { projection: { _id: 0, id: 1 } }); if (taken) { throw new LedgerMigrationConflictError([ `journal entry id ${expectedId} for legacy transaction ${legacyTransactionId} is taken by another entry`, ]); } signal.throwIfAborted(); await inTransaction(db, async (session) => { await entries.insertOne({ ...expected }, { session }); await faults.afterSynthesizedInsert?.(legacyTransactionId); const marked = await transactions.updateOne( { _id: doc._id, synthesizedIntoJournal: { $exists: false } }, { $set: { synthesizedIntoJournal: expectedId } }, { session }, ); if (marked.matchedCount !== 1) { throw new LedgerMigrationConflictError([ `legacy transaction ${legacyTransactionId} was marked by another writer during the migration`, ]); } }); report.transactionsSynthesized++; } report.maxDriftCents = drift.max; // ---------- append unchained entries to the hash chain ---------- const unchained = await entries .find({ schemaVersion: 2, sequenceNumber: { $exists: false } }) .toArray(); const groups = new Map(); const groupOf = (doc: TRawDocument) => { const skrType = doc.skrType as TSKRType; const fiscalYear = doc.fiscalYear as number; const key = groupKey(skrType, fiscalYear); const group = groups.get(key) ?? { skrType, fiscalYear, docs: [] }; groups.set(key, group); return group; }; for (const doc of unchained) { groupOf(doc).docs.push(doc); } // every chain holding imported legacy entries is resumed and verified as // well: a 6.x migrateToV2() interrupted after chaining its entries left the // counter behind the journal without leaving an entry unchained const legacyChains = await entries .find({ schemaVersion: 2, legacyImport: true }, { projection: { _id: 0, skrType: 1, fiscalYear: 1 } }) .toArray(); for (const doc of legacyChains) { groupOf(doc); } /** * Verifies one chain end to end against its stored counter and head — every * schemaVersion-2 entry must be chained — or, where the counter is to be * taken forward over the chained prefix, that prefix against the given state. */ const loadAndVerifyChain = async ( skrType: TSKRType, fiscalYear: number, chainedPrefixState?: { value: number; headHash: string }, ) => { const docs = await entries .find( chainedPrefixState ? { skrType, fiscalYear, schemaVersion: 2, sequenceNumber: { $exists: true } } : { skrType, fiscalYear, schemaVersion: 2 }, ) .sort({ sequenceNumber: 1 }) .toArray(); const state = chainedPrefixState ?? (await sequences.findOne({ skrType, fiscalYear })); return verifyChainDocuments( skrType, fiscalYear, docs.map(toChainedEntry), state ? { value: state.value as number, headHash: state.headHash as string } : null, ); }; for (const { skrType, fiscalYear, docs } of groups.values()) { docs.sort((a, b) => { const timeA = ((a.postedAt ?? a.createdAt ?? a.date) as Date).getTime(); const timeB = ((b.postedAt ?? b.createdAt ?? b.date) as Date).getTime(); if (timeA !== timeB) return timeA - timeB; return String(a.id).localeCompare(String(b.id)); }); // the journal is authoritative: resume after its highest chained entry const [top] = await entries .find({ skrType, fiscalYear, schemaVersion: 2, sequenceNumber: { $exists: true } }) .sort({ sequenceNumber: -1 }) .limit(1) .toArray(); let sequenceNumber = top ? (top.sequenceNumber as number) : 0; let headHash = top ? (top.entryHash as string) : core.genesisHash(skrType, fiscalYear); const state = await sequences.findOne({ skrType, fiscalYear }); const counter = state ? (state.value as number) : null; if (counter !== null && counter > sequenceNumber) { throw new LedgerMigrationConflictError([ `${skrType}/${fiscalYear}: the sequence counter is at ${counter}, ` + `ahead of the journal's highest sequence number ${sequenceNumber}`, ]); } if (counter === sequenceNumber && state?.headHash !== headHash) { throw new LedgerMigrationConflictError([ `${skrType}/${fiscalYear}: the stored chain head differs from the hash of entry ${sequenceNumber}`, ]); } // book nothing onto a chain that does not verify: the chained prefix must // be gapless, and every hash and link must hold up to its highest entry if (sequenceNumber > 0) { const prefix = await loadAndVerifyChain(skrType, fiscalYear, { value: sequenceNumber, headHash }); if (!prefix.valid) { throw new LedgerMigrationConflictError( prefix.issues.map((issue) => `${skrType}/${fiscalYear}: ${issue}`), ); } } if (counter === null || counter < sequenceNumber) { // only a 6.x migrateToV2() interrupted mid-chain leaves the counter // behind the journal: take it forward over the verified prefix signal.throwIfAborted(); const repaired = state === null ? await sequences.updateOne( { skrType, fiscalYear }, { $setOnInsert: { value: sequenceNumber, headHash } }, { upsert: true }, ) : await sequences.updateOne( { _id: state._id, value: counter, headHash: state.headHash }, { $set: { value: sequenceNumber, headHash } }, ); if (state !== null && repaired.matchedCount !== 1) { throw new LedgerMigrationConflictError([ `${skrType}/${fiscalYear}: the sequence counter changed during the migration`, ]); } } let chainedEntries = 0; for (const doc of docs) { const prevHash = headHash; const next = sequenceNumber + 1; const postedAt = (doc.postedAt ?? doc.createdAt ?? doc.date) as Date; const entryHash = core.entryHash( prevHash, buildHashPayload({ ...toChainedEntry(doc), sequenceNumber: next, postedAt }), ); signal.throwIfAborted(); await inTransaction(db, async (session) => { const chained = await entries.updateOne( { _id: doc._id, sequenceNumber: { $exists: false } }, { $set: { sequenceNumber: next, prevHash, entryHash, postedAt } }, { session }, ); await faults.beforeCounterAdvance?.({ skrType, fiscalYear, sequenceNumber: next }); const advanced = await sequences.updateOne( { skrType, fiscalYear, value: next - 1, headHash: prevHash }, { $set: { value: next, headHash: entryHash } }, { session }, ); if (chained.matchedCount !== 1 || advanced.matchedCount !== 1) { throw new LedgerMigrationConflictError([ `${skrType}/${fiscalYear}: entry ${String(doc.id)} or the chain head changed during the migration`, ]); } }); sequenceNumber = next; headHash = entryHash; chainedEntries++; await faults.afterChainedEntry?.({ skrType, fiscalYear, sequenceNumber }); } if (chainedEntries > 0) { report.chains.push({ skrType, fiscalYear, chainedEntries, headHash }); } } // ---------- verify every chain the step resumed or wrote to ---------- const issues: string[] = []; for (const { skrType, fiscalYear } of groups.values()) { const verification = await loadAndVerifyChain(skrType, fiscalYear); issues.push(...verification.issues.map((issue) => `${skrType}/${fiscalYear}: ${issue}`)); report.verifiedChains.push({ skrType, fiscalYear, entries: verification.entriesChecked, headHash: verification.headHash ?? '', }); } if (issues.length > 0) { throw new LedgerMigrationConflictError(issues); } return report; } /** The ledger unification step; tests pass fault seams, production passes none. */ export const createLedgerUnificationStep = (faults: ILedgerUnificationFaults = {}): ISkrMigrationStep => ({ id: 'ledger.1-unified-journal', description: 'Backfill exact cents on legacy entries and transactions, unify them into the journal, ' + 'and append unchained entries to the hash chain', run: (db, context) => unifyLedger(db, context, faults), }); export const ledgerUnificationStep = createLedgerUnificationStep();