import { sdkLogger as logger } from "../../services/reporting-service/logger.js"; import { MMApiClient } from "../../services/mm-api-service/mm-api-service.js"; import type { Checkpoint } from "../../services/mm-api-service/types/checkpoint.js"; import { MMReceiveTimesheet } from "../../services/mm-api-service/types/receive-types.js"; import type { IERPTimesheetHandler } from "../../types/erp-connector.js"; import { summarizeError } from "../error-utils.js"; import { getRuntimeStateJson, setRuntimeStateJson, } from "../local-data-store/sdk-runtime-state-db.js"; const MAX_PASSES = 10; /** * A row that has been failing for longer than this is dropped from the retry list, but only in a * run where another row succeeded (the ERP is reachable, so the row itself is the problem). It is * logged as an error and not retried again unless it changes in MM. The clock starts at the row's * first failure, so a backlog row after a long outage gets the full window. */ export const TIMESHEET_FAILURE_GIVE_UP_MS = 24 * 60 * 60 * 1000; type TimesheetOutcome = "success" | "failure" | "pending"; /** A row carried to the next cycle in the checkpoint value. */ export type TimesheetRetryEntry = { timesheet: MMReceiveTimesheet; /** When this version of the row first failed (host clock); null if it has not been attempted. */ firstFailedAt: string | null; attempts: number; }; /** What is stored in MM's timesheet export checkpoint. */ export type TimesheetCheckpointValue = { timestamp: string; failed: TimesheetRetryEntry[]; }; /** Parses MM's `updatedAt`; null when missing or unparseable. */ function updatedAtMs(timesheet: MMReceiveTimesheet): number | null { if (!timesheet.updatedAt) return null; const ms = Date.parse(timesheet.updatedAt); return Number.isNaN(ms) ? null : ms; } /** Digits past milliseconds (MM sends microseconds), padded so they compare as strings. */ function subMsDigits(updatedAt: string): string { const fraction = /T[\d:]+\.(\d+)/.exec(updatedAt)?.[1] ?? ""; return fraction.padEnd(9, "0").slice(3, 9); } /** * Orders two parseable `updatedAt` values at full precision. `Date` keeps only milliseconds, so * ties are broken on the remaining digits (offsets are whole minutes and do not affect them). */ export function compareUpdatedAt(a: string, b: string): number { const msDiff = Date.parse(a) - Date.parse(b); if (msDiff !== 0) return msDiff; const subA = subMsDigits(a); const subB = subMsDigits(b); return subA === subB ? 0 : subA < subB ? -1 : 1; } /** In-run identity: the same row updated in MM mid-run (e.g. clocked out) is a new attempt. */ function attemptKey(timesheet: MMReceiveTimesheet): string { return `${String(timesheet.timesheetRef)}|${timesheet.updatedAt ?? ""}`; } function byRef(a: MMReceiveTimesheet, b: MMReceiveTimesheet): number { return Number(a.timesheetRef) - Number(b.timesheetRef); } /** * Validates the retry list read back from the checkpoint. Throws on anything unexpected rather * than dropping rows silently; only this synchronizer writes the value. */ export function parseTimesheetRetries(value: unknown): TimesheetRetryEntry[] { if (value === undefined || value === null) return []; if (!Array.isArray(value)) { throw new Error("Timesheet export checkpoint has a malformed `failed` list (not an array)"); } return value.map((entry: unknown, index) => { const e = entry as Partial> | null; const timesheet = e?.timesheet as Partial | null | undefined; if ( !timesheet || typeof timesheet !== "object" || (typeof timesheet.timesheetRef !== "number" && typeof timesheet.timesheetRef !== "string") || typeof e?.attempts !== "number" || (e.firstFailedAt !== null && typeof e.firstFailedAt !== "string") ) { throw new Error(`Timesheet export checkpoint has a malformed \`failed\` entry at index ${index}`); } return { timesheet: new MMReceiveTimesheet(timesheet), firstFailedAt: e.firstFailedAt, attempts: e.attempts, }; }); } /** * Decides which rows go on the next cycle's retry list and which are given up on. * * Only rows that did not succeed are kept: rows that succeeded are behind the checkpoint and never * delivered again. A row keeps its first-failure time and attempt count for as long as the same * version of it is retried; a row that changed in MM starts over. * * When nothing succeeded (the ERP is likely unreachable), only rows that were already on the list * are kept, and the caller leaves the checkpoint where it was so this run's fetched rows come back * next cycle. That keeps the list from growing through a long outage. */ export function planTimesheetRetries(options: { latestByRef: ReadonlyMap; carried: readonly TimesheetRetryEntry[]; anySucceeded: boolean; nowMs: number; }): { failed: TimesheetRetryEntry[]; givenUp: TimesheetRetryEntry[] } { const { latestByRef, carried, anySucceeded, nowMs } = options; const carriedByKey = new Map(carried.map((entry) => [attemptKey(entry.timesheet), entry])); const failed: TimesheetRetryEntry[] = []; const givenUp: TimesheetRetryEntry[] = []; for (const { timesheet, outcome } of latestByRef.values()) { if (outcome === "success") continue; const prior = carriedByKey.get(attemptKey(timesheet)); if (!anySucceeded && !prior) continue; const attempted = outcome === "failure"; const entry: TimesheetRetryEntry = { timesheet, firstFailedAt: prior?.firstFailedAt ?? (attempted ? new Date(nowMs).toISOString() : null), attempts: (prior?.attempts ?? 0) + (attempted ? 1 : 0), }; const failingMs = entry.firstFailedAt === null ? 0 : nowMs - Date.parse(entry.firstFailedAt); if (attempted && anySucceeded && failingMs > TIMESHEET_FAILURE_GIVE_UP_MS) { givenUp.push(entry); } else { failed.push(entry); } } failed.sort((a, b) => byRef(a.timesheet, b.timesheet)); return { failed, givenUp }; } /** Local copy of a checkpoint MM did not accept, saved to MM before the next cycle fetches. */ function pendingCheckpointKey(connectorType: string): string { return `timesheetExportPendingCheckpoint:${connectorType}`; } /** * Synchronizes timesheet rows (clock in/out) from MachineMetrics to an ERP system. * * The checkpoint advances past every row fetched, so a row that succeeded is not delivered again. * Rows that failed are stored alongside the checkpoint timestamp in MM and retried on the next * cycle, in `timesheetRef` order with new rows. A person's later rows wait behind their earliest * unfinished row, so the ERP sees each person's clock events in order (see * {@link planTimesheetRetries} and {@link TIMESHEET_FAILURE_GIVE_UP_MS}). * * If MM rejects the checkpoint save, the value is kept in the local SQLite runtime state and saved * to MM at the start of the next cycle, before anything is fetched. Only a container recreate * between the two loses it, and rows since the last saved checkpoint are then delivered again. * * Callers should run labor ticket sync to completion before invoking this, so shop-floor labor * is written/closed before employee-level clock-out. */ export class TimesheetERPSynchronizer { static async syncToERP( connectorType: string, handler: IERPTimesheetHandler ): Promise { const mmApiClient = new MMApiClient(); const checkpointRef: Omit = { system: connectorType, table: "timesheets", checkpointType: "export", }; const pendingKey = pendingCheckpointKey(connectorType); try { const pending = getRuntimeStateJson(pendingKey); if (pending) { await mmApiClient.saveCheckpoint({ ...checkpointRef, checkpointValue: pending }); setRuntimeStateJson(pendingKey, null); logger.info("syncTimesheetsToERP: Saved the checkpoint left over from a failed save", { timestamp: pending.timestamp, failed: pending.failed.length, }); } const stored = (await mmApiClient.getCheckpoint({ ...checkpointRef })) as { timestamp?: unknown; failed?: unknown; }; let previousTimestamp: string; let carried: TimesheetRetryEntry[]; if (typeof stored.timestamp === "string" && stored.timestamp) { previousTimestamp = stored.timestamp; carried = parseTimesheetRetries(stored.failed); } else { previousTimestamp = new Date().toISOString(); carried = []; await mmApiClient.saveCheckpoint({ ...checkpointRef, checkpointValue: { timestamp: previousTimestamp, failed: [] }, }); logger.info("syncTimesheetsToERP: No checkpoint found, initialized it", { timestamp: previousTimestamp, }); } const attemptedKeys = new Set(); const latestByRef = new Map< string, { timesheet: MMReceiveTimesheet; outcome: TimesheetOutcome } >(); let maxSeenUpdatedAt: string | null = null; let processed = 0; let failed = 0; let waiting = 0; let attempt = 0; let refetchError: unknown = undefined; const fetchPending = async (): Promise => { const rows = await mmApiClient.fetchTimesheetUpdates({ system: connectorType, checkpointType: "export", }); for (const row of rows) { if ( row.updatedAt && updatedAtMs(row) !== null && (maxSeenUpdatedAt === null || compareUpdatedAt(row.updatedAt, maxSeenUpdatedAt) > 0) ) { maxSeenUpdatedAt = row.updatedAt; } } return rows.filter((row) => !attemptedKeys.has(attemptKey(row))).sort(byRef); }; // An earlier row of the same person has not gone through yet, so this one must wait behind it. const waitsBehindEarlierRow = (timesheet: MMReceiveTimesheet): boolean => { for (const other of latestByRef.values()) { if ( other.outcome !== "success" && String(other.timesheet.personId) === String(timesheet.personId) && Number(other.timesheet.timesheetRef) < Number(timesheet.timesheetRef) ) { return true; } } return false; }; const fetched = await fetchPending(); // A fetched row supersedes the carried version of the same row: it holds the row's latest state. const fetchedRefs = new Set(fetched.map((row) => String(row.timesheetRef))); const retries = carried .map((entry) => entry.timesheet) .filter((row) => !fetchedRefs.has(String(row.timesheetRef))); if (retries.length > 0) { logger.info("syncTimesheetsToERP: Retrying timesheets that did not go through earlier", { timesheetRefs: retries.map((t) => t.timesheetRef), }); } // One ordering for both, so a changed row comes before the same person's later rows. let timesheetsPendingUpdate = [...retries, ...fetched].sort(byRef); while (timesheetsPendingUpdate.length > 0) { attempt += 1; if (attempt > MAX_PASSES) { logger.warn( `syncTimesheetsToERP: Stopping after ${MAX_PASSES} passes; remaining pending=${timesheetsPendingUpdate.length}` ); for (const timesheet of timesheetsPendingUpdate) { latestByRef.set(String(timesheet.timesheetRef), { timesheet, outcome: "pending", }); } break; } for (const timesheet of timesheetsPendingUpdate) { attemptedKeys.add(attemptKey(timesheet)); if (waitsBehindEarlierRow(timesheet)) { waiting += 1; latestByRef.set(String(timesheet.timesheetRef), { timesheet, outcome: "pending", }); continue; } try { await handler.processTimesheetInERP(timesheet); processed += 1; latestByRef.set(String(timesheet.timesheetRef), { timesheet, outcome: "success", }); } catch (error: unknown) { logger.error("syncTimesheetsToERP: Failed to process timesheet", { timesheetRef: timesheet.timesheetRef, error: summarizeError(error), }); failed += 1; latestByRef.set(String(timesheet.timesheetRef), { timesheet, outcome: "failure", }); } } try { timesheetsPendingUpdate = await fetchPending(); } catch (error: unknown) { // Save what went through before surfacing the error, so it is not delivered again. refetchError = error; break; } } const anySucceeded = processed > 0; const plan = planTimesheetRetries({ latestByRef, carried, anySucceeded, nowMs: Date.now(), }); for (const { timesheet, firstFailedAt, attempts } of plan.givenUp) { logger.error( "syncTimesheetsToERP: Giving up on timesheet that keeps failing; it will not be retried unless it changes in MM", { timesheetRef: timesheet.timesheetRef, personId: timesheet.personId, updatedAt: timesheet.updatedAt, firstFailedAt, attempts, } ); } if (latestByRef.size > 0) { const checkpointValue: TimesheetCheckpointValue = { timestamp: anySucceeded ? (maxSeenUpdatedAt ?? previousTimestamp) : previousTimestamp, failed: plan.failed, }; try { await mmApiClient.saveCheckpoint({ ...checkpointRef, checkpointValue }); } catch (error: unknown) { setRuntimeStateJson(pendingKey, checkpointValue); logger.error( "syncTimesheetsToERP: Could not save the checkpoint to MM; kept it locally to save next cycle", { error: summarizeError(error) } ); throw error; } } logger.info("syncTimesheetsToERP: Completed", { processed, failed, waiting, retryNextCycle: plan.failed.length, givenUp: plan.givenUp.length, }); if (refetchError !== undefined) throw refetchError; } finally { await mmApiClient.destroy(); } } }