import { ERPObjType, MMWorkOrderOperationStatus, MMWorkOrderStatus, normalizeMMStatus, } from "../../types/erp-types.js"; import type { ErpWorkOrderKey, ErpWorkOrderOperationKey, IERPWorkOrderReconciliationHandler, } from "../../types/erp-connector.js"; import { MMApiClient } from "../../services/mm-api-service/mm-api-service.js"; import type { MMReceiveWorkOrder, MMReceiveWorkOrderOperation, } from "../../services/mm-api-service/types/receive-types.js"; import { MMSendWorkOrderOperationStatusPatch, MMSendWorkOrderStatusPatch, } from "../../services/mm-api-service/types/send-types.js"; import { BatchCacheManager } from "../../services/caching-service/batch-cache-manager.js"; import { MMEntityProcessor } from "./mm-entity-processor.js"; import type { WriteEntitiesToMMResult } from "./standard-process-drivers.js"; import { getEnvBoolean, getEnvFloat, getEnvNumber } from "../env.js"; import { sdkLogger as logger } from "../../services/reporting-service/logger.js"; /** * MM statuses eligible for deletion reconciliation, for work orders and work order * operations alike. Only rows in these statuses are compared against the ERP; everything * else (COMPLETED, CANCELLED, already DELETED, etc.) is left untouched. Values are in the * canonical form produced by {@link normalizeMMStatus}. */ export const RECONCILE_CANDIDATE_STATUSES: ReadonlySet = new Set([ MMWorkOrderStatus.RELEASED, MMWorkOrderStatus.ACTIVE, ]); export interface ReconcileDeletedWorkOrdersOptions { /** * Abort when more than this fraction of candidates would be deleted * (guard against a bad/partial ERP fetch mass-deleting jobs). Evaluated separately * for the work order set and the work order operation set. * Env fallback `RECONCILE_MAX_DELETE_FRACTION`; default 0.2. */ maxDeleteFraction?: number; /** * The fraction guard only applies when the deletion count exceeds this floor, * so small shops aren't blocked from legitimately deleting a handful of jobs. * Env fallback `RECONCILE_DELETE_GUARD_MIN_COUNT`; default 10. */ deleteGuardMinCount?: number; /** * Log what would be marked DELETED without writing to MM. * Env fallback `RECONCILE_WORK_ORDERS_DRY_RUN`; default false. */ dryRun?: boolean; /** Rows per MM GraphQL page when fetching work orders / operations (default 500). */ pageSize?: number; } export type ReconcileDeletedWorkOrdersStatus = | "completed" // deletions (possibly zero) written to MM | "dry-run"; // deletions identified and logged, nothing written export interface ReconcileDeletedWorkOrdersResult { status: ReconcileDeletedWorkOrdersStatus; /** * Only set on "dry-run" results whose deletion set would have tripped the * blast-radius guard on a real run; absent otherwise. */ reason?: string; /** * MM work orders in a candidate status (RELEASED / ACTIVE, case-insensitive). */ candidateCount: number; /** Composite keys identified for deletion. */ workOrdersToDelete: ErpWorkOrderKey[]; /** * Size of the work order deletion set applied to MM (0 unless status is "completed"). * Counts rows the dedupe caches skipped as already up to date too — the MM write * throws on any failure, so when this is returned the whole set is DELETED in MM. */ markedDeletedCount: number; /** * Whether the handler implements `selectWorkOrderOperationsToDelete()`. When false, * operations deleted from surviving jobs are not reconciled (the parent-deletion * cascade below still runs). */ workOrderOperationsReconciled: boolean; /** * MM work order operations eligible for the handler: status RELEASED / ACTIVE, no * `closedDate`, parent work order a live candidate. Computed even when the handler * does not implement `selectWorkOrderOperationsToDelete()` (see * `workOrderOperationsReconciled`), so a connector can see how many operations it * would be offered by opting in. */ workOrderOperationCandidateCount: number; /** Operation keys the handler selected for deletion (ops gone from surviving jobs). */ workOrderOperationsToDelete: ErpWorkOrderOperationKey[]; /** * Operation keys marked DELETED because their parent work order is DELETED in MM * (by this run or an earlier one). Not counted against the blast-radius guard: the * parent's deletion was already guarded. */ cascadedWorkOrderOperationsToDelete: ErpWorkOrderOperationKey[]; /** * Size of the operation deletion set (selected + cascaded) applied to MM * (0 unless status is "completed"). */ markedDeletedWorkOrderOperationCount: number; } /** Which reconciliation phase a {@link ReconcileDeletedWorkOrdersAbortedError} refers to. */ export type ReconcileEntity = "workOrders" | "workOrderOperations"; /** * Thrown when the blast-radius guard refuses to mark the computed deletion set as * DELETED (typically a bad/partial ERP fetch). Nothing is written for the aborted * phase; the next scheduled cycle retries. Note that when `entity` is * `workOrderOperations`, the work order phase of the same run has already completed * (its writes are idempotent and stand). Mirrors the SDK convention of typed exceptions * for abnormal outcomes (cf. {@code MMBatchValidationError}). */ export class ReconcileDeletedWorkOrdersAbortedError extends Error { public readonly entity: ReconcileEntity; public readonly candidateCount: number; /** Populated when `entity` is `workOrders`; empty otherwise. */ public readonly workOrdersToDelete: ErpWorkOrderKey[]; /** Populated when `entity` is `workOrderOperations`; empty otherwise. */ public readonly workOrderOperationsToDelete: ErpWorkOrderOperationKey[]; public readonly deleteFraction: number; public readonly maxDeleteFraction: number; public readonly deleteGuardMinCount: number; constructor(options: { message: string; entity?: ReconcileEntity; candidateCount: number; workOrdersToDelete?: ErpWorkOrderKey[]; workOrderOperationsToDelete?: ErpWorkOrderOperationKey[]; deleteFraction: number; maxDeleteFraction: number; deleteGuardMinCount: number; }) { super(options.message); this.name = "ReconcileDeletedWorkOrdersAbortedError"; this.entity = options.entity ?? "workOrders"; this.candidateCount = options.candidateCount; this.workOrdersToDelete = options.workOrdersToDelete ?? []; this.workOrderOperationsToDelete = options.workOrderOperationsToDelete ?? []; this.deleteFraction = options.deleteFraction; this.maxDeleteFraction = options.maxDeleteFraction; this.deleteGuardMinCount = options.deleteGuardMinCount; } } /** Injection seam for tests; production callers omit it. @internal */ export interface WorkOrderReconcilerDeps { fetchAllWorkOrders: (pageSize?: number) => Promise; writeWorkOrders: ( records: MMSendWorkOrderStatusPatch[], connectorType: string ) => Promise; fetchAllWorkOrderOperations: ( pageSize?: number ) => Promise; writeWorkOrderOperations: ( records: MMSendWorkOrderOperationStatusPatch[], connectorType: string ) => Promise; } const withMMApiClient = async ( fn: (client: MMApiClient) => Promise ): Promise => { const mmApiClient = new MMApiClient(); try { return await fn(mmApiClient); } finally { try { await mmApiClient.destroy(); } catch (destroyError) { logger.warn("reconcileDeletedWorkOrders: failed to destroy MMApiClient", { error: destroyError, }); } } }; /** * Writes the deletion patches for one entity type, then clears that type's dedupe cache. * * Deletions are status-only patches (identity keys + `status`): the MM import derives * `update_columns` from the fields present in each record, so nothing but the status is * written back. Re-sending full rows read from MM would instead round-trip every column * through the SDK's number coercions (e.g. a NULL cycle time becoming 0) and touch * `closedDate`, which erp-api may soon treat as clearable on opt-in. * * The dedupe cache is a content-addressable hash set: the hash of each row's pre-deletion * payload stays in it, so a row later recreated in the ERP with an identical payload * would be locally deduped and stay DELETED in MM for up to CACHE_TTL. Selective removal * is impossible (old hashes came from connector-built send instances), so clear the * type's entries — unconditionally, because a partial-success write (HTTP 207 → * MMBatchValidationError) still marks some rows DELETED in MM before throwing. Clearing * after a complete failure is harmless: the cost is one re-send of each row on subsequent * from-erp cycles, the same as routine TTL expiry. */ const writeDeletionsAndClearCache = async ( objType: ERPObjType.WORK_ORDERS | ERPObjType.WORK_ORDER_OPERATIONS, records: (MMSendWorkOrderStatusPatch | MMSendWorkOrderOperationStatusPatch)[], connectorType: string ): Promise => { const cacheManager = new BatchCacheManager(); try { return await MMEntityProcessor.writeEntities(objType, records, cacheManager, { connectorType, }); } finally { try { await cacheManager.removeRecordsByType(objType); } catch (clearError) { logger.warn( `reconcileDeletedWorkOrders: failed to clear ${ERPObjType[objType]} dedupe cache; identically recreated rows may stay DELETED until CACHE_TTL expiry`, { error: clearError } ); } finally { await cacheManager.destroy(); } } }; const defaultDeps: WorkOrderReconcilerDeps = { fetchAllWorkOrders: (pageSize?: number) => withMMApiClient((client) => client.fetchAllErpWorkOrdersFromGraphQL({ pageSize })), writeWorkOrders: (records, connectorType) => writeDeletionsAndClearCache(ERPObjType.WORK_ORDERS, records, connectorType), fetchAllWorkOrderOperations: (pageSize?: number) => withMMApiClient((client) => client.fetchAllErpWorkOrderOperationsFromGraphQL({ pageSize }) ), writeWorkOrderOperations: (records, connectorType) => writeDeletionsAndClearCache( ERPObjType.WORK_ORDER_OPERATIONS, records, connectorType ), }; /** * Normalizes one composite key part: null/undefined → "", trimmed, uppercased. */ const normalizeKeyPart = (part: string | number | null | undefined): string => (part ?? "").toString().trim().toUpperCase(); /** * Builds the canonical composite key string for a work order identity. * Case-insensitive and whitespace-tolerant so key-cosmetic differences between the * ERP fetch and MM's stored values never produce a false deletion. */ export const normalizeWorkOrderKey = (key: ErpWorkOrderKey): string => [key.workOrderId, key.lot, key.split, key.sub].map(normalizeKeyPart).join("|"); /** * Normalizes a sequence number key part. MM stores `sequenceNumber` numerically and * renders it back as e.g. "10", while ERPs commonly format the same operation as "010", * "10.0" or the number 10 — so a numeric-looking value is canonicalized through Number() * and only non-numeric values fall back to text normalization. */ const normalizeSequenceNumberPart = ( part: string | number | null | undefined ): string => { const text = normalizeKeyPart(part); if (text === "") { return text; } const numeric = Number(text); return Number.isFinite(numeric) ? String(numeric) : text; }; /** * Builds the canonical composite key string for a work order operation identity: * {@link normalizeWorkOrderKey} plus the sequence number (numeric-canonical, so "010", * "10.0", 10 and "10" all match). */ export const normalizeWorkOrderOperationKey = ( key: ErpWorkOrderOperationKey ): string => `${normalizeWorkOrderKey(key)}|${normalizeSequenceNumberPart(key.sequenceNumber)}`; /** * True when the status is eligible for deletion reconciliation (case-insensitive, * see {@link normalizeMMStatus}). */ export const isReconcileCandidateStatus = ( status: string | null | undefined ): boolean => RECONCILE_CANDIDATE_STATUSES.has(normalizeMMStatus(status)); /** * True when a work order operation is eligible for deletion reconciliation on its own * account: candidate status and not closed. (Parent work order liveness is checked by * the reconciler, which has the work order set at hand.) */ export const isReconcileCandidateWorkOrderOperation = ( operation: Pick ): boolean => isReconcileCandidateStatus(operation.status) && (operation.closedDate === null || operation.closedDate === undefined || operation.closedDate === ""); /** * A usable key part is a non-blank string or a finite number (ERP drivers commonly return * numeric columns as numbers; the key normalizers stringify them). */ const isUsableKeyPart = (value: unknown): boolean => (typeof value === "string" && value.trim().length > 0) || (typeof value === "number" && Number.isFinite(value)); const countInvalidKeys = (keys: unknown[], fields: string[]): number => { let invalid = 0; for (const key of keys) { const record = key as Record | null | undefined; if (fields.some((field) => !isUsableKeyPart(record?.[field]))) { invalid++; } } return invalid; }; /** * Matching helper for the common {@code selectWorkOrdersToDelete()} implementation: * returns the candidates whose composite key is absent from `activeErpKeys` * (trimmed, case-insensitive — see {@link normalizeWorkOrderKey}). * * Guards against handler bugs instead of diffing garbage: throws when `activeErpKeys` * is not an array or contains a key without a usable `workOrderId` (e.g. a * wrong-field-name mapping bug would mangle EVERY key, empty the effective active set, * and on a small candidate set wrongly delete jobs below the blast-radius floor). */ export const selectWorkOrdersMissingFromErp = ( candidates: MMReceiveWorkOrder[], activeErpKeys: ErpWorkOrderKey[] ): MMReceiveWorkOrder[] => { if (!Array.isArray(activeErpKeys)) { throw new Error( "selectWorkOrdersMissingFromErp: activeErpKeys must be an array of work order keys" ); } const invalidKeyCount = countInvalidKeys(activeErpKeys, ["workOrderId"]); if (invalidKeyCount > 0) { throw new Error( `selectWorkOrdersMissingFromErp: ${invalidKeyCount} of ${activeErpKeys.length} active key(s) lack a usable workOrderId (non-blank string or finite number); treating the active list as unreliable` ); } const activeKeySet = new Set(activeErpKeys.map(normalizeWorkOrderKey)); return candidates.filter( (workOrder) => !activeKeySet.has(normalizeWorkOrderKey(workOrder)) ); }; /** * Work order operation counterpart of {@link selectWorkOrdersMissingFromErp}: returns * the candidate operations whose composite key (work order key + `sequenceNumber`) is * absent from `activeErpKeys`. Throws on a non-array or on keys lacking a usable * `workOrderId` or `sequenceNumber` (non-blank string or finite number). */ export const selectWorkOrderOperationsMissingFromErp = ( candidates: MMReceiveWorkOrderOperation[], activeErpKeys: ErpWorkOrderOperationKey[] ): MMReceiveWorkOrderOperation[] => { if (!Array.isArray(activeErpKeys)) { throw new Error( "selectWorkOrderOperationsMissingFromErp: activeErpKeys must be an array of work order operation keys" ); } const invalidKeyCount = countInvalidKeys(activeErpKeys, [ "workOrderId", "sequenceNumber", ]); if (invalidKeyCount > 0) { throw new Error( `selectWorkOrderOperationsMissingFromErp: ${invalidKeyCount} of ${activeErpKeys.length} active key(s) lack a usable workOrderId or sequenceNumber (non-blank string or finite number); treating the active list as unreliable` ); } const activeKeySet = new Set(activeErpKeys.map(normalizeWorkOrderOperationKey)); return candidates.filter( (operation) => !activeKeySet.has(normalizeWorkOrderOperationKey(operation)) ); }; /** Cap on deletion keys embedded in log entries; full lists live in results/errors. */ const LOGGED_KEY_SAMPLE_LIMIT = 20; const sampleKeysForLog = (keys: string[]): string[] => keys.slice(0, LOGGED_KEY_SAMPLE_LIMIT); const toWorkOrderKey = ( workOrder: Pick ): ErpWorkOrderKey => ({ workOrderId: workOrder.workOrderId, lot: workOrder.lot, split: workOrder.split, sub: workOrder.sub, }); const toWorkOrderOperationKey = ( operation: MMReceiveWorkOrderOperation ): ErpWorkOrderOperationKey => ({ ...toWorkOrderKey(operation), sequenceNumber: operation.sequenceNumber, }); interface GuardSettings { maxDeleteFraction: number; deleteGuardMinCount: number; } interface GuardEvaluation { deleteFraction: number; tripped: boolean; /** Human-readable `(x% > y% and count > n)` fragment for logs/errors. */ detail: string; } const evaluateGuard = ( toDeleteCount: number, candidateCount: number, settings: GuardSettings ): GuardEvaluation => { const deleteFraction = candidateCount === 0 ? 0 : toDeleteCount / candidateCount; return { deleteFraction, tripped: toDeleteCount > settings.deleteGuardMinCount && deleteFraction > settings.maxDeleteFraction, detail: `(${(deleteFraction * 100).toFixed(1)}% > ${(settings.maxDeleteFraction * 100).toFixed(1)}% and count > ${settings.deleteGuardMinCount})`, }; }; /** * Enforces the handler's subset contract: every returned row must be one of the * candidates (matched by normalized key), resolved back to the canonical candidate * instances so connector-side copies or mutations can't alter what gets written. * Repeated rows collapse to one. */ const resolveSelection = ( selected: unknown, candidateByKey: Map, keyOf: (row: T) => string, methodName: string ): T[] => { if (!Array.isArray(selected)) { throw new Error( `reconcileDeletedWorkOrders: ${methodName}() must return an array of candidate rows (and throw on ERP fetch failure)` ); } const selectedKeys = new Set(); const foreignKeys: string[] = []; for (const row of selected) { let key: string; try { key = keyOf((row ?? {}) as T); } catch { key = String(row); } if (candidateByKey.has(key)) { selectedKeys.add(key); } else { foreignKeys.push(key); } } if (foreignKeys.length > 0) { throw new Error( `reconcileDeletedWorkOrders: ${methodName}() returned ${foreignKeys.length} row(s) that are not among the candidates (e.g. ${foreignKeys.slice(0, 5).join(", ")}); treating the selection as unreliable — no deletions applied` ); } return [...selectedKeys].map((key) => candidateByKey.get(key)!); }; export class WorkOrderReconciler { /** * Marks MM work orders — and their operations — as DELETED when the connector no * longer reports them as active in the ERP. See * {@code StandardProcessDrivers.reconcileDeletedWorkOrders} for the public entry point * and contract details. * * Two phases, work orders first: * 1. Work orders: candidates (RELEASED / ACTIVE) → handler selection → guard → write. * 2. Work order operations: (a) operations of work orders DELETED in MM (by this run * or earlier) are cascaded to DELETED without consulting the handler or the guard; * (b) open RELEASED / ACTIVE operations of surviving candidate work orders are * offered to `selectWorkOrderOperationsToDelete()` when implemented, guarded, and * written together with the cascade set. */ static async reconcileDeleted( connectorType: string, handler: IERPWorkOrderReconciliationHandler, options?: ReconcileDeletedWorkOrdersOptions, deps: WorkOrderReconcilerDeps = defaultDeps ): Promise { const guard = WorkOrderReconciler.resolveGuardSettings(options); const dryRun = options?.dryRun ?? getEnvBoolean("RECONCILE_WORK_ORDERS_DRY_RUN", false); // ---------------------------------------------------------------- work orders const allWorkOrders = await deps.fetchAllWorkOrders(options?.pageSize); // Dedupe by composite key: offset paging can repeat a row when rows shift under // concurrent writes, and duplicated candidates would skew the guard's fraction math. const candidateByKey = new Map(); // Work orders already DELETED in MM: their operations are cascaded below. const deletedParentKeys = new Set(); for (const workOrder of allWorkOrders) { const key = normalizeWorkOrderKey(workOrder); if (normalizeMMStatus(workOrder.status) === MMWorkOrderStatus.DELETED) { deletedParentKeys.add(key); } else if (isReconcileCandidateStatus(workOrder.status)) { if (!candidateByKey.has(key)) { candidateByKey.set(key, workOrder); } } } const candidates = [...candidateByKey.values()]; logger.info("reconcileDeletedWorkOrders: fetched work orders from MM", { total: allWorkOrders.length, candidates: candidates.length, alreadyDeleted: deletedParentKeys.size, }); let toDelete: MMReceiveWorkOrder[] = []; if (candidates.length > 0) { // Contract: the handler throws on any ERP fetch failure and returns a subset of // the candidates. A non-array return is a handler bug, not an empty selection. const selected = await handler.selectWorkOrdersToDelete(candidates); toDelete = resolveSelection( selected, candidateByKey, normalizeWorkOrderKey, "selectWorkOrdersToDelete" ); } const workOrdersToDelete = toDelete.map(toWorkOrderKey); const workOrderKeyStrings = toDelete.map(normalizeWorkOrderKey); logger.info("reconcileDeletedWorkOrders: connector selection received", { candidates: candidates.length, toDelete: toDelete.length, }); const workOrderGuard = evaluateGuard(toDelete.length, candidates.length, guard); const workOrderPhase = { candidateCount: candidates.length, workOrdersToDelete, guardTripped: workOrderGuard.tripped, guardDetail: workOrderGuard.detail, }; if (!dryRun && workOrderGuard.tripped) { // Blast-radius guard: a bad/partial ERP fetch must not mass-delete jobs. const message = `reconcileDeletedWorkOrders: refusing to mark ${toDelete.length} of ${candidates.length} candidate work orders as DELETED ` + `${workOrderGuard.detail}. If this volume is intentional, raise RECONCILE_MAX_DELETE_FRACTION temporarily.`; logger.error(message, { sampleKeys: sampleKeysForLog(workOrderKeyStrings) }); throw new ReconcileDeletedWorkOrdersAbortedError({ message, entity: "workOrders", candidateCount: candidates.length, workOrdersToDelete, deleteFraction: workOrderGuard.deleteFraction, maxDeleteFraction: guard.maxDeleteFraction, deleteGuardMinCount: guard.deleteGuardMinCount, }); } if (!dryRun && toDelete.length > 0) { // Status-only patches: only `status` reaches the upsert's update_columns. const deletedRecords = toDelete.map( (workOrder) => new MMSendWorkOrderStatusPatch( workOrder.workOrderId, workOrder.lot, workOrder.split, workOrder.sub, MMWorkOrderStatus.DELETED ) ); const writeResult = await deps.writeWorkOrders(deletedRecords, connectorType); logger.info( `reconcileDeletedWorkOrders: marked ${toDelete.length} work orders as DELETED in MM`, { upsertedEntities: writeResult.upsertedEntities, localDedupeCount: writeResult.localDedupeCount, apiDedupeCount: writeResult.apiDedupeCount, markedDeleted: toDelete.length, sampleKeys: sampleKeysForLog(workOrderKeyStrings), } ); } // Parents deleted by this run join the cascade set; survivors are the live parents // whose operations the handler may be asked about. for (const key of workOrderKeyStrings) { deletedParentKeys.add(key); } const liveParentKeys = new Set( [...candidateByKey.keys()].filter((key) => !deletedParentKeys.has(key)) ); // ------------------------------------------------------ work order operations const operationPhase = await WorkOrderReconciler.reconcileOperations( connectorType, handler, { deletedParentKeys, liveParentKeys, guard, dryRun, pageSize: options?.pageSize }, deps ); if (dryRun) { const reasons: string[] = []; if (workOrderPhase.guardTripped) { reasons.push( `blast-radius guard would abort a real run on work orders ${workOrderPhase.guardDetail}` ); } if (operationPhase.guardTripped) { reasons.push( `blast-radius guard would abort a real run on work order operations ${operationPhase.guardDetail}` ); } // Dry run previews everything, including what the guard would refuse. logger.info( `reconcileDeletedWorkOrders: DRY RUN — would mark ${toDelete.length} work orders and ${operationPhase.selected.length + operationPhase.cascaded.length} work order operations as DELETED` + (reasons.length > 0 ? `; ${reasons.join("; ")}` : ""), { workOrdersToDelete: toDelete.length, workOrderOperationsToDelete: operationPhase.selected.length, cascadedWorkOrderOperations: operationPhase.cascaded.length, sampleWorkOrderKeys: sampleKeysForLog(workOrderKeyStrings), sampleWorkOrderOperationKeys: sampleKeysForLog( operationPhase.selected.map(normalizeWorkOrderOperationKey) ), } ); return { status: "dry-run", reason: reasons.length > 0 ? reasons.join("; ") : undefined, candidateCount: candidates.length, workOrdersToDelete, markedDeletedCount: 0, workOrderOperationsReconciled: operationPhase.handlerImplemented, workOrderOperationCandidateCount: operationPhase.candidateCount, workOrderOperationsToDelete: operationPhase.selected.map( toWorkOrderOperationKey ), cascadedWorkOrderOperationsToDelete: operationPhase.cascaded.map( toWorkOrderOperationKey ), markedDeletedWorkOrderOperationCount: 0, }; } return { status: "completed", candidateCount: candidates.length, workOrdersToDelete, markedDeletedCount: toDelete.length, workOrderOperationsReconciled: operationPhase.handlerImplemented, workOrderOperationCandidateCount: operationPhase.candidateCount, workOrderOperationsToDelete: operationPhase.selected.map( toWorkOrderOperationKey ), cascadedWorkOrderOperationsToDelete: operationPhase.cascaded.map( toWorkOrderOperationKey ), markedDeletedWorkOrderOperationCount: operationPhase.selected.length + operationPhase.cascaded.length, }; } private static resolveGuardSettings( options: ReconcileDeletedWorkOrdersOptions | undefined ): GuardSettings { // getEnvFloat, not getEnvNumber: the latter parses via parseInt and would turn "0.2" into 0. const rawMaxDeleteFraction = options?.maxDeleteFraction ?? getEnvFloat("RECONCILE_MAX_DELETE_FRACTION", 0.2); const rawDeleteGuardMinCount = options?.deleteGuardMinCount ?? getEnvNumber("RECONCILE_DELETE_GUARD_MIN_COUNT", 10); // These parameterize a safety control, so out-of-range values are corrected, not // trusted: fraction ∈ [0,1] (1 disables the fraction check — the deliberate // escape hatch for an intentional mass purge), count ≥ 0, non-finite → default. const maxDeleteFraction = Number.isFinite(rawMaxDeleteFraction) ? Math.min(Math.max(rawMaxDeleteFraction, 0), 1) : 0.2; const deleteGuardMinCount = Number.isFinite(rawDeleteGuardMinCount) ? Math.max(Math.floor(rawDeleteGuardMinCount), 0) : 10; if ( maxDeleteFraction !== rawMaxDeleteFraction || deleteGuardMinCount !== rawDeleteGuardMinCount ) { logger.warn( "reconcileDeletedWorkOrders: guard settings out of range; adjusted", { rawMaxDeleteFraction, maxDeleteFraction, rawDeleteGuardMinCount, deleteGuardMinCount, } ); } return { maxDeleteFraction, deleteGuardMinCount }; } /** * Operation phase. Fetches all operations from MM, cascades those of deleted parents, * offers the open operations of live parents to the handler (when implemented), and — * unless dry-running — writes the union as DELETED. */ private static async reconcileOperations( connectorType: string, handler: IERPWorkOrderReconciliationHandler, context: { deletedParentKeys: ReadonlySet; liveParentKeys: ReadonlySet; guard: GuardSettings; dryRun: boolean; pageSize?: number; }, deps: WorkOrderReconcilerDeps ): Promise<{ handlerImplemented: boolean; candidateCount: number; selected: MMReceiveWorkOrderOperation[]; cascaded: MMReceiveWorkOrderOperation[]; guardTripped: boolean; guardDetail: string; }> { const handlerImplemented = typeof handler.selectWorkOrderOperationsToDelete === "function"; if (!handlerImplemented) { logger.info( "reconcileDeletedWorkOrders: handler does not implement selectWorkOrderOperationsToDelete(); operations removed from surviving jobs are not reconciled (operations of deleted jobs still are)" ); } const allOperations = await deps.fetchAllWorkOrderOperations(context.pageSize); const cascadedByKey = new Map(); const candidateByKey = new Map(); for (const operation of allOperations) { if (!isReconcileCandidateWorkOrderOperation(operation)) { continue; } const parentKey = normalizeWorkOrderKey(operation); const key = normalizeWorkOrderOperationKey(operation); if (context.deletedParentKeys.has(parentKey)) { if (!cascadedByKey.has(key)) { cascadedByKey.set(key, operation); } } else if (context.liveParentKeys.has(parentKey)) { if (!candidateByKey.has(key)) { candidateByKey.set(key, operation); } } // Operations of non-candidate parents (COMPLETED, CANCELLED, unknown to MM, …) // are left alone: the connector's active-operation list would not cover them. } const cascaded = [...cascadedByKey.values()]; const candidates = [...candidateByKey.values()]; logger.info("reconcileDeletedWorkOrders: fetched work order operations from MM", { total: allOperations.length, candidates: candidates.length, cascadedFromDeletedWorkOrders: cascaded.length, }); let selected: MMReceiveWorkOrderOperation[] = []; if (handlerImplemented && candidates.length > 0) { const returned = await handler.selectWorkOrderOperationsToDelete!(candidates); selected = resolveSelection( returned, candidateByKey, normalizeWorkOrderOperationKey, "selectWorkOrderOperationsToDelete" ); logger.info( "reconcileDeletedWorkOrders: connector operation selection received", { candidates: candidates.length, toDelete: selected.length } ); } const selectedKeyStrings = selected.map(normalizeWorkOrderOperationKey); // Guard only the handler's selection: the cascade follows already-guarded parent // deletions and would otherwise trip the guard on any job with many operations. const operationGuard = evaluateGuard(selected.length, candidates.length, context.guard); if (context.dryRun) { return { handlerImplemented, candidateCount: candidates.length, selected, cascaded, guardTripped: operationGuard.tripped, guardDetail: operationGuard.detail, }; } if (operationGuard.tripped) { const message = `reconcileDeletedWorkOrders: refusing to mark ${selected.length} of ${candidates.length} candidate work order operations as DELETED ` + `${operationGuard.detail}. Work order deletions of this run already stand. If this volume is intentional, raise RECONCILE_MAX_DELETE_FRACTION temporarily.`; logger.error(message, { sampleKeys: sampleKeysForLog(selectedKeyStrings) }); throw new ReconcileDeletedWorkOrdersAbortedError({ message, entity: "workOrderOperations", candidateCount: candidates.length, workOrderOperationsToDelete: selected.map(toWorkOrderOperationKey), deleteFraction: operationGuard.deleteFraction, maxDeleteFraction: context.guard.maxDeleteFraction, deleteGuardMinCount: context.guard.deleteGuardMinCount, }); } const toDelete = [...cascaded, ...selected]; if (toDelete.length > 0) { const deletedRecords = toDelete.map( (operation) => new MMSendWorkOrderOperationStatusPatch( operation.workOrderId, operation.lot, operation.split, operation.sub, operation.sequenceNumber, MMWorkOrderOperationStatus.DELETED ) ); const writeResult = await deps.writeWorkOrderOperations( deletedRecords, connectorType ); logger.info( `reconcileDeletedWorkOrders: marked ${toDelete.length} work order operations as DELETED in MM (${selected.length} selected by the connector, ${cascaded.length} cascaded from deleted work orders)`, { upsertedEntities: writeResult.upsertedEntities, localDedupeCount: writeResult.localDedupeCount, apiDedupeCount: writeResult.apiDedupeCount, markedDeleted: toDelete.length, sampleKeys: sampleKeysForLog(toDelete.map(normalizeWorkOrderOperationKey)), } ); } return { handlerImplemented, candidateCount: candidates.length, selected, cascaded, guardTripped: false, guardDetail: operationGuard.detail, }; } }