import fs from 'fs'; import path from 'path'; import logger from '../../services/reporting-service/logger.js'; import type { LogEntry, LogLevelString } from './mm-connector-logger.js'; /** * Implement a custom LogDeduper or use the provided FileLogDeduper */ export interface LogDeduper { decide(entry: LogEntry, now: number): Promise; onSuccess(entry: LogEntry, now: number): Promise; retryFailedTransmissions?(send: (entry: LogEntry, message: string) => Promise): Promise; } interface LogDedupeEntry { lastTransmitted: number; suppressedCount: number; firstUnsentEventTs: number; // Earliest unsent occurrence time for the current unsent batch (if any) lastEventTs: number; // Most recent occurrence time (allowed or suppressed) ttlMs?: number; // TTL the entry was written under; sweeps honor this, not the sweeper's TTL (absent in pre-upgrade stores) level: LogLevelString; message: string; } /** * LogDedupeStore is the store for of the most recent log entries for each dedupeKey */ interface LogDedupeStore { [key: string]: LogDedupeEntry; } /** * FileLogDeduper implements the LogDeduper interface to deduplicate * logs by storing them in a file; this allows deduping across jobs. */ export class FileLogDeduper implements LogDeduper { private readonly storeFilePath: string; private readonly windowMs: number; private readonly ttlMs: number; private readonly sweepIntervalMs: number; private lastSweepTsMs: number; private readonly DEFAULT_WINDOW_TEN_MINS = 600; private readonly DEFAULT_TTL_ONE_HOUR = 3600; private readonly DEFAULT_SWEEP_INTERVAL_FIVE_MINS = 300; private readonly DEFAULT_STORE_FILE_PATH = path.join('/tmp', 'log-deduplication.json'); /** * Ctor. * @param storeFilePath: The path to the file where the deduplication store is stored; recommended is to use the default * @param windowSeconds: Suppression window. Duplicates within this period are suppressed. * @param ttlSeconds: Eviction TTL. Store entries for keys inactive beyond this are removed. Enforced to be ≥ windowSeconds. * Stored per entry, so instances with different TTLs can safely share one store file. * @param sweepIntervalSeconds: Efficiency parameter. How often (min interval) to run opportunistic eviction; retry always sweeps * The sweep is lazy, used only when the store is accessed */ constructor({ storeFilePath = this.DEFAULT_STORE_FILE_PATH, windowSeconds = this.DEFAULT_WINDOW_TEN_MINS, ttlSeconds = this.DEFAULT_TTL_ONE_HOUR, sweepIntervalSeconds = this.DEFAULT_SWEEP_INTERVAL_FIVE_MINS } : { storeFilePath?: string; windowSeconds?: number; ttlSeconds?: number; sweepIntervalSeconds?: number; } = {}) { this.storeFilePath = storeFilePath; this.windowMs = Math.max(1, windowSeconds) * 1000; this.ttlMs = Math.max(this.windowMs, Math.max(1, ttlSeconds) * 1000); this.sweepIntervalMs = Math.max(1, sweepIntervalSeconds) * 1000; this.lastSweepTsMs = 0; this.ensureStoreFileExists(); } /** * Deduplication gating function * Returns the formatted message to send, or null to suppress * Decision is based on the dedupeKey and the time of the entry */ async decide(entry: LogEntry, now: number): Promise { if (!entry.dedupeKey || typeof entry.dedupeKey !== 'string' || entry.dedupeKey.trim().length === 0) { throw new Error('dedupeKey is required and must be a non-empty string'); } const key = entry.dedupeKey; return this.withLock(async () => { const store = this.readStore(); // Opportunistic eviction gated by sweep interval if (now - this.lastSweepTsMs >= this.sweepIntervalMs) { this.evictExpiredInStore(store, now); this.lastSweepTsMs = now; this.writeStore(store); } const existing = store[key]; if (existing) { const withinWindow = existing.lastTransmitted > 0 && (existing.lastTransmitted + this.windowMs) > now; if (withinWindow) { // Suppress and increment count store[key] = { ...existing, suppressedCount: existing.suppressedCount + 1, firstUnsentEventTs: existing.suppressedCount === 0 ? (entry.eventTime ?? now) : existing.firstUnsentEventTs, lastEventTs: entry.eventTime ?? now, ttlMs: this.ttlMs, level: entry.level, message: entry.message }; this.writeStore(store); return null; } // Outside window: allow, include timestamp and prior suppression count const messageToSend = this.formatMessage(entry.message, entry.eventTime ?? now, existing.suppressedCount, existing.firstUnsentEventTs); store[key] = { ...existing, suppressedCount: 0, firstUnsentEventTs: 0, lastEventTs: entry.eventTime ?? now, ttlMs: this.ttlMs, level: entry.level, message: entry.message }; this.writeStore(store); return messageToSend; } // New entry: allow, timestamp, no suppression count const messageToSend = this.formatMessage(entry.message, entry.eventTime ?? now, 0); store[key] = { lastTransmitted: 0, suppressedCount: 0, firstUnsentEventTs: entry.eventTime ?? now, lastEventTs: entry.eventTime ?? now, ttlMs: this.ttlMs, level: entry.level, message: entry.message }; this.writeStore(store); return messageToSend; }); } async onSuccess(entry: LogEntry, now: number): Promise { if (!entry.dedupeKey || typeof entry.dedupeKey !== 'string' || entry.dedupeKey.trim().length === 0) { throw new Error('dedupeKey is required and must be a non-empty string'); } const key = entry.dedupeKey; await this.withLock(async () => { const store = this.readStore(); const existing = store[key]; if (existing) { store[key] = { ...existing, lastTransmitted: now, firstUnsentEventTs: 0, suppressedCount: 0, ttlMs: this.ttlMs }; this.writeStore(store); } }); } async retryFailedTransmissions(send: (entry: LogEntry, message: string) => Promise): Promise { // Best-effort retry: first sweep expired entries, then iterate over entries with lastTransmitted === 0 const now = Date.now(); const entries = await this.withLock(async () => { const store = this.readStore(); // Always sweep on retry this.evictExpiredInStore(store, now); this.lastSweepTsMs = now; this.writeStore(store); return Object.entries(store) .filter(([, rec]) => rec.lastTransmitted === 0) .map(([key, rec]) => ({ key, rec })); }); for (const { key, rec } of entries) { try { const message = this.formatMessage(rec.message, rec.lastEventTs, rec.suppressedCount, rec.firstUnsentEventTs); await send({ level: rec.level, message: rec.message, dedupeKey: key, eventTime: rec.lastEventTs }, message); // Mark success await this.withLock(async () => { const store = this.readStore(); const current = store[key]; if (current) { store[key] = { ...current, lastTransmitted: Date.now(), suppressedCount: 0, ttlMs: this.ttlMs }; this.writeStore(store); } }); } catch (err) { // Leave entry for future retries; do not throw logger.error('Failed to retry failed transmission', { key, rec, error: err }); return; } } } // --- Internals --- private ensureStoreFileExists(): void { try { if (!fs.existsSync(this.storeFilePath)) { fs.writeFileSync(this.storeFilePath, JSON.stringify({}), 'utf-8'); } } catch { // ignore } } private readStore(): LogDedupeStore { try { if (!fs.existsSync(this.storeFilePath)) return {}; const content = fs.readFileSync(this.storeFilePath, 'utf-8'); return content ? (JSON.parse(content) as LogDedupeStore) : {}; } catch { return {}; } } private writeStore(store: LogDedupeStore): void { try { fs.writeFileSync(this.storeFilePath, JSON.stringify(store, null, 2), 'utf-8'); } catch { // ignore } } private formatMessage(message: string, eventTs: number, suppressedCount: number, firstUnsentEventTs?: number): string { const timestamp = new Date(eventTs).toISOString(); const base = `${timestamp} | ${message}`; if (suppressedCount > 0) { const since = firstUnsentEventTs && firstUnsentEventTs > 0 ? ` since ${new Date(firstUnsentEventTs).toISOString()}` : ''; return `${base} (${suppressedCount} suppressed${since})`; } return base; } private async withLock(fn: () => Promise): Promise { const lockPath = `${this.storeFilePath}.lock`; const start = Date.now(); // Try to acquire the lock with retries (best effort) // Avoid infinite waits; cap at ~3 seconds while (true) { try { const fd = fs.openSync(lockPath, 'wx'); try { const result = await fn(); return result; } finally { try { fs.closeSync(fd); } catch {/* ignore */} try { fs.unlinkSync(lockPath); } catch {/* ignore */} } } catch { // Lock exists if (Date.now() - start > 3000) { // Timed out; proceed without lock to avoid deadlock return await fn(); } await new Promise(resolve => setTimeout(resolve, 50)); } } } /** * Evict expired entries from the store based on each entry's own TTL and the key's last transmitted time. * Entries written by pre-upgrade versions carry no ttlMs; fall back to the sweeper's TTL for those. */ private evictExpiredInStore(store: LogDedupeStore, now: number): void { const keys = Object.keys(store); if (keys.length === 0) return; for (const key of keys) { const rec = store[key]; const referenceTs = rec.lastTransmitted > 0 ? rec.lastTransmitted : rec.lastEventTs; if (now - referenceTs > (rec.ttlMs ?? this.ttlMs)) { delete store[key]; } } } }